const test = require('brittle') const crypto = require('hypercore-crypto') const HyperDHT = require('hyperdht') const setupTestnet = require('hyperdht/testnet') const promClient = require('prom-client') const c = require('compact-encoding') const b4a = require('b4a') const Protomux = require('protomux') const process = require('process') const { getEncoding } = require('./spec/hyperschema') const { HyperTracker, HyperTrackerClient, HyperTrackerMultiClient } = require('.') const [NS_ANNOUNCE] = crypto.namespace('hyperdiscovery', 1) const AnnouncePayload = getEncoding('@hyperdiscovery/announce') test('subscriber receives event from announcer', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const subscriber = new HyperTrackerClient(server.publicKey, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() const announced = t.test('announce event') announced.plan(1) subscriber.on('announce', (ann) => { announced.alike(ann.publicKey, keyPair.publicKey, 'announced public key matches') }) subscriber.subscribe(keyPair.publicKey) announcer.announce(keyPair) await announced }) test('lookup returns the latest announce', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() t.alike(await server.lookup(keyPair.publicKey), null, 'no record') const bump = Date.now() announcer.announce(keyPair, { bump }) const record = await waitForRecord(server, keyPair.publicKey) t.ok(record, 'record exists after announce') t.is(record.bumped, bump, 'stores the bump timestamp') const bump2 = bump + 10_000 announcer.announce(keyPair, { bump: bump2 }) await new Promise((resolve) => setTimeout(resolve, 800)) const updated = await server.lookup(keyPair.publicKey) t.is(updated.bumped, bump2, 'lookup reflects a newer announce') }) test('unsubscribe stops future announces', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const subscriber = new HyperTrackerClient(server.publicKey, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() let receivedCount = 0 const firstAnnounce = t.test('first announce while subscribed') firstAnnounce.plan(1) subscriber.on('announce', () => { receivedCount++ if (receivedCount === 1) firstAnnounce.pass('subscriber receives an announce while subscribed') }) subscriber.subscribe(keyPair.publicKey) announcer.announce(keyPair, { bump: Date.now() }) await firstAnnounce subscriber.unsubscribe(keyPair.publicKey) announcer.announce(keyPair, { bump: Date.now() + 10_000 }) await new Promise((resolve) => setTimeout(resolve, 1500)) t.is(receivedCount, 1, 'no further announce received after unsubscribing') }) test('stale announce does not overwrite a newer record', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() const bump = Date.now() announcer.announce(keyPair, { bump }) const record = await waitForRecord(server, keyPair.publicKey) t.ok(record, 'record exists after first announce') announcer.announce(keyPair, { bump: bump - 10_000 }) await new Promise((resolve) => setTimeout(resolve, 800)) const stale = await server.lookup(keyPair.publicKey) t.is(stale.bumped, record.bumped, 'bump is unchanged by an older announce') t.is(stale.updated, record.updated, 'record is untouched by an older announce') const newerBump = bump + 10_000 announcer.announce(keyPair, { bump: newerBump }) await new Promise((resolve) => setTimeout(resolve, 800)) const updated = await server.lookup(keyPair.publicKey) t.is(updated.bumped, newerBump, 'a genuinely newer announce still updates the record') }) test('concurrent announces keep the newest bump', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const KEYS = 5 const PER_KEY = 100 const base = Date.now() const newest = base + PER_KEY - 1 const keys = [] const pending = [] // Collisions happen per key, so depth per key is what matters -- a few keys // with many announces each, not many keys with two announces each. for (let i = 0; i < KEYS; i++) { const keyPair = crypto.keyPair() keys.push(keyPair) for (let bump = base; bump <= newest; bump++) { pending.push(server.announce(signAnnounce(keyPair, bump), null)) } } await Promise.all(pending) for (const keyPair of keys) { const record = await server.lookup(keyPair.publicKey) t.is(record && record.bumped, newest, 'newest bump wins') } }) test('rejects announce bumps too far in the future', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() // TIME_SLACK is 60s; a bump far beyond that must be rejected. announcer.announce(keyPair, { bump: Date.now() + 120_000 }) await waitFor(() => server.stats.onannounceCount === 1) t.is(server.stats.announces, 0, 'future bump is not accepted') t.alike(await server.lookup(keyPair.publicKey), null, 'no record stored for future bump') // A bump within the slack window is still accepted. const withinSlack = Date.now() + 30_000 announcer.announce(keyPair, { bump: withinSlack }) const record = await waitForRecord(server, keyPair.publicKey) t.is(server.stats.announces, 1, 'bump within slack is accepted') t.is(record.bumped, withinSlack, 'stores the within-slack bump') }) test('subscribing after an announce delivers the current record', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() const bump = Date.now() announcer.announce(keyPair, { bump }) await waitForRecord(server, keyPair.publicKey) const subscriber = new HyperTrackerClient(server.publicKey, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const caughtUp = t.test('catch-up announce') caughtUp.plan(1) subscriber.on('announce', (ann) => { caughtUp.is(ann.bumped, bump, 'delivers the already-stored bump on subscribe') }) subscriber.subscribe(keyPair.publicKey) await caughtUp }) test('stats', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() t.alike( server.stats, { streamsAdded: 0, streamsEnded: 0, subscriptionsAdded: 0, subscriptionsRemoved: 0, announces: 0, announcesSent: 0, flushErrors: 0, onsubscribeCount: 0, onunsubscribeCount: 0, onannounceCount: 0 }, 'stats start at zero' ) const subscriber = new HyperTrackerClient(server.publicKey, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() const announced = t.test('announce event') announced.plan(1) subscriber.on('announce', (ann) => { announced.alike(ann.publicKey, keyPair.publicKey, 'announced public key matches') }) subscriber.subscribe(keyPair.publicKey) announcer.announce(keyPair) await announced t.is(server.stats.streamsAdded, 2, 'two streams added') t.is(server.stats.streamsEnded, 0, 'no streams ended yet') t.is(server.stats.subscriptionsAdded, 1, 'one subscription added') t.is(server.stats.subscriptionsRemoved, 0, 'no subscriptions removed yet') t.is(server.stats.announces, 1, 'one successful announce') t.is(server.stats.announcesSent, 1, 'one bump sent to subscriber') t.is(server.stats.onsubscribeCount, 1, 'one subscribe message') t.is(server.stats.onannounceCount, 1, 'one announce message') t.is(server.stats.onunsubscribeCount, 0, 'no unsubscribe yet') subscriber.unsubscribe(keyPair.publicKey) await waitFor(() => server.stats.onunsubscribeCount === 1) t.is(server.stats.onunsubscribeCount, 1, 'one unsubscribe message') t.is(server.stats.subscriptionsRemoved, 1, 'one subscription removed') await subscriber.close() await waitFor(() => server.stats.streamsEnded === 1) t.is(server.stats.streamsAdded, 2, 'stream add count is cumulative') t.is(server.stats.streamsEnded, 1, 'subscriber stream ended') t.is(server.stats.subscriptionsAdded, 1, 'subscription add count is cumulative') t.is(server.stats.subscriptionsRemoved, 1, 'subscription remove count is cumulative') }) test('metrics', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() promClient.register.clear() server.registerMetrics(promClient) t.teardown(() => promClient.register.clear()) const subscriber = new HyperTrackerClient(server.publicKey, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() subscriber.subscribe(keyPair.publicKey) subscriber.subscribe(keyPair.publicKey) await waitFor(() => server.stats.onsubscribeCount === 2) announcer.announce(keyPair) await waitFor(() => server.stats.announces === 1) subscriber.unsubscribe(keyPair.publicKey) await waitFor(() => server.stats.onunsubscribeCount === 1) await subscriber.close() await waitFor(() => server.stats.streamsEnded === 1) const metrics = await promClient.register.metrics() t.ok(metrics.includes('hypertracker_streams_added 2'), 'streams added metric') t.ok(metrics.includes('hypertracker_streams_ended 1'), 'streams ended metric') t.ok( metrics.includes('hypertracker_subscriptions_added 1'), 'duplicate subscribe is not counted as added' ) t.ok(metrics.includes('hypertracker_subscriptions_removed 1'), 'subscriptions removed metric') t.ok(metrics.includes('hypertracker_subscription_keys 0'), 'subscription keys metric') t.ok(metrics.includes('hypertracker_announces 1'), 'announces metric') t.ok(metrics.includes('hypertracker_announces_sent 1'), 'announces sent metric') t.ok(metrics.includes('hypertracker_flush_errors 0'), 'flush errors metric') t.ok(metrics.includes('hypertracker_onsubscribe_count 2'), 'onsubscribe count metric') t.ok(metrics.includes('hypertracker_onunsubscribe_count 1'), 'onunsubscribe count metric') t.ok(metrics.includes('hypertracker_onannounce_count 1'), 'onannounce count metric') }) test('accepted announces survive a graceful close', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const keyPair = crypto.keyPair() const bump = Date.now() announcer.announce(keyPair, { bump }) const record = await waitForRecord(server, keyPair.publicKey) t.ok(record, 'record is visible before close') t.is(record && record.bumped, bump, 'bump is stored before close') await server.close() const reopenDht = new HyperDHT({ bootstrap }) t.teardown(() => reopenDht.destroy(), { order: 4000 }) const reopened = new HyperTracker(storage, { dht: reopenDht }) t.teardown(() => reopened.close(), { order: 2500 }) await reopened.ready() const persisted = await reopened.lookup(keyPair.publicKey) t.ok(persisted, 'record still exists after reopening the same storage') t.is(persisted && persisted.bumped, bump, 'bump survived the graceful close') }) test('a failed flush does not stop future flushes', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() // The write lock keeps flushes off in-flight inserts, but the underlying // commit can still fail on a disk error. That must cost one cycle, not the // whole process: a dead flush loop leaks the pending batch forever. const realFlush = server.db.flush.bind(server.db) let thrown = false server.db.flush = () => { if (thrown) return realFlush() thrown = true return Promise.reject(new Error('disk I/O error from commit')) } const keyPair = crypto.keyPair() const bump = Date.now() await server.announce(signAnnounce(keyPair, bump), null) await server._flush() t.is(server.stats.flushErrors, 1, 'the failure is counted rather than swallowed') await server._flush() t.is(server.stats.flushErrors, 1, 'the next flush succeeds') t.is(server.db.updates.size, 0, 'the pending batch drains instead of growing forever') const persisted = await server.lookup(keyPair.publicKey) t.is(persisted && persisted.bumped, bump, 'the write survived the failed flush') }) test('closing with announces in flight persists every pending write', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) await server.ready() const KEYS = 20 const PER_KEY = 20 const base = Date.now() const newest = base + PER_KEY - 1 const keyPairs = [] const announcing = [] for (let i = 0; i < KEYS; i++) keyPairs.push(crypto.keyPair()) // Several announces per key, so each per-key chain is genuinely backed up // when close starts. One announce per key never queues deeper than 1 and // drains on its own, which hides whether close waits for anything. for (let bump = base; bump <= newest; bump++) { for (const keyPair of keyPairs) { announcing.push(server.announce(signAnnounce(keyPair, bump), null)) } } await server.close() const results = await Promise.allSettled(announcing) const accepted = results.filter((r) => r.status === 'fulfilled' && r.value === true).length const reopened = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => reopened.close(), { order: 3000 }) await reopened.ready() let persisted = 0 for (const keyPair of keyPairs) { const record = await reopened.lookup(keyPair.publicKey) if (record && record.bumped === newest) persisted++ } t.is(accepted, KEYS * PER_KEY, 'the drain let every queued announce finish') t.is(persisted, KEYS, 'every key persisted its newest bump') }) // Builds the wire form of an announce so tests can call server.announce() // directly, without routing through a client and the network. function signAnnounce(keyPair, bump) { const announce = { bump } const state = { buffer: null, start: 0, end: 0 } c.fixed32.preencode(state, NS_ANNOUNCE) AnnouncePayload.preencode(state, announce) state.buffer = b4a.allocUnsafe(state.end) c.fixed32.encode(state, NS_ANNOUNCE) AnnouncePayload.encode(state, announce) return { publicKey: keyPair.publicKey, announce, signature: crypto.sign(state.buffer, keyPair.secretKey) } } test('multi client emits connect once per tracker, carrying the client', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const multi = new HyperTrackerMultiClient( trackers.map((tracker) => tracker.publicKey), { dht } ) t.teardown(() => multi.close(), { order: 2000 }) const seen = [] multi.on('connect', (client) => seen.push(client)) multi.connect() await waitFor(() => seen.length === 3) t.is(seen.length, 3, 'one connect per tracker') t.is(new Set(seen).size, 3, 'each event carried a distinct client') t.ok( seen.every((client) => multi.clients.includes(client)), 'every event carried one of our own clients' ) await waitFor(() => trackers.every((tracker) => tracker.stats.streamsAdded === 1)) t.ok( trackers.every((tracker) => tracker.stats.streamsAdded === 1), 'every tracker really accepted a stream' ) }) test('multi client announces to every tracker with one shared bump', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const multi = new HyperTrackerMultiClient( trackers.map((tracker) => tracker.publicKey), { dht } ) t.teardown(() => multi.close(), { order: 2000 }) let connected = 0 multi.on('connect', () => connected++) multi.connect() await waitFor(() => connected === 3) const keyPair = crypto.keyPair() await multi.announce(keyPair) const bumps = [] for (const tracker of trackers) { const record = await waitForRecord(tracker, keyPair.publicKey) t.ok(record, 'tracker stored the announce') bumps.push(record.bumped) } t.is(new Set(bumps).size, 1, 'all three trackers stored an identical bump') }) test('multi client emits a single announce event for all trackers', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const keys = trackers.map((tracker) => tracker.publicKey) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) await subscriberDht.ready() const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) await announcerDht.ready() const subscriber = new HyperTrackerMultiClient(keys, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerMultiClient(keys, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) let connected = 0 subscriber.on('connect', () => connected++) subscriber.connect() await waitFor(() => connected === 3) const keyPair = crypto.keyPair() let events = 0 subscriber.on('announce', () => events++) subscriber.subscribe(keyPair.publicKey) await announcer.announce(keyPair, { bump: Date.now() }) await waitFor(() => events > 0) await new Promise((resolve) => setTimeout(resolve, 1500)) t.is(events, 1, 'three trackers echoed the same bump, one event surfaced') await announcer.announce(keyPair, { bump: Date.now() + 10_000 }) await waitFor(() => events === 2) t.is(events, 2, 'a newer bump still surfaces') }) test('multi client unsubscribe stops announces on every tracker', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const keys = trackers.map((tracker) => tracker.publicKey) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) await subscriberDht.ready() const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) await announcerDht.ready() const subscriber = new HyperTrackerMultiClient(keys, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerMultiClient(keys, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) let connected = 0 subscriber.on('connect', () => connected++) subscriber.connect() await waitFor(() => connected === 3) const keyPair = crypto.keyPair() let events = 0 subscriber.on('announce', () => events++) subscriber.subscribe(keyPair.publicKey) await announcer.announce(keyPair, { bump: Date.now() }) await waitFor(() => events === 1) subscriber.unsubscribe(keyPair.publicKey) await announcer.announce(keyPair, { bump: Date.now() + 10_000 }) await new Promise((resolve) => setTimeout(resolve, 1500)) t.is(events, 1, 'no further announce after unsubscribing from every tracker') }) test('multi client keeps working when a tracker dies', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const subscriberDht = new HyperDHT({ bootstrap }) t.teardown(() => subscriberDht.destroy(), { order: 4000 }) await subscriberDht.ready() const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) await announcerDht.ready() const keys = trackers.map((tracker) => tracker.publicKey) const subscriber = new HyperTrackerMultiClient(keys, { dht: subscriberDht }) t.teardown(() => subscriber.close(), { order: 2000 }) const announcer = new HyperTrackerMultiClient(keys, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const unhandled = [] const onUnhandled = (err) => unhandled.push(err) process.on('unhandledRejection', onUnhandled) t.teardown(() => process.off('unhandledRejection', onUnhandled)) let connected = 0 subscriber.on('connect', () => connected++) subscriber.connect() announcer.connect() await waitFor(() => connected === 3) await waitFor(() => trackers.every((tracker) => tracker.stats.streamsAdded >= 2)) let events = 0 subscriber.on('announce', () => events++) const alive = crypto.keyPair() subscriber.subscribe(alive.publicKey) await announcer.announce(alive, { bump: Date.now() }) await waitFor(() => events === 1) await trackers[0].close() t.execution(() => subscriber.subscribe(crypto.keyPair().publicKey), 'subscribe survives') t.execution(() => subscriber.unsubscribe(alive.publicKey), 'unsubscribe survives') t.execution(() => subscriber.connect(), 'connect survives') await t.execution(announcer.announce(crypto.keyPair()), 'announce resolves') await t.execution(announcer.suspend(), 'suspend resolves') t.execution(() => announcer.resume(), 'resume survives') const after = crypto.keyPair() subscriber.subscribe(after.publicKey) await announcer.announce(after, { bump: Date.now() }) await waitFor(() => events === 2) t.is(events, 2, 'announces still flow through the surviving trackers') await new Promise((resolve) => setTimeout(resolve, 1000)) t.is( unhandled.length, 0, `no unhandled rejections (${unhandled.map((e) => e.message).join('; ')})` ) }) test('cork batches subscribes until uncork', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const client = new HyperTrackerClient(server.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) client.connect() await new Promise((resolve) => client.on('connect', resolve)) await waitFor(() => server.stats.streamsAdded === 1) client.cork() for (let i = 0; i < 10; i++) client.subscribe(crypto.keyPair().publicKey) await new Promise((resolve) => setTimeout(resolve, 500)) t.is(server.stats.onsubscribeCount, 0, 'nothing reaches the tracker while corked') client.uncork() await waitFor(() => server.stats.onsubscribeCount === 10) t.is(server.stats.onsubscribeCount, 10, 'uncork flushes every subscribe') }) test('cork nests, so only the outermost uncork flushes', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const client = new HyperTrackerClient(server.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) client.connect() await new Promise((resolve) => client.on('connect', resolve)) await waitFor(() => server.stats.streamsAdded === 1) client.cork() client.cork() client.subscribe(crypto.keyPair().publicKey) client.uncork() await new Promise((resolve) => setTimeout(resolve, 500)) t.is(server.stats.onsubscribeCount, 0, 'inner uncork does not flush') client.uncork() await waitFor(() => server.stats.onsubscribeCount === 1) t.is(server.stats.onsubscribeCount, 1, 'outer uncork flushes') }) test('multi client setKeys replaces the tracker set', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const multi = new HyperTrackerMultiClient([trackers[0].publicKey, trackers[2].publicKey], { dht }) t.teardown(() => multi.close(), { order: 2000 }) const [first, second] = multi.clients t.is(multi.clients.length, 2, 'sanity: 2 clients for 2 keys') multi.setKeys([trackers[1].publicKey, trackers[0].publicKey]) t.is(multi.clients.length, 2, '2 clients for 2 keys') t.is(multi.clients[0], first, 'client[0] stayed as same reference') t.ok( b4a.equals(multi.clients[1].remotePublicKey, trackers[1].publicKey), 'client[1] is a new tracker[1]' ) t.ok(second.closed || second.closing, 'previously added tracker[2] is now closed') }) test('multi client setKeys connects a new tracker only when explicitly called', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const multi = new HyperTrackerMultiClient([trackers[0].publicKey], { dht }) t.teardown(() => multi.close(), { order: 2000 }) let seen = [] multi.on('connect', (client) => seen.push(client)) multi.setKeys([trackers[1].publicKey]) await new Promise((resolve) => setTimeout(resolve, 500)) t.is(seen.length, 0, 'setKeys before connect() does not try to connect') multi.connect() await waitFor(() => seen.length === 1) seen = [] multi.setKeys([trackers[1].publicKey, trackers[2].publicKey]) t.is(multi.clients.length, 2) multi.connect() await waitFor(() => seen.length === 1) await new Promise((resolve) => setTimeout(resolve, 500)) t.is(seen.length, 1) t.is(seen[0], multi.clients[1], 'only new key emitted connect') t.is(trackers[1].stats.streamsAdded, 1, 'old tracker did not reconnect') }) test('multi client setKeys during suspend connects on resume', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const trackers = await createTrackers(t, bootstrap, 3) const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const multi = new HyperTrackerMultiClient([trackers[0].publicKey], { dht }) t.teardown(() => multi.close(), { order: 2000 }) let seen = [] multi.on('connect', (client) => seen.push(client)) multi.connect() await waitFor(() => seen.length === 1) seen = [] await multi.suspend() multi.setKeys([trackers[0].publicKey, trackers[1].publicKey]) t.ok(multi.clients[1].suspended, 'new tracker inherits suspend') await new Promise((resolve) => setTimeout(resolve, 500)) t.is(seen.length, 0, 'no connect event while suspended') multi.resume() await waitFor(() => seen.length === 2) t.pass('trackers reconnect after resuming') }) test('resume without suspend keeps the existing connection', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const client = new HyperTrackerClient(server.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) let connects = 0 client.on('connect', () => connects++) client.connect() await waitFor(() => connects === 1) await waitFor(() => server.stats.streamsAdded === 1) const connection = client.connection client.resume() await new Promise((resolve) => setTimeout(resolve, 500)) t.ok(client.connection === connection, 'client kept its connection') t.is(connects, 1, 'no extra connect event') t.is(server.stats.streamsAdded, 1, 'tracker did not get a new stream') }) test('client should not lose announce on reconnect', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) announcer.connect() await new Promise((resolve) => announcer.once('connect', resolve)) const keyPair = crypto.keyPair() announcer.connection.destroy() announcer.announce(keyPair) const record = await waitForRecord(server, keyPair.publicKey) t.ok(record, 'announce reaches the tracker after the client reconnects') }) test('rejected channel does not pile up connections', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() let open = 0 let maxOpen = 0 const server = serverDht.createServer((connection) => { maxOpen = Math.max(maxOpen, ++open) connection.on('close', () => open--) Protomux.from(connection) }) t.teardown(() => server.close(), { order: 3000 }) const keyPair = crypto.keyPair() await server.listen(keyPair) const client = new HyperTrackerClient(keyPair.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) client.connect() await new Promise((resolve) => setTimeout(resolve, 7000)) t.is(maxOpen, 1, 'never more than one connection open') }) test('suspend and resume while connecting keeps a single connection', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const client = new HyperTrackerClient(server.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) let connects = 0 client.on('connect', () => connects++) client.connect() client.suspend() client.resume() await waitFor(() => connects > 0) const connection = client.connection await new Promise((resolve) => setTimeout(resolve, 6000)) t.ok(client.connection === connection, 'client kept its connection') t.is(connects, 1, 'no extra connect event') t.is(server.stats.streamsAdded, 1, 'tracker did not get a new stream') }) test('suspend and resume during retry backoff keeps a single connection', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const clientDht = new HyperDHT({ bootstrap }) t.teardown(() => clientDht.destroy(), { order: 4000 }) await clientDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const client = new HyperTrackerClient(server.publicKey, { dht: clientDht }) t.teardown(() => client.close(), { order: 2000 }) let connects = 0 client.on('connect', () => connects++) client.connect() const failed = client.connection failed.destroy() await new Promise((resolve) => failed.once('close', resolve)) client.suspend() client.resume() await waitFor(() => connects > 0) await new Promise((resolve) => setTimeout(resolve, 1000)) t.is(connects, 1, 'no extra connect event') t.is(server.stats.streamsAdded, 1, 'tracker got a single stream') }) test('announce bad data while reconnecting throws and leaves the client working', async (t) => { const testnet = await setupTestnet() const { bootstrap } = testnet t.teardown(() => testnet.destroy(), { order: 5000 }) const serverDht = new HyperDHT({ bootstrap }) t.teardown(() => serverDht.destroy(), { order: 4000 }) await serverDht.ready() const announcerDht = new HyperDHT({ bootstrap }) t.teardown(() => announcerDht.destroy(), { order: 4000 }) await announcerDht.ready() const storage = await t.tmp() const server = new HyperTracker(storage, { dht: serverDht }) t.teardown(() => server.close(), { order: 3000 }) await server.ready() const announcer = new HyperTrackerClient(server.publicKey, { dht: announcerDht }) t.teardown(() => announcer.close(), { order: 2000 }) const unhandled = [] const onUnhandled = (err) => unhandled.push(err) process.on('unhandledRejection', onUnhandled) t.teardown(() => process.off('unhandledRejection', onUnhandled)) let connects = 0 announcer.on('connect', () => connects++) announcer.connect() await waitFor(() => connects === 1) const keyPair = crypto.keyPair() const badData = { publicKey: b4a.alloc(64), secretKey: keyPair.secretKey } announcer.connection.destroy() t.exception(() => announcer.announce(badData), 'announce bad data throws to the caller') await t.execution( waitFor(() => connects === 2), 'client reconnects' ) announcer.announce(keyPair) const record = await waitForRecord(server, keyPair.publicKey) t.ok(record, 'announce after the reconnect reaches the tracker') t.is(unhandled.length, 0, 'no unhandled rejections') }) async function createTrackers(t, bootstrap, n) { const trackers = [] for (let i = 0; i < n; i++) { const dht = new HyperDHT({ bootstrap }) t.teardown(() => dht.destroy(), { order: 4000 }) await dht.ready() const tracker = new HyperTracker(await t.tmp(), { dht }) t.teardown(() => tracker.close(), { order: 3000 }) await tracker.ready() trackers.push(tracker) } return trackers } async function waitForRecord(server, publicKey, timeout = 5000) { const start = Date.now() while (Date.now() - start < timeout) { const record = await server.lookup(publicKey) if (record) return record await new Promise((resolve) => setTimeout(resolve, 100)) } return null } async function waitFor(condition, timeout = 5000) { const start = Date.now() while (Date.now() - start < timeout) { if (condition()) return await new Promise((resolve) => setTimeout(resolve, 100)) } throw new Error('timed out waiting for condition') }