// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. import type { ITerminal } from '@rushstack/terminal'; import type { IDaemonEventEnvelope, IDaemonPhasedOperationSelection, IDaemonPhasedRequest, IDaemonPhasedRequestResult } from '@rushstack/rush-daemon-protocol'; import { RUSHD_OPERATION_HEADER, RUSHD_OPERATION_STREAM_CLOSED } from '@rushstack/rush-daemon-protocol'; import { OperationStatus } from '@microsoft/rush-lib'; import type { IInputsSnapshot, IOperationRunnerContext, IPhasedCommandEngineRequestSettings, IRushConfigurationProjectForSnapshot } from '@microsoft/rush-lib'; import { PhasedRequestRouter } from '../PhasedRequestRouter'; import { TEST_ENGINE_SHAPE, TestOperationRunner, TestPhasedRequestClient, createRoutingFixture } from './PhasedRequestRouterTestUtilities'; import type { ITestClientWrite, ITestRoutingFixture } from './PhasedRequestRouterTestUtilities'; import { TEST_REPO_ROOT } from './TestWorkspaceSession'; const OPERATION_A: string = 'project-a (_phase:test)'; const OPERATION_B: string = 'project-b (_phase:test)'; const OPERATION_C: string = 'project-c (_phase:test)'; const SESSION_VARIABLE: string = 'COPILOT_AGENT_SESSION_ID'; type TestOperationAction = (terminal: ITerminal, context: IOperationRunnerContext) => Promise; interface IDeferred { readonly promise: Promise; readonly resolve: () => void; } function createDeferred(): IDeferred { let resolvePromise: (() => void) | undefined; const promise: Promise = new Promise((resolve) => { resolvePromise = resolve; }); return { promise, resolve: () => resolvePromise?.() }; } function select(operationId: string): IDaemonPhasedOperationSelection { return { enabledState: true, operationId }; } function createRequest( requestId: string, ...selectedOperationIds: ReadonlyArray ): IDaemonPhasedRequest { return { commandName: 'build', commandOrigin: 'built-in', engineShape: TEST_ENGINE_SHAPE, environment: {}, operationSelection: selectedOperationIds.map(select), requestId }; } function createFixture(options?: { readonly actionAAsync?: TestOperationAction; readonly actionBAsync?: TestOperationAction; readonly actionCAsync?: TestOperationAction; readonly statusA?: OperationStatus; }): ITestRoutingFixture { return createRoutingFixture( new Map([ [ OPERATION_A, new TestOperationRunner( OPERATION_A, options?.statusA ?? OperationStatus.Success, options?.actionAAsync ) ], [OPERATION_B, new TestOperationRunner(OPERATION_B, OperationStatus.Success, options?.actionBAsync)], [OPERATION_C, new TestOperationRunner(OPERATION_C, OperationStatus.Success, options?.actionCAsync)] ]), [[OPERATION_B, OPERATION_A]] ); } function getResultOperationIds(result: IDaemonPhasedRequestResult): ReadonlyArray { return result.operationResults.map(({ operationId }) => operationId); } /** One input file of project-c: the version on disk, and the version that the latest reconcile read. */ class TestWorkspaceInput { #diskVersion: number = 0; #snapshotVersion: number | undefined; public edit(): void { this.#diskVersion++; } public read(): void { this.#snapshotVersion = this.#diskVersion; } public describe(): string { return `snapshot=${this.#snapshotVersion} disk=${this.#diskVersion}`; } } async function settleAsync(): Promise { for (let turn: number = 0; turn < 20; turn++) { await new Promise((resolve) => setImmediate(resolve)); } } interface IExecutionLeaseTracker { readonly events: string[]; } function trackExecutionLease(fixture: ITestRoutingFixture): IExecutionLeaseTracker { const events: string[] = []; fixture.session.acquireExecutionLeaseAsync = async (): Promise => { events.push('acquired'); return { [Symbol.asyncDispose]: async (): Promise => { events.push('released'); } }; }; return { events }; } function trackResult( resultPromise: Promise, label: string, events: string[] ): Promise { return resultPromise.then((result: IDaemonPhasedRequestResult) => { events.push(`result:${label}`); return result; }); } function isStreamClosedEvent(write: ITestClientWrite, operationId: string): boolean { const payload: unknown = write.event?.payload; return ( typeof payload === 'object' && payload !== null && (payload as { name?: unknown }).name === RUSHD_OPERATION_STREAM_CLOSED && write.event !== undefined && eventOperationId(write.event) === operationId ); } function eventOperationId(event: IDaemonEventEnvelope): string | undefined { if (event.scope?.operationId) { return event.scope.operationId; } const payload: unknown = event.payload; if (typeof payload !== 'object' || payload === null) { return undefined; } const operationId: unknown = (payload as { operationId?: unknown }).operationId; if (typeof operationId === 'string') { return operationId; } const data: unknown = (payload as { data?: unknown }).data; return typeof data === 'object' && data !== null ? ((data as { operationId?: string }).operationId ?? undefined) : undefined; } describe('shared phased request batching', () => { it('schedules separate iterations for overlapping requests with different request settings', async () => { const fixture: ITestRoutingFixture = createFixture(); const graph: ITestRoutingFixture['graph'] = fixture.graph; const scheduledSettings: IPhasedCommandEngineRequestSettings[] = []; const originalScheduleAsync: typeof graph.scheduleIterationAsync = graph.scheduleIterationAsync.bind(graph); const scheduleSpy: jest.SpyInstance = jest .spyOn(graph, 'scheduleIterationAsync') .mockImplementation((...args: Parameters) => { scheduledSettings.push({ parallelism: graph.parallelism, quietMode: graph.quietMode, isIncrementalBuildAllowed: args[0].isIncrementalBuildAllowed as boolean }); return originalScheduleAsync(...args); }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const defaultSettings: IPhasedCommandEngineRequestSettings = { parallelism: 4, quietMode: true, isIncrementalBuildAllowed: true }; const verboseSerialSettings: IPhasedCommandEngineRequestSettings = { parallelism: 1, quietMode: false, isIncrementalBuildAllowed: false }; const [first, second] = await Promise.all([ router.executeAsync( createRequest('default', OPERATION_A), new TestPhasedRequestClient('one'), false, undefined, defaultSettings ), router.executeAsync( createRequest('verbose-serial', OPERATION_B), new TestPhasedRequestClient('two'), false, undefined, verboseSerialSettings ) ]); expect(scheduleSpy).toHaveBeenCalledTimes(2); expect(scheduledSettings).toEqual([defaultSettings, verboseSerialSettings]); expect(first).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(second).toMatchObject({ exitCode: 0, outcome: 'success' }); }); it('merges overlapping selections into one real graph iteration and executes shared operations once', async () => { const fixture: ITestRoutingFixture = createFixture(); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const [dependency, consumer] = await Promise.all([ router.executeAsync(createRequest('dependency', OPERATION_A), new TestPhasedRequestClient('one')), router.executeAsync(createRequest('consumer', OPERATION_B), new TestPhasedRequestClient('two')) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(fixture.runners.get(OPERATION_B)?.runCount).toBe(1); expect(getResultOperationIds(dependency)).toEqual([OPERATION_A]); expect(getResultOperationIds(consumer)).toEqual([OPERATION_A, OPERATION_B]); }); it('gives each operation of a shared iteration the environment of the first request that selected it', async () => { const sessions: Map = new Map(); const record = (operationId: string): TestOperationAction => async (terminal: ITerminal, context: IOperationRunnerContext): Promise => { sessions.set(operationId, context.environment?.[SESSION_VARIABLE]); }; const fixture: ITestRoutingFixture = createFixture({ actionAAsync: record(OPERATION_A), actionBAsync: record(OPERATION_B), actionCAsync: record(OPERATION_C) }); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const withSession = (request: IDaemonPhasedRequest, session?: string): IDaemonPhasedRequest => ({ ...request, environment: session === undefined ? {} : { [SESSION_VARIABLE]: session } }); const daemonSession: string | undefined = process.env[SESSION_VARIABLE]; process.env[SESSION_VARIABLE] = 'daemon'; try { await Promise.all([ router.executeAsync( withSession(createRequest('a', OPERATION_A), 'session-A'), new TestPhasedRequestClient('one') ), // Project B depends on project A, which the first request already selected. router.executeAsync( withSession(createRequest('b', OPERATION_B), 'session-B'), new TestPhasedRequestClient('two') ), router.executeAsync( withSession(createRequest('c', OPERATION_C)), new TestPhasedRequestClient('three') ) ]); } finally { if (daemonSession === undefined) delete process.env[SESSION_VARIABLE]; else process.env[SESSION_VARIABLE] = daemonSession; } expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(sessions).toEqual( new Map([ [OPERATION_A, 'session-A'], [OPERATION_B, 'session-B'], [OPERATION_C, undefined] ]) ); }); it('gives each operation the request id of the request whose environment it gets', async () => { const fixture: ITestRoutingFixture = createFixture(); const iterations: string[][] = []; fixture.graph.hooks.beforeExecuteIterationAsync.tap('test plugin', (records, options) => { const lines: string[] = []; for (const operation of records.keys()) { const session: string | undefined = options.getOperationEnvironment?.(operation)[SESSION_VARIABLE]; lines.push(`${operation.name}: ${options.getOperationRequestId?.(operation)} ${session}`); } iterations.push(lines.sort()); }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const withSession = (request: IDaemonPhasedRequest, session: string): IDaemonPhasedRequest => ({ ...request, environment: { [SESSION_VARIABLE]: session } }); await Promise.all([ router.executeAsync( withSession(createRequest('x', OPERATION_A, OPERATION_C), 'session-X'), new TestPhasedRequestClient('x') ), router.executeAsync( withSession(createRequest('y', OPERATION_B), 'session-Y'), new TestPhasedRequestClient('y') ) ]); await router.executeAsync( withSession(createRequest('z', OPERATION_A), 'session-Z'), new TestPhasedRequestClient('z') ); // An operation that no participant selected gets the first participant's environment and request id. expect(iterations).toEqual([ [`${OPERATION_A}: x session-X`, `${OPERATION_B}: y session-Y`, `${OPERATION_C}: x session-X`], [`${OPERATION_A}: z session-Z`, `${OPERATION_B}: z session-Z`, `${OPERATION_C}: z session-Z`] ]); }); describe('with a plugin that changes process.env in beforeExecuteIterationAsync', () => { const ADDED_VARIABLE: string = 'RUSHD_TEST_ADDED'; const CHANGED_VARIABLE: string = 'RUSHD_TEST_CHANGED'; const REMOVED_VARIABLE: string = 'RUSHD_TEST_REMOVED'; const NAMES: ReadonlyArray = [ SESSION_VARIABLE, ADDED_VARIABLE, CHANGED_VARIABLE, REMOVED_VARIABLE ]; let savedValues: ReadonlyArray = []; beforeEach(() => { savedValues = NAMES.map((name: string) => process.env[name]); process.env[SESSION_VARIABLE] = 'daemon'; delete process.env[ADDED_VARIABLE]; process.env[CHANGED_VARIABLE] = 'original'; process.env[REMOVED_VARIABLE] = 'original'; }); afterEach(() => { NAMES.forEach((name: string, index: number) => { const value: string | undefined = savedValues[index]; if (value === undefined) delete process.env[name]; else process.env[name] = value; }); }); function createFixtureWithPlugin(options?: Parameters[0]): ITestRoutingFixture { const fixture: ITestRoutingFixture = createFixture(options); let iteration: number = 0; fixture.graph.hooks.beforeExecuteIterationAsync.tap('test plugin', () => { iteration++; process.env[ADDED_VARIABLE] = `added-${iteration}`; process.env[CHANGED_VARIABLE] = `changed-${iteration}`; delete process.env[REMOVED_VARIABLE]; }); return fixture; } function withSession(request: IDaemonPhasedRequest, session: string): IDaemonPhasedRequest { return { ...request, environment: { [SESSION_VARIABLE]: session } }; } function recordEnvironment(seen: string[], operationId: string): TestOperationAction { return async (terminal: ITerminal, context: IOperationRunnerContext): Promise => { const values: string[] = NAMES.map((name: string) => context.environment?.[name] ?? ''); seen.push(`${operationId}: ${values.join(' ')}`); }; } it("starts each operation from what the same iteration's hook set, changed and deleted, with its requester's session", async () => { const seen: string[] = []; const fixture: ITestRoutingFixture = createFixtureWithPlugin({ actionAAsync: recordEnvironment(seen, OPERATION_A), actionCAsync: recordEnvironment(seen, OPERATION_C) }); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync( withSession(createRequest('a', OPERATION_A), 'session-A'), new TestPhasedRequestClient('a') ), router.executeAsync( withSession(createRequest('c', OPERATION_C), 'session-C'), new TestPhasedRequestClient('c') ) ]); await router.executeAsync( withSession(createRequest('a-again', OPERATION_A), 'session-A-again'), new TestPhasedRequestClient('a-again') ); expect(scheduleSpy).toHaveBeenCalledTimes(2); expect([...seen].sort()).toEqual([ `${OPERATION_A}: session-A added-1 changed-1 `, `${OPERATION_A}: session-A-again added-2 changed-2 `, `${OPERATION_C}: session-C added-1 changed-1 ` ]); }); it('hashes from one copy of each requester environment while scheduling and starts from a copy taken after the hook', async () => { const hashedEnvironments: Map< IRushConfigurationProjectForSnapshot, Readonly> | undefined > = new Map(); const seen: string[] = []; const fixture: ITestRoutingFixture = createFixtureWithPlugin({ actionAAsync: recordEnvironment(seen, OPERATION_A), actionBAsync: recordEnvironment(seen, OPERATION_B), actionCAsync: recordEnvironment(seen, OPERATION_C) }); const inputsSnapshot: IInputsSnapshot = { getOperationOwnStateHash: ( project: IRushConfigurationProjectForSnapshot, operationName?: string, environment?: Readonly> ): string => { hashedEnvironments.set(project, environment); return 'hash'; }, getTrackedFileHashesForOperation: () => new Map(), hasUncommittedChanges: false, hashes: new Map(), rootDirectory: TEST_REPO_ROOT }; Object.assign(fixture.session, { inputsSnapshot }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync( withSession(createRequest('x', OPERATION_A, OPERATION_C), 'session-X'), new TestPhasedRequestClient('x') ), router.executeAsync( withSession(createRequest('y', OPERATION_B), 'session-Y'), new TestPhasedRequestClient('y') ) ]); const [hashedA, hashedB, hashedC] = [OPERATION_A, OPERATION_B, OPERATION_C].map((operationId: string) => hashedEnvironments.get(fixture.operations.get(operationId)!.associatedProject) ); expect(hashedA?.[SESSION_VARIABLE]).toBe('session-X'); expect(hashedB?.[SESSION_VARIABLE]).toBe('session-Y'); // The same object. Each is a whole environment, so a failure prints only a boolean. expect(hashedC === hashedA).toBe(true); expect(hashedA?.[CHANGED_VARIABLE]).toBe('original'); expect([...seen].sort()).toEqual([ `${OPERATION_A}: session-X added-1 changed-1 `, `${OPERATION_B}: session-Y added-1 changed-1 `, `${OPERATION_C}: session-X added-1 changed-1 ` ]); }); }); describe('with an operation that hashes a variable that does not select a daemon', () => { const TERMINAL_VARIABLE: string = 'WT_SESSION'; const HOST_VARIABLE: string = 'RUSHD_TEST_HOST_VARIABLE'; interface IHashedVariableFixture { readonly fixture: ITestRoutingFixture; readonly router: PhasedRequestRouter; readonly runs: ReadonlyArray; readonly scheduleSpy: jest.SpyInstance; } function createHashedVariableFixture( dependsOnEnvVars: string[], actionCAsync?: TestOperationAction ): IHashedVariableFixture { const runs: (string | undefined)[] = []; const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (terminal: ITerminal, context: IOperationRunnerContext): Promise => { runs.push(context.environment?.[TERMINAL_VARIABLE]); }, actionCAsync }); fixture.operations.get(OPERATION_A)!.settings = { operationName: '_phase:test', dependsOnEnvVars }; return { fixture, router: new PhasedRequestRouter(fixture.session), runs, scheduleSpy: jest.spyOn(fixture.graph, 'scheduleIterationAsync') }; } function createRequestWithEnvironment( requestId: string, environment: Record, operationId: string ): IDaemonPhasedRequest { return { ...createRequest(requestId, operationId), environment }; } it('schedules separate iterations for queued requests that disagree on its value', async () => { const occupierStarted: IDeferred = createDeferred(); const releaseOccupier: IDeferred = createDeferred(); const { fixture, router, runs, scheduleSpy } = createHashedVariableFixture( [TERMINAL_VARIABLE], async (): Promise => { occupierStarted.resolve(); await releaseOccupier.promise; } ); const occupier: Promise = router.executeAsync( createRequest('occupier', OPERATION_C), new TestPhasedRequestClient('occupier') ); await occupierStarted.promise; const results: Promise = Promise.all([ router.executeAsync( createRequestWithEnvironment('x', { [TERMINAL_VARIABLE]: 'wt-X' }, OPERATION_A), new TestPhasedRequestClient('x') ), router.executeAsync( createRequestWithEnvironment('y', { [TERMINAL_VARIABLE]: 'wt-Y' }, OPERATION_A), new TestPhasedRequestClient('y') ) ]); releaseOccupier.resolve(); await occupier; const [x, y] = await results; expect(scheduleSpy).toHaveBeenCalledTimes(3); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(2); expect(runs).toEqual(['wt-X', 'wt-Y']); expect(x).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(y).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(getResultOperationIds(x)).toEqual([OPERATION_A]); expect(getResultOperationIds(y)).toEqual([OPERATION_A]); }); it('shares one iteration for requests that agree on its value', async () => { const { fixture, router, runs, scheduleSpy } = createHashedVariableFixture([TERMINAL_VARIABLE]); await Promise.all([ router.executeAsync( createRequestWithEnvironment( 'x', { [TERMINAL_VARIABLE]: 'wt-X', [SESSION_VARIABLE]: 'session-X' }, OPERATION_A ), new TestPhasedRequestClient('x') ), router.executeAsync( createRequestWithEnvironment( 'y', { [TERMINAL_VARIABLE]: 'wt-X', [SESSION_VARIABLE]: 'session-Y' }, OPERATION_A ), new TestPhasedRequestClient('y') ) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(runs).toEqual(['wt-X']); }); it('shares one iteration for an unset value and an empty one, which operations hash alike', async () => { const { fixture, router, runs, scheduleSpy } = createHashedVariableFixture([TERMINAL_VARIABLE]); await Promise.all([ router.executeAsync(createRequest('unset', OPERATION_A), new TestPhasedRequestClient('unset')), router.executeAsync( createRequestWithEnvironment('empty', { [TERMINAL_VARIABLE]: '' }, OPERATION_A), new TestPhasedRequestClient('empty') ) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(runs).toEqual([undefined]); }); it('keeps sharing one iteration when requests differ only in a variable that operations take from the daemon', async () => { const { fixture, router, scheduleSpy } = createHashedVariableFixture([HOST_VARIABLE]); await Promise.all([ router.executeAsync( createRequestWithEnvironment('x', { [HOST_VARIABLE]: 'x' }, OPERATION_A), new TestPhasedRequestClient('x') ), router.executeAsync( createRequestWithEnvironment('y', { [HOST_VARIABLE]: 'y' }, OPERATION_A), new TestPhasedRequestClient('y') ) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); }); }); it('shares one iteration for disjoint selections while isolating streams, events, and results', async () => { const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (terminal: ITerminal): Promise => terminal.writeLine('only-a'), actionCAsync: async (terminal: ITerminal): Promise => terminal.writeLine('only-c') }); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const clientA: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const clientC: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const [resultA, resultC] = await Promise.all([ router.executeAsync(createRequest('a', OPERATION_A), clientA), router.executeAsync(createRequest('c', OPERATION_C), clientC) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(getResultOperationIds(resultA)).toEqual([OPERATION_A]); expect(getResultOperationIds(resultC)).toEqual([OPERATION_C]); expect(getWrittenOperationIds(clientA)).toEqual(new Set([OPERATION_A])); expect(getWrittenOperationIds(clientC)).toEqual(new Set([OPERATION_C])); expect(getHeaderData(clientA)).toEqual([ { completedOperations: 1, operationId: OPERATION_A, totalOperations: 1 } ]); expect(getHeaderData(clientC)).toEqual([ { completedOperations: 1, operationId: OPERATION_C, totalOperations: 1 } ]); }); it.each([false, true])( 'announces a shared iteration to each client with only its own operations (quiet mode: %s)', async (quietMode: boolean) => { const fixture: ITestRoutingFixture = createFixture(); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const settings: IPhasedCommandEngineRequestSettings = { parallelism: 4, quietMode, isIncrementalBuildAllowed: true }; const clientB: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const clientC: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync(createRequest('b', OPERATION_B), clientB, false, undefined, settings), router.executeAsync(createRequest('c', OPERATION_C), clientC, false, undefined, settings) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(getIterationAnnouncement(clientB)).toEqual([ ...(quietMode ? [] : ['Selected 2 operations:', ` ${OPERATION_A}`, ` ${OPERATION_B}`, '']), 'Executing a maximum of 2 simultaneous processes...' ]); expect(getIterationAnnouncement(clientC)).toEqual([ ...(quietMode ? [] : ['Selected 1 operation:', ` ${OPERATION_C}`, '']), 'Executing a maximum of 1 simultaneous processes...' ]); } ); it.each([false, true])( 'announces an operation that two clients of a shared iteration both select to each of them (quiet mode: %s)', async (quietMode: boolean) => { const fixture: ITestRoutingFixture = createFixture(); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const settings: IPhasedCommandEngineRequestSettings = { parallelism: 4, quietMode, isIncrementalBuildAllowed: true }; const clientA: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const clientB: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync(createRequest('a', OPERATION_A), clientA, false, undefined, settings), router.executeAsync(createRequest('b', OPERATION_B), clientB, false, undefined, settings) ]); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(getIterationAnnouncement(clientA)).toEqual([ ...(quietMode ? [] : ['Selected 1 operation:', ` ${OPERATION_A}`, '']), 'Executing a maximum of 1 simultaneous processes...' ]); expect(getIterationAnnouncement(clientB)).toEqual([ ...(quietMode ? [] : ['Selected 2 operations:', ` ${OPERATION_A}`, ` ${OPERATION_B}`, '']), 'Executing a maximum of 2 simultaneous processes...' ]); expect(getHeaderData(clientA)).toEqual([ { completedOperations: 1, operationId: OPERATION_A, totalOperations: 1 } ]); expect(getHeaderData(clientB)).toEqual([ { completedOperations: 1, operationId: OPERATION_A, totalOperations: 2 }, { completedOperations: 2, operationId: OPERATION_B, totalOperations: 2 } ]); } ); it.each([false, true])( 'does not announce a shared iteration to a client that has nothing to run in it, as when it is alone (quiet mode: %s)', async (quietMode: boolean) => { const fixture: ITestRoutingFixture = createFixture(); let upToDate: boolean = false; fixture.graph.hooks.configureIteration.tap('project-a is up to date', (records) => { for (const record of records.values()) { if (upToDate && record.operation.name === OPERATION_A) { record.enabled = false; } } }); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const settings: IPhasedCommandEngineRequestSettings = { parallelism: 4, quietMode, isIncrementalBuildAllowed: true }; const clientA: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const clientC: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('zero'), false, undefined, settings ); upToDate = true; await Promise.all([ router.executeAsync(createRequest('a', OPERATION_A), clientA, false, undefined, settings), router.executeAsync(createRequest('c', OPERATION_C), clientC, false, undefined, settings) ]); expect(scheduleSpy).toHaveBeenCalledTimes(2); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(getIterationAnnouncement(clientA)).toEqual([]); expect(getIterationAnnouncement(clientC)).toEqual([ ...(quietMode ? [] : ['Selected 1 operation:', ` ${OPERATION_C}`, '']), 'Executing a maximum of 1 simultaneous processes...' ]); } ); it('publishes a coalesced client result as soon as its own closure settles', async () => { const releaseB: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (terminal: ITerminal): Promise => terminal.writeLine('from-a'), actionBAsync: async (): Promise => releaseB.promise }); const lease: IExecutionLeaseTracker = trackExecutionLease(fixture); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const clientA: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const clientB: TestPhasedRequestClient = new TestPhasedRequestClient('two'); // A slow reader must still receive all of its operation output before its early result. clientA.onWriteAsync = async (): Promise => { await new Promise((resolve) => setImmediate(resolve)); }; const resultAPromise: Promise = trackResult( router.executeAsync(createRequest('a', OPERATION_A), clientA), 'a', lease.events ); const resultBPromise: Promise = trackResult( router.executeAsync(createRequest('b', OPERATION_B), clientB), 'b', lease.events ); const resultA: IDaemonPhasedRequestResult = await resultAPromise; expect(lease.events).toEqual(['acquired', 'result:a']); expect(fixture.graph.status).toBe(OperationStatus.Executing); expect(resultA).toMatchObject({ exitCode: 0, outcome: 'success', scheduled: true }); expect(resultA.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Success }) ]); expect(getWrittenOperationIds(clientA)).toEqual(new Set([OPERATION_A])); expect(clientA.writes.some((write: ITestClientWrite) => write.text?.includes('from-a'))).toBe(true); expect(clientA.writes.findIndex((write) => isStreamClosedEvent(write, OPERATION_A))).toBeGreaterThan(-1); expect(clientA.writes[clientA.writes.length - 1]?.result).toBe(resultA); expect(getHeaderData(clientA)).toEqual([ { completedOperations: 1, operationId: OPERATION_A, totalOperations: 1 } ]); const clientAWriteCount: number = clientA.writes.length; releaseB.resolve(); const resultB: IDaemonPhasedRequestResult = await resultBPromise; // The last participant keeps the ordinary contract: its result follows execution lease release. expect(lease.events).toEqual(['acquired', 'result:a', 'released', 'result:b']); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); expect(fixture.runners.get(OPERATION_B)?.runCount).toBe(1); expect(resultB).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(getResultOperationIds(resultB)).toEqual([OPERATION_A, OPERATION_B]); expect(clientA.writes).toHaveLength(clientAWriteCount); }); it('publishes an early failure result while the batch continues for other clients', async () => { const releaseC: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => releaseC.promise, statusA: OperationStatus.Failure }); fixture.graph.parallelism = 2; const lease: IExecutionLeaseTracker = trackExecutionLease(fixture); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const failedPromise: Promise = trackResult( router.executeAsync(createRequest('failed', OPERATION_A), new TestPhasedRequestClient('one')), 'failed', lease.events ); const continuingPromise: Promise = trackResult( router.executeAsync(createRequest('continuing', OPERATION_C), new TestPhasedRequestClient('two')), 'continuing', lease.events ); const failed: IDaemonPhasedRequestResult = await failedPromise; expect(lease.events).toEqual(['acquired', 'result:failed']); expect(failed).toMatchObject({ aborted: false, exitCode: 1, outcome: 'failure' }); expect(failed.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Failure }) ]); releaseC.resolve(); const continuing: IDaemonPhasedRequestResult = await continuingPromise; expect(lease.events).toEqual(['acquired', 'result:failed', 'released', 'result:continuing']); expect(continuing).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(getResultOperationIds(continuing)).toEqual([OPERATION_C]); }); it('aborts the iteration when the only client still needing it cancels after an early result', async () => { const operationCStarted: IDeferred = createDeferred(); const releaseC: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => { operationCStarted.resolve(); await releaseC.promise; } }); fixture.graph.parallelism = 2; const abortSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'abortCurrentIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const cancelledClient: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const finishedPromise: Promise = router.executeAsync( createRequest('finished', OPERATION_A), new TestPhasedRequestClient('one') ); const cancelledPromise: Promise = router.executeAsync( createRequest('cancelled', OPERATION_C), cancelledClient ); const finished: IDaemonPhasedRequestResult = await finishedPromise; await operationCStarted.promise; expect(finished).toMatchObject({ exitCode: 0, outcome: 'success' }); const abortCallCountBeforeCancellation: number = abortSpy.mock.calls.length; cancelledClient.abortController.abort(); expect(abortSpy.mock.calls.length).toBeGreaterThan(abortCallCountBeforeCancellation); releaseC.resolve(); const cancelled: IDaemonPhasedRequestResult = await cancelledPromise; expect(cancelled).toMatchObject({ aborted: true, outcome: 'aborted' }); }); it('derives shared and disjoint failure results from each client subset', async () => { const fixture: ITestRoutingFixture = createFixture({ statusA: OperationStatus.Failure }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const [failed, blocked, disjoint] = await Promise.all([ router.executeAsync(createRequest('failed', OPERATION_A), new TestPhasedRequestClient('one')), router.executeAsync(createRequest('blocked', OPERATION_B), new TestPhasedRequestClient('two')), router.executeAsync(createRequest('disjoint', OPERATION_C), new TestPhasedRequestClient('three')) ]); expect(failed).toMatchObject({ exitCode: 1, outcome: 'failure' }); expect(blocked).toMatchObject({ exitCode: 1, outcome: 'failure' }); expect(blocked.operationResults).toEqual( expect.arrayContaining([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Failure }), expect.objectContaining({ operationId: OPERATION_B, status: OperationStatus.Blocked }) ]) ); expect(disjoint).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(getResultOperationIds(disjoint)).toEqual([OPERATION_C]); }); it('removes a client cancelled before scheduling without running its selection', async () => { const fixture: ITestRoutingFixture = createFixture(); const reconcileStarted: IDeferred = createDeferred(); const releaseReconcile: IDeferred = createDeferred(); fixture.session.onReconcileAsync = async (): Promise => { reconcileStarted.resolve(); await releaseReconcile.promise; }; const client: TestPhasedRequestClient = new TestPhasedRequestClient(); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const resultPromise: Promise = new PhasedRequestRouter( fixture.session ).executeAsync(createRequest('cancelled', OPERATION_A), client); await reconcileStarted.promise; client.abortController.abort(); releaseReconcile.resolve(); const result: IDaemonPhasedRequestResult = await resultPromise; expect(result).toMatchObject({ aborted: true, outcome: 'aborted', scheduled: false }); expect(scheduleSpy).not.toHaveBeenCalled(); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(0); }); it('unsubscribes one mid-run cancellation without aborting work required by another client', async () => { const operationStarted: IDeferred = createDeferred(); const operationCStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; }, // Keep the continuing client's work outstanding until after the cancellation. actionCAsync: async (): Promise => { operationCStarted.resolve(); await releaseOperation.promise; } }); fixture.graph.parallelism = 2; const cancelledClient: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const continuingClient: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const abortSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'abortCurrentIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const cancelled = router.executeAsync(createRequest('cancelled', OPERATION_A), cancelledClient); const continuing = router.executeAsync(createRequest('continuing', OPERATION_C), continuingClient); await Promise.all([operationStarted.promise, operationCStarted.promise]); const abortCallCountBeforeCancellation: number = abortSpy.mock.calls.length; cancelledClient.abortController.abort(); // The continuing client still needs its running operation, so the cancellation itself aborts nothing. Once that // operation finishes, work that only the cancelled client needed may be aborted (PhasedRequestCancellation.test). expect(abortSpy).toHaveBeenCalledTimes(abortCallCountBeforeCancellation); releaseOperation.resolve(); const [cancelledResult, continuingResult] = await Promise.all([cancelled, continuing]); expect(cancelledResult).toMatchObject({ aborted: true, outcome: 'aborted' }); expect(continuingResult).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(1); }); it('answers a client that cancels during a shared operation immediately, without waiting for the batch', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const cancelledClient: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const continuingClient: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const cancelled = router.executeAsync(createRequest('cancelled', OPERATION_A), cancelledClient); const continuing = router.executeAsync(createRequest('continuing', OPERATION_A), continuingClient); await operationStarted.promise; cancelledClient.abortController.abort(); // The shared operation is still running for the other client. const cancelledResult: IDaemonPhasedRequestResult = await cancelled; releaseOperation.resolve(); const continuingResult: IDaemonPhasedRequestResult = await continuing; expect(cancelledResult).toMatchObject({ aborted: true, outcome: 'aborted' }); expect(cancelledResult.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Aborted }) ]); expect(continuingResult).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); }); it('prefers a current abort after invalidating a retained warm success', async () => { const fixture: ITestRoutingFixture = createFixture(); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = await router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); expect(first.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Success }) ]); fixture.graph.invalidateOperations(undefined, 'rerun'); fixture.graph.hooks.beforeExecuteIterationAsync.tapPromise( 'abort current iteration', async (): Promise => OperationStatus.Aborted ); const second = await router.executeAsync( createRequest('second', OPERATION_A), new TestPhasedRequestClient('two') ); expect(second).toMatchObject({ exitCode: 1, outcome: 'aborted', scheduled: true }); expect(second.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Aborted }) ]); }); it('keeps failure for the continuing client when another client cancels during a failing shared operation', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; }, statusA: OperationStatus.Failure }); const cancelledClient: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const cancelled = router.executeAsync(createRequest('cancelled', OPERATION_A), cancelledClient); const continuing = router.executeAsync( createRequest('continuing', OPERATION_A), new TestPhasedRequestClient('two') ); await operationStarted.promise; cancelledClient.abortController.abort(); const cancelledResult: IDaemonPhasedRequestResult = await cancelled; releaseOperation.resolve(); const continuingResult: IDaemonPhasedRequestResult = await continuing; // The cancelled client detached before the shared operation failed. expect(cancelledResult).toMatchObject({ aborted: true, outcome: 'aborted' }); expect(continuingResult).toMatchObject({ aborted: false, exitCode: 1, outcome: 'failure' }); expect(continuingResult.operationResults).toEqual([ expect.objectContaining({ operationId: OPERATION_A, status: OperationStatus.Failure }) ]); expect(fixture.runners.get(OPERATION_A)?.runCount).toBe(1); }); it('aborts the shared iteration when every client cancels', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const firstClient: TestPhasedRequestClient = new TestPhasedRequestClient('one'); const secondClient: TestPhasedRequestClient = new TestPhasedRequestClient('two'); const abortSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'abortCurrentIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync(createRequest('first', OPERATION_A), firstClient); const second = router.executeAsync(createRequest('second', OPERATION_B), secondClient); await operationStarted.promise; const abortCallCountBeforeCancellation: number = abortSpy.mock.calls.length; firstClient.abortController.abort(); secondClient.abortController.abort(); releaseOperation.resolve(); const results: ReadonlyArray = await Promise.all([first, second]); expect(results).toEqual([ expect.objectContaining({ aborted: true, outcome: 'aborted' }), expect.objectContaining({ aborted: true, outcome: 'aborted' }) ]); expect(abortSpy.mock.calls.length).toBeGreaterThan(abortCallCountBeforeCancellation); expect(fixture.runners.get(OPERATION_B)?.runCount).toBe(0); }); it('puts arrivals after execution begins into a later batch and reconciles once per batch', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); fixture.session.onReconcileAsync = jest.fn(async (): Promise => undefined); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await operationStarted.promise; const late = router.executeAsync( createRequest('late', OPERATION_C), new TestPhasedRequestClient('two') ); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(0); releaseOperation.resolve(); await Promise.all([first, late]); expect(scheduleSpy).toHaveBeenCalledTimes(2); expect(fixture.session.onReconcileAsync).toHaveBeenCalledTimes(2); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(1); }); it('puts a compatible request received during the reconcile into a later batch that reconciles again', async () => { const input: TestWorkspaceInput = new TestWorkspaceInput(); const events: string[] = []; const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => { events.push(`run:C ${input.describe()}`); } }); const inputRead: IDeferred = createDeferred(); const finishFirstReconcile: IDeferred = createDeferred(); let reconcileCount: number = 0; fixture.session.onReconcileAsync = async (): Promise => { reconcileCount++; events.push('reconcile:start'); input.read(); if (reconcileCount === 1) { inputRead.resolve(); // The reconcile has read project-c's input and is still reading the rest of the workspace. await finishFirstReconcile.promise; } events.push('reconcile:end'); }; const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await inputRead.promise; // Another client changes project-c's input, then submits a compatible build that needs it. input.edit(); events.push('edit'); const second = router.executeAsync( createRequest('second', OPERATION_C), new TestPhasedRequestClient('two') ); await settleAsync(); finishFirstReconcile.resolve(); expect(await first).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(await second).toMatchObject({ exitCode: 0, outcome: 'success' }); // Joining the first batch would run project-c on the inputs read before its change ("snapshot=0 disk=1"). expect(events).toEqual([ 'reconcile:start', 'edit', 'reconcile:end', 'reconcile:start', 'reconcile:end', 'run:C snapshot=1 disk=1' ]); expect(scheduleSpy).toHaveBeenCalledTimes(2); }); it('lets a compatible request that is pending before the reconcile join the batch and see its change', async () => { const input: TestWorkspaceInput = new TestWorkspaceInput(); const events: string[] = []; const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => { events.push(`run:C ${input.describe()}`); } }); const leaseRequested: IDeferred = createDeferred(); const grantLease: IDeferred = createDeferred(); fixture.session.acquireExecutionLeaseAsync = async (): Promise => { leaseRequested.resolve(); await grantLease.promise; return { [Symbol.asyncDispose]: async (): Promise => undefined }; }; fixture.session.onReconcileAsync = async (): Promise => { events.push('reconcile'); input.read(); }; const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await leaseRequested.promise; input.edit(); events.push('edit'); const second = router.executeAsync( createRequest('second', OPERATION_C), new TestPhasedRequestClient('two') ); await settleAsync(); grantLease.resolve(); expect(await first).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(await second).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(events).toEqual(['edit', 'reconcile', 'run:C snapshot=1 disk=1']); expect(scheduleSpy).toHaveBeenCalledTimes(1); }); it('lets a request received before the reconcile began join the batch when it reaches the router during it', async () => { const input: TestWorkspaceInput = new TestWorkspaceInput(); const events: string[] = []; const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => { events.push(`run:C ${input.describe()}`); } }); const reconcileStarted: IDeferred = createDeferred(); const finishReconcile: IDeferred = createDeferred(); fixture.session.onReconcileAsync = async (): Promise => { events.push('reconcile:start'); input.read(); reconcileStarted.resolve(); await finishReconcile.promise; events.push('reconcile:end'); }; const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); // The second client changes project-c's input and the daemon receives its request, which then waits (for // example behind a graph load) and reaches the router only while the first batch reconciles. input.edit(); events.push('edit'); const secondReceivedTimeMs: number = performance.now(); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await reconcileStarted.promise; const second = router.executeAsync( createRequest('second', OPERATION_C), new TestPhasedRequestClient('two'), false, undefined, undefined, undefined, secondReceivedTimeMs ); await settleAsync(); finishReconcile.resolve(); expect(await first).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(await second).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(events).toEqual(['edit', 'reconcile:start', 'reconcile:end', 'run:C snapshot=1 disk=1']); expect(scheduleSpy).toHaveBeenCalledTimes(1); }); it('waits for connecting clients before the reconcile, so that a request one sends meanwhile joins the batch', async () => { const input: TestWorkspaceInput = new TestWorkspaceInput(); const events: string[] = []; const fixture: ITestRoutingFixture = createFixture({ actionCAsync: async (): Promise => { events.push(`run:C ${input.describe()}`); } }); fixture.session.onReconcileAsync = async (): Promise => { events.push('reconcile'); input.read(); }; const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); let second: Promise | undefined; const waitForConnectingClientsAsync: jest.Mock, []> = jest.fn(async (): Promise => { // A client that connected while the daemon was busy changes project-c's input, then sends a compatible build. events.push('wait'); input.edit(); second = router.executeAsync( createRequest('second', OPERATION_C), new TestPhasedRequestClient('two'), false, undefined, undefined, undefined, performance.now() ); await settleAsync(); }); const firstClient: TestPhasedRequestClient = new TestPhasedRequestClient('one'); firstClient.waitForConnectingClientsAsync = waitForConnectingClientsAsync; expect(await router.executeAsync(createRequest('first', OPERATION_A), firstClient)).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(await second).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(events).toEqual(['wait', 'reconcile', 'run:C snapshot=1 disk=1']); expect(scheduleSpy).toHaveBeenCalledTimes(1); expect(waitForConnectingClientsAsync).toHaveBeenCalledTimes(1); }); it('does not wait for connecting clients before a batch that no other request can join', async () => { const fixture: ITestRoutingFixture = createFixture(); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const waitForConnectingClientsAsync: jest.Mock, []> = jest.fn( async (): Promise => undefined ); const client: TestPhasedRequestClient = new TestPhasedRequestClient('one'); client.waitForConnectingClientsAsync = waitForConnectingClientsAsync; expect( await router.executeAsync({ ...createRequest('list', OPERATION_A), commandName: 'list' }, client) ).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(waitForConnectingClientsAsync).not.toHaveBeenCalled(); }); it('lets a late shared build wait past a default timeout while a compatible batch executes', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await operationStarted.promise; let lateSettled: boolean = false; const late = router .executeAsync( { ...createRequest('late', OPERATION_C), admission: { waitTimeoutIsDefault: true, waitTimeoutMs: 20 } }, new TestPhasedRequestClient('two') ) .finally(() => { lateSettled = true; }); await new Promise((resolve) => setTimeout(resolve, 100)); expect(lateSettled).toBe(false); releaseOperation.resolve(); const [firstResult, lateResult] = await Promise.all([first, late]); expect(firstResult.outcome).toBe('success'); expect(lateResult.outcome).toBe('success'); expect(lateResult.admissionErrorCode).toBeUndefined(); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(1); }); it('enforces an explicit timeout while a late shared build waits for a compatible batch', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); await operationStarted.promise; const lateResult: IDaemonPhasedRequestResult = await router.executeAsync( { ...createRequest('late', OPERATION_C), admission: { waitTimeoutMs: 20 } }, new TestPhasedRequestClient('two') ); expect(lateResult).toMatchObject({ admissionErrorCode: 'wait-timeout', outcome: 'failure' }); releaseOperation.resolve(); expect((await first).outcome).toBe('success'); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(0); }); it('serializes concurrent shared-read requests instead of merging or deadlocking them', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const scheduleSpy: jest.SpyInstance = jest.spyOn(fixture.graph, 'scheduleIterationAsync'); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const firstRequest: IDaemonPhasedRequest = { ...createRequest('first', OPERATION_A), commandName: 'list' }; const first = router.executeAsync(firstRequest, new TestPhasedRequestClient('one')); await operationStarted.promise; const second = router.executeAsync( { ...createRequest('second', OPERATION_C), commandName: 'list' }, new TestPhasedRequestClient('two') ); const third = router.executeAsync( { ...createRequest('third', OPERATION_C), commandName: 'list' }, new TestPhasedRequestClient('three') ); releaseOperation.resolve(); await Promise.all([first, second, third]); expect(scheduleSpy).toHaveBeenCalledTimes(3); }); it('applies graph admission to same-turn shared-read requests', async () => { const operationStarted: IDeferred = createDeferred(); const releaseOperation: IDeferred = createDeferred(); const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (): Promise => { operationStarted.resolve(); await releaseOperation.promise; } }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = router.executeAsync( { ...createRequest('first', OPERATION_A), commandName: 'list' }, new TestPhasedRequestClient('one') ); const noWait = router.executeAsync( { ...createRequest('no-wait', OPERATION_C), admission: { noWait: true }, commandName: 'list' }, new TestPhasedRequestClient('two') ); const noWaitResult: IDaemonPhasedRequestResult = await noWait; expect(noWaitResult).toMatchObject({ admissionErrorCode: 'no-wait', outcome: 'failure' }); await operationStarted.promise; releaseOperation.resolve(); await first; }); it('keeps true enabled state dominant across merged selections', async () => { const fixture: ITestRoutingFixture = createFixture(); const enabledStates: Array = []; fixture.graph.hooks.onIterationScheduled.tap('capture enabled state', () => { enabledStates.push(fixture.operations.get(OPERATION_A)?.enabled); }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync( { ...createRequest('ignore-dependency', OPERATION_A), operationSelection: [ { enabledState: 'ignore-dependency-changes', operationId: OPERATION_A } ] }, new TestPhasedRequestClient('one') ), router.executeAsync( createRequest('requires-dependency', OPERATION_B), new TestPhasedRequestClient('two') ) ]); expect(enabledStates).toEqual([true]); }); it('preserves per-client backpressure and final-result ordering in a merged batch', async () => { const fixture: ITestRoutingFixture = createFixture({ actionAAsync: async (terminal: ITerminal): Promise => { terminal.writeLine('first'); terminal.writeErrorLine('second'); } }); const clients: ReadonlyArray = [ new TestPhasedRequestClient('one'), new TestPhasedRequestClient('two') ]; const concurrentWrites: number[] = [0, 0]; const maximumConcurrentWrites: number[] = [0, 0]; clients.forEach((client: TestPhasedRequestClient, index: number) => { client.onWriteAsync = async (): Promise => { concurrentWrites[index]++; maximumConcurrentWrites[index] = Math.max( maximumConcurrentWrites[index], concurrentWrites[index] ); await new Promise((resolve) => setImmediate(resolve)); concurrentWrites[index]--; }; }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); await Promise.all([ router.executeAsync(createRequest('one', OPERATION_A), clients[0]), router.executeAsync(createRequest('two', OPERATION_A), clients[1]) ]); expect(maximumConcurrentWrites).toEqual([1, 1]); for (const client of clients) { expect(client.writes[client.writes.length - 1]?.result).toBeDefined(); } }); it('cleans up a failed batch so a later batch can execute', async () => { const fixture: ITestRoutingFixture = createFixture(); let schedulingCount: number = 0; fixture.graph.hooks.onIterationScheduled.tap('fail first batch', () => { if (schedulingCount++ === 0) { throw new Error('first batch failed'); } }); const router: PhasedRequestRouter = new PhasedRequestRouter(fixture.session); const first = await router.executeAsync( createRequest('first', OPERATION_A), new TestPhasedRequestClient('one') ); const second = await router.executeAsync( createRequest('second', OPERATION_C), new TestPhasedRequestClient('two') ); expect(first).toMatchObject({ errorMessage: 'first batch failed', outcome: 'failure' }); expect(second).toMatchObject({ exitCode: 0, outcome: 'success' }); expect(fixture.runners.get(OPERATION_C)?.runCount).toBe(1); }); }); /** The activity lines that announced an iteration to a client, from the `Selected` listing through the `Executing` line. */ function getIterationAnnouncement(client: TestPhasedRequestClient): ReadonlyArray { const texts: string[] = client.writes.flatMap(({ event }) => event?.type === 'activityChanged' && event.scope === undefined ? [(event.payload as { text: string }).text] : [] ); const executingIndexes: number[] = texts.flatMap((text: string, index: number) => text.startsWith('Executing a maximum of ') ? [index] : [] ); if (executingIndexes.length === 0) { expect(texts.some((text: string) => text.startsWith('Selected '))).toBe(false); return []; } expect(executingIndexes).toHaveLength(1); const selectedIndex: number = texts.findIndex((text: string) => text.startsWith('Selected ')); return texts.slice(selectedIndex === -1 ? executingIndexes[0] : selectedIndex, executingIndexes[0] + 1); } function getWrittenOperationIds(client: TestPhasedRequestClient): ReadonlySet { const operationIdSet: Set = new Set(); for (const write of client.writes) { if (write.operationId) { operationIdSet.add(write.operationId); } if (write.event) { const operationId: string | undefined = eventOperationId(write.event); if (operationId) { operationIdSet.add(operationId); } } } return operationIdSet; } function getHeaderData( client: TestPhasedRequestClient ): ReadonlyArray<{ completedOperations: number; operationId: string; totalOperations: number }> { return client.writes.flatMap(({ event }) => { const payload: unknown = event?.payload; if ( typeof payload !== 'object' || payload === null || (payload as { name?: unknown }).name !== RUSHD_OPERATION_HEADER ) { return []; } const data: unknown = (payload as { data?: unknown }).data; return typeof data === 'object' && data !== null ? [ data as { completedOperations: number; operationId: string; totalOperations: number; } ] : []; }); }