diff --git a/CHANGELOG.md b/CHANGELOG.md index 74226b5e7d..82034a1055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This is the log of notable changes to EAS CLI and related packages. - [eas-cli] Make `eas simulator` the canonical command and keep `eas simulator:start` as an alias. ([#4112](https://github.com/expo/eas-cli/pull/4112) by [@szdziedzic](https://github.com/szdziedzic)) - [eas-cli] Validate local composite functions referenced from workflow job hooks during `eas workflow:validate`. ([#4064](https://github.com/expo/eas-cli/pull/4064) by [@sswrk](https://github.com/sswrk)) - [eas-cli] Add `eas sim` as a shortcut for `eas simulator` commands, e.g. `eas sim:list` runs `eas simulator:list`. ([#4150](https://github.com/expo/eas-cli/pull/4150) by [@szdziedzic](https://github.com/szdziedzic)) +- [build-tools] Support an optional `max_idle_time_minutes` input on `eas/start_argent_remote_session` and `eas/start_agent_device_remote_session`; the step stops the device run session after that many minutes without observed session events. ([#4157](https://github.com/expo/eas-cli/pull/4157) by [@szdziedzic](https://github.com/szdziedzic)) ### 🐛 Bug fixes diff --git a/packages/build-tools/src/steps/functions/__tests__/startArgentRemoteSession-orchestration.test.ts b/packages/build-tools/src/steps/functions/__tests__/startArgentRemoteSession-orchestration.test.ts index 94f9b4b5b8..43834ed400 100644 --- a/packages/build-tools/src/steps/functions/__tests__/startArgentRemoteSession-orchestration.test.ts +++ b/packages/build-tools/src/steps/functions/__tests__/startArgentRemoteSession-orchestration.test.ts @@ -68,7 +68,10 @@ describe('createStartArgentRemoteSessionBuildFunction orchestration', () => { jest.mocked(pollArgentArtifactsForUploadAsync).mockResolvedValue(undefined); mockStopAsync.mockResolvedValue(undefined); mockTunnelStopAsync.mockResolvedValue(undefined); - jest.mocked(startArgentEventCollectionAsync).mockResolvedValue({ stopAsync: mockStopAsync }); + jest.mocked(startArgentEventCollectionAsync).mockResolvedValue({ + stopAsync: mockStopAsync, + getLastEventObservedAt: () => undefined, + }); jest.mocked(getDeviceRunSessionIdOrThrow).mockReturnValue('device-run-session-id'); jest.mocked(getNgrokTunnelDomainOrThrow).mockReturnValue('tunnel.example.com'); @@ -108,7 +111,10 @@ describe('createStartArgentRemoteSessionBuildFunction orchestration', () => { global: { runtimePlatform: BuildRuntimePlatform.LINUX }, } as unknown as BuildStepContext, { - inputs: { package_version: { value: undefined } }, + inputs: { + package_version: { value: undefined }, + max_idle_time_minutes: { value: undefined }, + }, outputs: {}, env: { EXISTING: 'value' }, } as never diff --git a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts index e73990886f..5e8a442ac1 100644 --- a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts @@ -55,6 +55,11 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( required: false, allowedValueTypeName: BuildStepInputValueTypeName.STRING, }), + BuildStepInput.createProvider({ + id: 'max_idle_time_minutes', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, + }), ], fn: async ({ logger, global }, { inputs, env, signal }) => { // Fail fast before any expensive setup if the injected env @@ -66,6 +71,8 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( const ngrokAuthtoken = getNgrokAuthtokenOrThrow(env); const packageVersion = inputs.package_version.value as string | undefined; + // A missing or non-positive value disables the idle timeout (opt-in feature). + const maxIdleTimeMinutes = inputs.max_idle_time_minutes.value as number | undefined; const { runtimePlatform } = global; logger.info( `Starting agent-device remote session (version: ${packageVersion ?? 'latest'}, runtime: ${runtimePlatform}).` @@ -140,6 +147,13 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( deviceRunSessionId, logger, signal, + idleTimeout: + maxIdleTimeMinutes !== undefined && maxIdleTimeMinutes > 0 + ? { + maxIdleTimeMinutes, + getLastEventObservedAt: eventCollection.getLastEventObservedAt, + } + : undefined, }); } finally { if (serveSim) { diff --git a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts index a62c2dfe91..3ca2c7a104 100644 --- a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts @@ -68,6 +68,11 @@ export function createStartArgentRemoteSessionBuildFunction( required: false, allowedValueTypeName: BuildStepInputValueTypeName.STRING, }), + BuildStepInput.createProvider({ + id: 'max_idle_time_minutes', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, + }), ], fn: async ({ logger, global }, { inputs, env, signal }) => { // Fail fast before any expensive setup if the injected env @@ -79,6 +84,8 @@ export function createStartArgentRemoteSessionBuildFunction( const ngrokAuthtoken = getNgrokAuthtokenOrThrow(env); const packageVersion = inputs.package_version.value as string | undefined; + // A missing or non-positive value disables the idle timeout (opt-in feature). + const maxIdleTimeMinutes = inputs.max_idle_time_minutes.value as number | undefined; warnIfArgentPackageVersionCannotBeVerified({ packageVersion, logger }); const versionSpec = packageVersion ?? 'latest'; const { runtimePlatform } = global; @@ -215,6 +222,13 @@ export function createStartArgentRemoteSessionBuildFunction( deviceRunSessionId, logger, signal, + idleTimeout: + maxIdleTimeMinutes !== undefined && maxIdleTimeMinutes > 0 + ? { + maxIdleTimeMinutes, + getLastEventObservedAt: eventCollection.getLastEventObservedAt, + } + : undefined, }); } finally { if (serveSim) { diff --git a/packages/build-tools/src/steps/utils/__tests__/agentDeviceEvents.test.ts b/packages/build-tools/src/steps/utils/__tests__/agentDeviceEvents.test.ts index 5527053833..638e9b9a55 100644 --- a/packages/build-tools/src/steps/utils/__tests__/agentDeviceEvents.test.ts +++ b/packages/build-tools/src/steps/utils/__tests__/agentDeviceEvents.test.ts @@ -159,6 +159,37 @@ describe(startAgentDeviceEventCollectionAsync, () => { ); }); + it('tracks the local arrival time of the most recent event', async () => { + const stateDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'agent-device-events-')); + const eventsDir = path.join(stateDir, 'sessions', 'default'); + await fs.promises.mkdir(eventsDir, { recursive: true }); + const collection = await startAgentDeviceEventCollectionAsync({ + ctx: createContext(), + deviceRunSessionId: 'session-id', + stateDir, + logger: createLogger(), + pollIntervalMs: 10, + }); + + try { + expect(collection.getLastEventObservedAt()).toBeUndefined(); + + const before = new Date(); + await fs.promises.writeFile( + path.join(eventsDir, 'events.ndjson'), + `${JSON.stringify(createAgentDeviceEvent({ requestId: 'request-1', command: 'tap' }))}\n` + ); + await waitForAsync(() => expect(mockEventLogStream.write).toHaveBeenCalledTimes(1)); + + const observedAt = collection.getLastEventObservedAt(); + expect(observedAt).toBeInstanceOf(Date); + expect(observedAt!.getTime()).toBeGreaterThanOrEqual(before.getTime()); + } finally { + await collection.stopAsync(); + await fs.promises.rm(stateDir, { recursive: true, force: true }); + } + }); + it('preserves unknown kinds and tolerates optional upstream field changes', async () => { const stateDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'agent-device-events-')); const eventsDir = path.join(stateDir, 'sessions', 'default'); @@ -547,6 +578,9 @@ describe(startAgentDeviceEventCollectionAsync, () => { ); expect(mockEventLogStream.init).not.toHaveBeenCalled(); expect(HttpLogStream).not.toHaveBeenCalled(); + // Without collection, activity is invisible: the collection reports fresh + // activity so an enabled idle timeout never stops the session. + expect(collection.getLastEventObservedAt()).toBeInstanceOf(Date); }); it('continues persisting the artifact when real-time log setup fails', async () => { diff --git a/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts b/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts index 9c6c33593f..87ffc7092a 100644 --- a/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts +++ b/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts @@ -51,7 +51,8 @@ function createStatusCtxMock( | { status: 'NEW' | 'IN_PROGRESS' | 'STOPPED' | 'ERRORED' } | { error: Error } | { data: unknown } - )[] + )[], + { ensureStoppedError }: { ensureStoppedError?: Error } = {} ): CustomBuildContext { const query = jest.fn(() => { const result = results.shift(); @@ -80,9 +81,25 @@ function createStatusCtxMock( }; }); + const mutation = jest.fn(() => ({ + toPromise: async () => { + if (ensureStoppedError) { + return { error: ensureStoppedError }; + } + return { + data: { + deviceRunSession: { + ensureDeviceRunSessionStopped: { id: 'drs-id', status: 'STOPPED' }, + }, + }, + }; + }, + })); + return { graphqlClient: { query, + mutation, }, } as unknown as CustomBuildContext; } @@ -423,6 +440,97 @@ describe(waitForDeviceRunSessionStoppedAsync, () => { { level: 'warning' } ); }); + + describe('with an idle timeout', () => { + beforeEach(() => { + // Fake timers make Date.now() advance by exactly the poll interval per + // loop iteration, so idle time accumulates deterministically. + jest.useFakeTimers(); + jest.mocked(setTimeoutAsync).mockImplementation(async delayMs => { + jest.advanceTimersByTime(delayMs ?? 0); + return undefined; + }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + function manyInProgressStatuses(): { status: 'IN_PROGRESS' }[] { + return Array.from({ length: 20 }, () => ({ status: 'IN_PROGRESS' as const })); + } + + it('stops the session when no activity is observed within the max idle time', async () => { + const ctx = createStatusCtxMock(manyInProgressStatuses()); + const logger = createLoggerMock(); + + await waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger, + idleTimeout: { + maxIdleTimeMinutes: 1, + getLastEventObservedAt: () => undefined, + }, + }); + + // One minute at the 5-second poll interval is 12 status polls. + expect(ctx.graphqlClient.query).toHaveBeenCalledTimes(12); + expect(ctx.graphqlClient.mutation).toHaveBeenCalledTimes(1); + expect(logger.info).toHaveBeenCalledWith( + 'Device run session drs-id had no activity for 1 minute(s) (max idle time). Stopping the session.' + ); + }); + + it('keeps the session alive while events keep arriving', async () => { + const ctx = createStatusCtxMock([...manyInProgressStatuses(), { status: 'STOPPED' }]); + const logger = createLoggerMock(); + + await waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger, + idleTimeout: { + maxIdleTimeMinutes: 1, + // Fresh activity on every check; 20 polls exceed one minute, so the + // session would have been stopped without these events. + getLastEventObservedAt: () => new Date(), + }, + }); + + expect(ctx.graphqlClient.query).toHaveBeenCalledTimes(21); + expect(ctx.graphqlClient.mutation).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith('Device run session drs-id was stopped.'); + }); + + it('still returns when the session cannot be marked stopped', async () => { + const ctx = createStatusCtxMock(manyInProgressStatuses(), { + ensureStoppedError: new Error('forbidden'), + }); + const logger = createLoggerMock(); + + await waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger, + idleTimeout: { + maxIdleTimeMinutes: 1, + getLastEventObservedAt: () => undefined, + }, + }); + + expect(ctx.graphqlClient.mutation).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + { err: expect.any(Error) }, + 'Could not mark device run session drs-id as stopped. The session job ends anyway.' + ); + expect(jest.mocked(Sentry).capture).toHaveBeenCalledWith( + 'Could not mark idle device run session as stopped', + expect.any(Error), + { level: 'warning', extras: { deviceRunSessionId: 'drs-id' } } + ); + }); + }); }); describe(ensureFfmpegInstalledAsync, () => { diff --git a/packages/build-tools/src/steps/utils/agentDeviceEvents.ts b/packages/build-tools/src/steps/utils/agentDeviceEvents.ts index e15904c07a..50b0bf3c47 100644 --- a/packages/build-tools/src/steps/utils/agentDeviceEvents.ts +++ b/packages/build-tools/src/steps/utils/agentDeviceEvents.ts @@ -6,6 +6,7 @@ import { z } from 'zod'; import { type CustomBuildContext } from '../../customBuildContext'; import { type DeviceRunSessionEvent, + type DeviceRunSessionEventCollection, type DeviceRunSessionEventParseFailure, startDeviceRunSessionEventCollectionAsync, } from './deviceRunSessionEvents'; @@ -46,7 +47,7 @@ export async function startAgentDeviceEventCollectionAsync({ stateDir: string; logger: bunyan; pollIntervalMs?: number; -}): Promise<{ stopAsync: () => Promise }> { +}): Promise { return startDeviceRunSessionEventCollectionAsync({ ctx, deviceRunSessionId, diff --git a/packages/build-tools/src/steps/utils/argentEvents.ts b/packages/build-tools/src/steps/utils/argentEvents.ts index 2e47073ee2..85a558dad5 100644 --- a/packages/build-tools/src/steps/utils/argentEvents.ts +++ b/packages/build-tools/src/steps/utils/argentEvents.ts @@ -6,6 +6,7 @@ import { z } from 'zod'; import { type CustomBuildContext } from '../../customBuildContext'; import { type DeviceRunSessionEvent, + type DeviceRunSessionEventCollection, type DeviceRunSessionEventParseFailure, startDeviceRunSessionEventCollectionAsync, } from './deviceRunSessionEvents'; @@ -75,7 +76,7 @@ export async function startArgentEventCollectionAsync({ eventLogPath: string; logger: bunyan; pollIntervalMs?: number; -}): Promise<{ stopAsync: () => Promise }> { +}): Promise { return startDeviceRunSessionEventCollectionAsync({ ctx, deviceRunSessionId, diff --git a/packages/build-tools/src/steps/utils/deviceRunSessionEvents.ts b/packages/build-tools/src/steps/utils/deviceRunSessionEvents.ts index 47dea296e9..a0d5e4243a 100644 --- a/packages/build-tools/src/steps/utils/deviceRunSessionEvents.ts +++ b/packages/build-tools/src/steps/utils/deviceRunSessionEvents.ts @@ -94,6 +94,17 @@ type EventFileState = { decoder: StringDecoder; }; +export type DeviceRunSessionEventCollection = { + stopAsync: () => Promise; + /** + * Local arrival time of the most recently collected event, or `undefined` + * when no event has been observed yet. Uses the collector's clock rather + * than the event's embedded timestamp so producer clock skew cannot affect + * idle detection. + */ + getLastEventObservedAt: () => Date | undefined; +}; + export async function startDeviceRunSessionEventCollectionAsync({ ctx, deviceRunSessionId, @@ -106,7 +117,7 @@ export async function startDeviceRunSessionEventCollectionAsync({ source: DeviceRunSessionEventSource; logger: bunyan; pollIntervalMs?: number; -}): Promise<{ stopAsync: () => Promise }> { +}): Promise { const { producer } = source; let didReportEventLogFailure = false; const reportEventLogFailure = (error: Error, operation: 'setup' | 'cleanup'): void => { @@ -136,7 +147,13 @@ export async function startDeviceRunSessionEventCollectionAsync({ const error = err instanceof Error ? err : new Error(String(err)); logger.warn({ err: error }, 'Could not persist device run session events to the artifact.'); reportEventLogFailure(error, 'setup'); - return { stopAsync: async () => {} }; + return { + stopAsync: async () => {}, + // Collection is disabled, so session activity is invisible here. Report + // fresh activity so an enabled idle timeout never stops a session it + // cannot observe. + getLastEventObservedAt: () => new Date(), + }; } let didReportRealtimeLogFailure = false; @@ -166,6 +183,7 @@ export async function startDeviceRunSessionEventCollectionAsync({ const states = new Map(); const controller = new AbortController(); + let lastEventObservedAt: Date | undefined; let parseFailureCount = 0; const parseFailureCounts: Record = { 'invalid-json': 0, @@ -190,6 +208,7 @@ export async function startDeviceRunSessionEventCollectionAsync({ source, deviceRunSessionId, writeEvent: event => { + lastEventObservedAt = new Date(); eventLogStream.write(event); realtimeLogStream?.write({ ...event, logId: event.eventId }); }, @@ -259,6 +278,7 @@ export async function startDeviceRunSessionEventCollectionAsync({ }); return { + getLastEventObservedAt: () => lastEventObservedAt, stopAsync: async () => { controller.abort(); await pollingPromise; diff --git a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts index 61a2dc0992..7505af4af7 100644 --- a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts +++ b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts @@ -50,6 +50,17 @@ const DEVICE_RUN_SESSION_STATUS_QUERY = graphql(` } `); +const ENSURE_DEVICE_RUN_SESSION_STOPPED_MUTATION = graphql(` + mutation EnsureDeviceRunSessionStopped($deviceRunSessionId: ID!) { + deviceRunSession { + ensureDeviceRunSessionStopped(deviceRunSessionId: $deviceRunSessionId) { + id + status + } + } + } +`); + const DEVICE_RUN_SESSION_STATUS_POLL_INTERVAL_MS = 5_000; export function getDeviceRunSessionIdOrThrow(env: BuildStepEnv): string { @@ -118,23 +129,56 @@ export async function selectXcodeDeveloperDirectoryAsync({ }); } +export type DeviceRunSessionIdleTimeout = { + /** Stop the session after this many minutes without observed activity. */ + maxIdleTimeMinutes: number; + /** + * Local arrival time of the most recent session event, or `undefined` when + * no event has been observed yet. The idle clock starts when the wait + * begins, so a session nobody ever connects to still times out. + */ + getLastEventObservedAt: () => Date | undefined; +}; + export async function waitForDeviceRunSessionStoppedAsync({ ctx, deviceRunSessionId, logger, signal, + idleTimeout, }: { ctx: CustomBuildContext; deviceRunSessionId: string; logger: bunyan; signal?: AbortSignal; + idleTimeout?: DeviceRunSessionIdleTimeout; }): Promise { logger.info( `Remote session is live. Polling device run session ${deviceRunSessionId} until it is stopped.` ); + if (idleTimeout) { + logger.info( + `The session stops automatically after ${idleTimeout.maxIdleTimeMinutes} minute(s) without activity.` + ); + } let pollErrorCount = 0; + let lastActivityAt = new Date(); while (!signal?.aborted) { + if (idleTimeout) { + const lastEventObservedAt = idleTimeout.getLastEventObservedAt(); + if (lastEventObservedAt && lastEventObservedAt > lastActivityAt) { + lastActivityAt = lastEventObservedAt; + } + if (Date.now() - lastActivityAt.getTime() >= idleTimeout.maxIdleTimeMinutes * 60_000) { + logger.info( + `Device run session ${deviceRunSessionId} had no activity for ` + + `${idleTimeout.maxIdleTimeMinutes} minute(s) (max idle time). Stopping the session.` + ); + await ensureDeviceRunSessionStoppedSafelyAsync({ ctx, deviceRunSessionId, logger }); + return; + } + } try { const result = await ctx.graphqlClient .query(DEVICE_RUN_SESSION_STATUS_QUERY, { deviceRunSessionId }) @@ -174,6 +218,37 @@ export async function waitForDeviceRunSessionStoppedAsync({ } } +// Best effort: when this fails, the caller still tears the session down and the +// job run finishes, which clients also treat as the session ending. +async function ensureDeviceRunSessionStoppedSafelyAsync({ + ctx, + deviceRunSessionId, + logger, +}: { + ctx: CustomBuildContext; + deviceRunSessionId: string; + logger: bunyan; +}): Promise { + try { + const result = await ctx.graphqlClient + .mutation(ENSURE_DEVICE_RUN_SESSION_STOPPED_MUTATION, { deviceRunSessionId }) + .toPromise(); + if (result.error) { + throw result.error; + } + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + Sentry.capture('Could not mark idle device run session as stopped', error, { + level: 'warning', + extras: { deviceRunSessionId }, + }); + logger.warn( + { err: error }, + `Could not mark device run session ${deviceRunSessionId} as stopped. The session job ends anyway.` + ); + } +} + async function sleepUntilAbortedAsync( timeoutMs: number, signal: AbortSignal | undefined