diff --git a/src/__tests__/client-shared.test.ts b/src/__tests__/client-shared.test.ts index bb6f41db8..356e0fc19 100644 --- a/src/__tests__/client-shared.test.ts +++ b/src/__tests__/client-shared.test.ts @@ -151,3 +151,31 @@ test('serializeSnapshotResult maps capture quality annotation to public snapshot snapshotQuality, }); }); + +test('serializeSnapshotResult includes snapshot diagnostics', () => { + const snapshotDiagnostics = { + stats: { + count: 3, + p50Ms: 450, + p95Ms: 1_800, + maxMs: 1_800, + slowThresholdMs: 1_500, + platform: 'android', + }, + warning: 'Warning: android snapshots are slow in this run: p95 1800ms over 3 captures.', + } as const; + const data = serializeSnapshotResult({ + nodes: [], + truncated: false, + snapshotDiagnostics, + identifiers: { + session: 'qa', + }, + }); + + assert.deepEqual(data, { + nodes: [], + truncated: false, + snapshotDiagnostics, + }); +}); diff --git a/src/__tests__/snapshot-diagnostics.test.ts b/src/__tests__/snapshot-diagnostics.test.ts new file mode 100644 index 000000000..568c2bcdd --- /dev/null +++ b/src/__tests__/snapshot-diagnostics.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'vitest'; +import { + mergeSnapshotDiagnostics, + recordSnapshotTiming, + summarizeSnapshotDiagnostics, +} from '../snapshot-diagnostics.ts'; + +test('records session snapshot timing stats', () => { + const session = {}; + + recordSnapshotTiming(session, { durationMs: 400, backend: 'android', platform: 'android' }); + recordSnapshotTiming(session, { durationMs: 2_100, backend: 'android', platform: 'android' }); + + expect(summarizeSnapshotDiagnostics(session)).toEqual({ + stats: { + count: 2, + p50Ms: 400, + p95Ms: 2_100, + maxMs: 2_100, + slowThresholdMs: 1_500, + platform: 'android', + backends: { android: 2 }, + }, + warning: expect.stringContaining('p95 2100ms over 2 captures'), + }); +}); + +test('merges snapshot diagnostics without inflating capture count', () => { + const merged = mergeSnapshotDiagnostics([ + { + stats: { + count: 1, + p50Ms: 300, + p95Ms: 300, + maxMs: 300, + slowThresholdMs: 1_500, + platform: 'android', + }, + }, + { + stats: { + count: 2, + p50Ms: 500, + p95Ms: 1_900, + maxMs: 1_900, + slowThresholdMs: 1_500, + platform: 'android', + }, + }, + ]); + + expect(merged?.stats).toMatchObject({ + count: 3, + p50Ms: 500, + p95Ms: 1_900, + maxMs: 1_900, + platform: 'android', + }); + expect(merged?.warning).toContain('p95 1900ms over 3 captures'); +}); diff --git a/src/backend.ts b/src/backend.ts index eccc00018..d389f341f 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -15,6 +15,7 @@ import type { ClickButton } from './core/click-button.ts'; import type { DeviceRotation } from './core/device-rotation.ts'; import type { ScrollDirection } from './core/scroll-gesture.ts'; import type { SessionSurface } from './core/session-surface.ts'; +import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; import type { SnapshotCaptureAnalysis, SnapshotCaptureAnnotations, @@ -49,6 +50,7 @@ export type BackendSnapshotResult = { snapshot?: SnapshotState; appName?: string; appBundleId?: string; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; } & SnapshotCaptureAnnotations; export type BackendSnapshotOptions = SnapshotOptions & { diff --git a/src/client-shared.ts b/src/client-shared.ts index f3bc60673..57230a73d 100644 --- a/src/client-shared.ts +++ b/src/client-shared.ts @@ -178,6 +178,7 @@ export function serializeSnapshotResult(result: CaptureSnapshotResult): Record > { const visibility = readObject(data.visibility); const unchanged = readObject(data.unchanged); + const snapshotDiagnostics = readSnapshotDiagnosticsSummary(data.snapshotDiagnostics); return { ...(visibility ? { visibility: visibility as CaptureSnapshotResult['visibility'] } : {}), ...readSerializedSnapshotCaptureAnnotations(data), ...(unchanged ? { unchanged: unchanged as CaptureSnapshotResult['unchanged'] } : {}), + ...(snapshotDiagnostics ? { snapshotDiagnostics } : {}), }; } diff --git a/src/commands/capture/index.test.ts b/src/commands/capture/index.test.ts index b05bb9be6..90a5e50b1 100644 --- a/src/commands/capture/index.test.ts +++ b/src/commands/capture/index.test.ts @@ -12,6 +12,7 @@ import { waitCliReader, waitDaemonWriter, } from './index.ts'; +import { snapshotCliOutput } from './output.ts'; function flags(overrides: Partial = {}): CliFlags { return overrides as CliFlags; @@ -50,6 +51,32 @@ describe('capture command interface', () => { }); }); + test('routes snapshot diagnostics warning to stderr output', () => { + const output = snapshotCliOutput({ + result: { + nodes: [], + truncated: false, + identifiers: {}, + snapshotDiagnostics: { + stats: { + count: 2, + p50Ms: 400, + p95Ms: 1_900, + maxMs: 1_900, + slowThresholdMs: 1_500, + platform: 'ios', + }, + warning: 'Warning: ios snapshots are slow in this run: p95 1900ms over 2 captures.', + }, + }, + }); + + expect(output.stderr).toBe( + 'Warning: ios snapshots are slow in this run: p95 1900ms over 2 captures.\n', + ); + expect(output.text).not.toContain('snapshots are slow'); + }); + test('reads screenshot path and writes screenshot flags', () => { const input = screenshotCliReader( ['page.png'], diff --git a/src/commands/capture/output.ts b/src/commands/capture/output.ts index 7425cc823..273e4e4ce 100644 --- a/src/commands/capture/output.ts +++ b/src/commands/capture/output.ts @@ -16,6 +16,9 @@ export function snapshotCliOutput(params: { data, // Programmatic SDK callers can see `unchanged`; CLI --json hides it for schema compatibility. jsonData: withoutUnchanged(data), + stderr: params.result.snapshotDiagnostics?.warning + ? `${params.result.snapshotDiagnostics.warning}\n` + : undefined, text: formatSnapshotText(data, { raw: params.raw, flatten: params.interactiveOnly, diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index c9d93064a..41ec783fe 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -1,4 +1,5 @@ import type { BackendSnapshotResult } from '../../../backend.ts'; +import type { SnapshotDiagnosticsSummary } from '../../../snapshot-diagnostics.ts'; import type { AgentDeviceRuntime, CommandSessionRecord } from '../../../runtime-contract.ts'; import { publicSnapshotCaptureAnnotations, @@ -38,6 +39,7 @@ export type SnapshotCommandResult = { appBundleId?: string; visibility?: SnapshotVisibility; unchanged?: SnapshotUnchanged; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; } & PublicSnapshotCaptureAnnotations; export type DiffSnapshotCommandResult = { @@ -84,6 +86,9 @@ export const snapshotCommand: RuntimeCommand< warnings: capture.warnings, }), ...(unchanged ? { unchanged } : {}), + ...(capture.result.snapshotDiagnostics + ? { snapshotDiagnostics: capture.result.snapshotDiagnostics } + : {}), ...snapshotAppFields(capture), }; }; diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 643de7c03..6a4287fbb 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -8,6 +8,7 @@ import { runCmdBackground, type ExecBackgroundResult } from '../../../utils/exec import type { DaemonInvokeFn, DaemonRequest, DaemonResponse, SessionAction } from '../../types.ts'; import type { CommandFlags } from '../../../core/dispatch.ts'; import { SessionStore } from '../../session-store.ts'; +import { makeIosSession } from '../../../__tests__/test-utils/index.ts'; import { buildReplayVarScope, collectReplayShellEnv, @@ -475,6 +476,110 @@ test('runReplayScriptFile dispatches resolved literals with file env overridden } }); +test('runReplayScriptFile reports snapshot diagnostics from per-action session samples', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-snapshot-samples-')); + const scriptPath = path.join(root, 'flow.ad'); + fs.writeFileSync(scriptPath, ['snapshot', 'snapshot', ''].join('\n')); + const sessionStore = new SessionStore(path.join(root, 'state')); + sessionStore.set( + 's', + makeIosSession('s', { + snapshotDiagnostics: { samples: [] }, + }), + ); + let captures = 0; + + const response = await runReplayScriptFile({ + req: { + token: 't', + session: 's', + command: 'replay', + positionals: [scriptPath], + meta: { cwd: root }, + }, + sessionName: 's', + logPath: path.join(root, 'log'), + sessionStore, + invoke: async (): Promise => { + captures += 1; + const session = sessionStore.get('s'); + session?.snapshotDiagnostics?.samples.push({ + durationMs: captures === 1 ? 400 : 1_900, + backend: 'xctest', + platform: 'ios', + }); + return { + ok: true, + data: { + snapshotDiagnostics: { + stats: { + count: captures, + p50Ms: captures === 1 ? 400 : 1_900, + p95Ms: captures === 1 ? 400 : 1_900, + maxMs: captures === 1 ? 400 : 1_900, + slowThresholdMs: 1_500, + platform: 'ios', + }, + }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + const diagnostics = response.data?.snapshotDiagnostics as + | { stats?: { count?: number }; warning?: string } + | undefined; + assert.equal(diagnostics?.stats?.count, 2); + assert.match(String(diagnostics?.warning), /p95 1900ms over 2 captures/); +}); + +test('runReplayScriptFile reports snapshot diagnostics on replay failure', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-snapshot-failure-')); + const scriptPath = path.join(root, 'flow.ad'); + fs.writeFileSync(scriptPath, ['snapshot', 'click "Missing"', ''].join('\n')); + const sessionStore = new SessionStore(path.join(root, 'state')); + sessionStore.set( + 's', + makeIosSession('s', { + snapshotDiagnostics: { samples: [] }, + }), + ); + let captures = 0; + + const response = await runReplayScriptFile({ + req: { + token: 't', + session: 's', + command: 'replay', + positionals: [scriptPath], + meta: { cwd: root }, + }, + sessionName: 's', + logPath: path.join(root, 'log'), + sessionStore, + invoke: async (): Promise => { + captures += 1; + const session = sessionStore.get('s'); + session?.snapshotDiagnostics?.samples.push({ + durationMs: captures === 1 ? 450 : 2_100, + backend: 'xctest', + platform: 'ios', + }); + if (captures === 1) return { ok: true, data: {} }; + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'button missing' } }; + }, + }); + + assert.equal(response.ok, false); + const diagnostics = response.error.details?.snapshotDiagnostics as + | { stats?: { count?: number; p95Ms?: number }; warning?: string } + | undefined; + assert.equal(diagnostics?.stats?.count, 2); + assert.equal(diagnostics?.stats?.p95Ms, 2_100); + assert.match(String(diagnostics?.warning), /p95 2100ms over 2 captures/); +}); + test('runReplayScriptFile applies CLI env overrides before Maestro compat mapping', async () => { const { response, calls } = await runReplayFixture({ label: 'maestro-env', diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 9eef59ed4..f732be1b7 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -14,6 +14,7 @@ import { } from '../../request-cancel.ts'; import { withDeviceInventoryProvider } from '../../../core/dispatch-resolve.ts'; import type { DeviceInfo } from '../../../utils/device.ts'; +import { makeAndroidSession } from '../../../__tests__/test-utils/index.ts'; function makeSessionStore(): SessionStore { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-session-test-suite-')); @@ -241,6 +242,120 @@ test('test emits skip progress without synthetic duration', async () => { expect(testEvents[0]?.durationMs).toBeUndefined(); }); +test('test aggregates snapshot diagnostics from replay session samples', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-snapshots-')); + fs.writeFileSync(path.join(root, '01-first.ad'), 'context platform=android\nopen "Demo"\n'); + fs.writeFileSync(path.join(root, '02-second.ad'), 'context platform=android\nopen "Demo"\n'); + let captures = 0; + + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-snapshot-diagnostics' }, + flags: { platform: 'android' }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async (req) => { + const session = + sessionStore.get(req.session) ?? + makeAndroidSession(req.session, { + snapshotDiagnostics: { samples: [] }, + }); + session.snapshotDiagnostics ??= { samples: [] }; + captures += 1; + session.snapshotDiagnostics.samples.push({ + durationMs: captures === 1 ? 400 : 1_900, + backend: 'android', + platform: 'android', + }); + sessionStore.set(req.session, session); + return { ok: true, data: { replayed: 1, healed: 0 } }; + }, + }); + + const data = expectOkData(response); + expect(data.snapshotDiagnostics).toMatchObject({ + stats: { + count: 2, + p50Ms: 400, + p95Ms: 1_900, + maxMs: 1_900, + platform: 'android', + }, + warning: expect.stringContaining('p95 1900ms over 2 captures'), + }); + expect((data.tests as Array>)[1]?.snapshotDiagnostics).toMatchObject({ + stats: { + count: 1, + p95Ms: 1_900, + }, + }); +}); + +test('test aggregates snapshot diagnostics from failed replay session samples', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-snapshot-fail-')); + fs.writeFileSync(path.join(root, '01-fail.ad'), 'context platform=android\nopen "Demo"\n'); + + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-snapshot-diagnostics-fail' }, + flags: { platform: 'android' }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async (req) => { + const session = + sessionStore.get(req.session) ?? + makeAndroidSession(req.session, { + snapshotDiagnostics: { samples: [] }, + }); + session.snapshotDiagnostics ??= { samples: [] }; + session.snapshotDiagnostics.samples.push({ + durationMs: 2_100, + backend: 'android', + platform: 'android', + }); + sessionStore.set(req.session, session); + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'open failed' }, + }; + }, + }); + + const data = expectOkData(response); + expect(data.failed).toBe(1); + expect(data.snapshotDiagnostics).toMatchObject({ + stats: { + count: 1, + p95Ms: 2_100, + platform: 'android', + }, + warning: expect.stringContaining('p95 2100ms over 1 captures'), + }); + expect((data.tests as Array>)[0]).toMatchObject({ + status: 'failed', + snapshotDiagnostics: { + stats: { + count: 1, + p95Ms: 2_100, + }, + }, + }); +}); + test('test stops the suite when the parent request is canceled during an active replay attempt', async () => { const sessionStore = makeSessionStore(); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-parent-cancel-')); diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index b1dfaf09c..ee751ec03 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -17,6 +17,11 @@ import { readReplayCliEnvEntries, readReplayShellEnvSource, } from '../../replay/vars.ts'; +import { + summarizeSnapshotTimingSamples, + type SnapshotDiagnosticsSummary, + type SnapshotTimingSample, +} from '../../snapshot-diagnostics.ts'; // fallow-ignore-next-line complexity export async function runReplayScriptFile(params: { @@ -82,11 +87,13 @@ export async function runReplayScriptFile(params: { }); const shouldUpdate = req.flags?.replayUpdate === true; const actionTracePath = tracePath ?? sessionStore.get(sessionName)?.trace?.outPath; + const snapshotDiagnosticSamples: SnapshotTimingSample[] = []; let healed = 0; for (let index = 0; index < actions.length; index += 1) { const action = actions[index]; if (!action || action.command === 'replay') continue; + const sampleStart = readSessionSnapshotSampleCount(sessionStore, sessionName); let response = await invokeReplayAction({ req: replayReq, sessionName, @@ -98,13 +105,23 @@ export async function runReplayScriptFile(params: { tracePath: actionTracePath, invoke, }); + snapshotDiagnosticSamples.push( + ...readSessionSnapshotSamplesSince(sessionStore, sessionName, sampleStart), + ); if (response.ok) { collectReplayActionArtifactPaths(response).forEach((entry) => artifactPaths.add(entry)); continue; } collectReplayActionArtifactPaths(response).forEach((entry) => artifactPaths.add(entry)); if (!shouldUpdate) { - return withReplayFailureContext(response, action, index, resolved, [...artifactPaths]); + return withReplayFailureDiagnostics( + response, + action, + index, + resolved, + [...artifactPaths], + snapshotDiagnosticSamples, + ); } const nextAction = await healReplayAction({ @@ -114,10 +131,18 @@ export async function runReplayScriptFile(params: { sessionStore, }); if (!nextAction) { - return withReplayFailureContext(response, action, index, resolved, [...artifactPaths]); + return withReplayFailureDiagnostics( + response, + action, + index, + resolved, + [...artifactPaths], + snapshotDiagnosticSamples, + ); } actions[index] = nextAction; + const healedSampleStart = readSessionSnapshotSampleCount(sessionStore, sessionName); response = await invokeReplayAction({ req: replayReq, sessionName, @@ -129,9 +154,19 @@ export async function runReplayScriptFile(params: { tracePath: actionTracePath, invoke, }); + snapshotDiagnosticSamples.push( + ...readSessionSnapshotSamplesSince(sessionStore, sessionName, healedSampleStart), + ); if (!response.ok) { collectReplayActionArtifactPaths(response).forEach((entry) => artifactPaths.add(entry)); - return withReplayFailureContext(response, nextAction, index, resolved, [...artifactPaths]); + return withReplayFailureDiagnostics( + response, + nextAction, + index, + resolved, + [...artifactPaths], + snapshotDiagnosticSamples, + ); } collectReplayActionArtifactPaths(response).forEach((entry) => artifactPaths.add(entry)); healed += 1; @@ -140,6 +175,7 @@ export async function runReplayScriptFile(params: { if (shouldUpdate && healed > 0) { writeReplayScript(resolved, actions, sessionStore.get(sessionName)); } + const snapshotDiagnosticsSummary = summarizeSnapshotTimingSamples(snapshotDiagnosticSamples); return { ok: true, data: { @@ -147,6 +183,7 @@ export async function runReplayScriptFile(params: { healed, session: sessionName, artifactPaths: [...artifactPaths], + ...(snapshotDiagnosticsSummary ? { snapshotDiagnostics: snapshotDiagnosticsSummary } : {}), }, }; } catch (err) { @@ -211,12 +248,31 @@ function buildReplayMetadataFlags( }; } +function withReplayFailureDiagnostics( + response: DaemonResponse, + action: SessionAction, + index: number, + replayPath: string, + artifactPaths: string[], + snapshotDiagnosticSamples: SnapshotTimingSample[], +): DaemonResponse { + return withReplayFailureContext( + response, + action, + index, + replayPath, + artifactPaths, + summarizeSnapshotTimingSamples(snapshotDiagnosticSamples), + ); +} + function withReplayFailureContext( response: DaemonResponse, action: SessionAction, index: number, replayPath: string, artifactPaths: string[] = [], + snapshotDiagnostics?: SnapshotDiagnosticsSummary, ): DaemonResponse { if (response.ok) return response; const step = index + 1; @@ -235,6 +291,7 @@ function withReplayFailureContext( action: action.command, positionals: action.positionals ?? [], artifactPaths, + ...(snapshotDiagnostics ? { snapshotDiagnostics } : {}), }, }, }; @@ -274,6 +331,18 @@ export function collectReplayActionArtifactPaths(response: DaemonResponse): stri return [...new Set(candidates.filter((candidate) => isReplayArtifactPath(candidate)))]; } +function readSessionSnapshotSampleCount(sessionStore: SessionStore, sessionName: string): number { + return sessionStore.get(sessionName)?.snapshotDiagnostics?.samples.length ?? 0; +} + +function readSessionSnapshotSamplesSince( + sessionStore: SessionStore, + sessionName: string, + start: number, +): SnapshotTimingSample[] { + return sessionStore.get(sessionName)?.snapshotDiagnostics?.samples.slice(start) ?? []; +} + function isReplayArtifactPath(candidate: string): boolean { try { return fs.statSync(candidate).isFile(); diff --git a/src/daemon/handlers/session-test-attempt.ts b/src/daemon/handlers/session-test-attempt.ts index b734abfb9..286674010 100644 --- a/src/daemon/handlers/session-test-attempt.ts +++ b/src/daemon/handlers/session-test-attempt.ts @@ -16,6 +16,7 @@ import { runReplayTestAttempt } from './session-test-runtime.ts'; import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; import type { ReplayTestShardContext } from './session-test-sharding.ts'; import { isRequestCanceled } from '../request-cancel.ts'; +import { readSnapshotDiagnosticsSummary } from '../../snapshot-diagnostics.ts'; type ReplayTestCaseResult = Extract; type ReplayTestAttemptFailure = NonNullable< @@ -271,9 +272,9 @@ function buildReplayTestPassedResult( finalAttemptDurationMs: outcome.finalAttemptDurationMs, attempts: outcome.attempts, artifactsDir: context.testArtifactsDir, - replayed: typeof response.data?.replayed === 'number' ? response.data.replayed : 0, - healed: typeof response.data?.healed === 'number' ? response.data.healed : 0, + ...replayTestResponseMetrics(response), ...replayTestWarningsResultMetadata(response.data?.warnings), + ...replayTestSnapshotDiagnosticsResultMetadata(response.data?.snapshotDiagnostics), ...replayTestShardResultMetadata(shard), ...(outcome.attemptFailures.length > 0 ? { attemptFailures: outcome.attemptFailures } : {}), }; @@ -311,6 +312,9 @@ function buildReplayTestFailedResult( attempts: outcome.attempts, artifactsDir: context.testArtifactsDir, error, + ...replayTestSnapshotDiagnosticsResultMetadata( + readReplayResponseSnapshotDiagnostics(outcome.finalResponse), + ), ...replayTestShardResultMetadata(shard), }; } @@ -322,6 +326,15 @@ function replayTestFailureError( return { code: 'COMMAND_FAILED', message: 'Unknown replay test failure' }; } +function replayTestResponseMetrics( + response: Extract, +): Pick, 'replayed' | 'healed'> { + return { + replayed: typeof response.data?.replayed === 'number' ? response.data.replayed : 0, + healed: typeof response.data?.healed === 'number' ? response.data.healed : 0, + }; +} + function replayTestWarningsResultMetadata( warnings: unknown, ): Pick, 'warnings'> { @@ -330,6 +343,19 @@ function replayTestWarningsResultMetadata( return filtered.length > 0 ? { warnings: filtered } : {}; } +function replayTestSnapshotDiagnosticsResultMetadata( + value: unknown, +): Pick { + const snapshotDiagnostics = readSnapshotDiagnosticsSummary(value); + return snapshotDiagnostics ? { snapshotDiagnostics } : {}; +} + +function readReplayResponseSnapshotDiagnostics(response: DaemonResponse | undefined): unknown { + return response?.ok + ? response.data?.snapshotDiagnostics + : response?.error.details?.snapshotDiagnostics; +} + function replayTestShardResultMetadata( shard: ReplayTestShardContext | undefined, ): Pick { diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 86bc25105..ac837c029 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -25,6 +25,7 @@ import { type ReplayTestShardPlan, } from './session-test-sharding.ts'; import { isRequestCanceled } from '../request-cancel.ts'; +import { mergeSnapshotDiagnostics } from '../../snapshot-diagnostics.ts'; type ReplayTestEntry = ReturnType[number]; type ReplayTestQueuedEntry = { @@ -408,6 +409,9 @@ function summarizeReplayTestResults( const failed = failedResults.length; const skipped = results.filter((result) => result.status === 'skipped').length; const executed = passed + failed; + const snapshotDiagnostics = mergeSnapshotDiagnostics( + results.map((result) => (result.status === 'skipped' ? undefined : result.snapshotDiagnostics)), + ); return { total, executed, @@ -418,5 +422,6 @@ function summarizeReplayTestResults( durationMs, failures: failedResults, tests: results, + ...(snapshotDiagnostics ? { snapshotDiagnostics } : {}), }; } diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 0403e333f..c83d4c81f 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -47,6 +47,7 @@ import { snapshotCaptureAnnotationsFrom, type SnapshotCaptureAnnotations, } from '../../snapshot-capture-annotations.ts'; +import { recordSnapshotTiming } from '../../snapshot-diagnostics.ts'; type CaptureSnapshotParams = { device: SessionState['device']; @@ -306,7 +307,13 @@ async function capturePostActionSnapshotAttempt( } async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise { + const startedAt = Date.now(); const data = await captureSnapshotData(params); + recordSnapshotTiming(params.session, { + durationMs: Date.now() - startedAt, + backend: data.backend, + platform: params.device.platform, + }); return { data, snapshot: buildSnapshotState(data, resolveSnapshotStateFlags(params)), diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index cecacab99..cbfa24ec1 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -17,6 +17,7 @@ import { import { createDaemonRuntimePolicy } from './runtime-policy.ts'; import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { maybeBuildAndroidSnapshotTimeoutFailure } from './android-snapshot-timeout-evidence.ts'; +import { summarizeSnapshotDiagnostics } from '../snapshot-diagnostics.ts'; export async function dispatchSnapshotViaRuntime(params: { req: DaemonRequest; @@ -284,9 +285,11 @@ function createDaemonSnapshotBackend(params: { logPath, snapshotScope, }); + const snapshotDiagnostics = summarizeSnapshotDiagnostics(session); return { snapshot: capture.snapshot, ...snapshotCaptureAnnotationsFrom(capture), + ...(snapshotDiagnostics ? { snapshotDiagnostics } : {}), appName: session?.appBundleId ? (session.appName ?? session.appBundleId) : undefined, appBundleId: session?.appBundleId, }; diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 156ca6823..bf8ff6b5a 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -23,6 +23,10 @@ import type { AppleXctracePerfCapture, AppleXctracePerfMode, } from '../platforms/ios/perf-xctrace.ts'; +import type { + SnapshotDiagnosticsState, + SnapshotDiagnosticsSummary, +} from '../snapshot-diagnostics.ts'; export type DaemonInstallSource = PublicDaemonInstallSource; export type SessionRuntimeHints = PublicSessionRuntimeHints; @@ -69,6 +73,7 @@ export type ReplaySuiteTestPassed = { shardIndex?: number; shardCount?: number; deviceId?: string; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; }; export type ReplaySuiteTestFailed = { @@ -83,6 +88,7 @@ export type ReplaySuiteTestFailed = { shardIndex?: number; shardCount?: number; deviceId?: string; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; }; export type ReplaySuiteTestSkipped = { @@ -115,6 +121,7 @@ export type ReplaySuiteResult = { durationMs: number; failures: ReplaySuiteTestFailed[]; tests: ReplaySuiteTestResult[]; + snapshotDiagnostics?: SnapshotDiagnosticsSummary; }; export type DaemonResponse = PublicDaemonResponse; @@ -231,6 +238,7 @@ export type SessionState = { androidSnapshotFreshness?: AndroidSnapshotFreshness; postGestureStabilization?: PostGestureStabilization; pendingInteractionOutcome?: PendingInteractionOutcome; + snapshotDiagnostics?: SnapshotDiagnosticsState; trace?: { outPath: string; startedAt: number; diff --git a/src/snapshot-diagnostics.ts b/src/snapshot-diagnostics.ts new file mode 100644 index 000000000..5dad88c78 --- /dev/null +++ b/src/snapshot-diagnostics.ts @@ -0,0 +1,201 @@ +import type { SnapshotBackend } from './utils/snapshot.ts'; +import type { Platform } from './utils/device.ts'; + +const SLOW_SNAPSHOT_P95_WARNING_MS = 1_500; + +export type SnapshotTimingSample = { + durationMs: number; + backend?: SnapshotBackend; + platform?: Platform; +}; + +export type SnapshotTimingStats = { + count: number; + p50Ms: number; + p95Ms: number; + maxMs: number; + slowThresholdMs: number; + platform?: Platform; + backends?: Record; +}; + +export type SnapshotDiagnosticsState = { + samples: SnapshotTimingSample[]; +}; + +export type SnapshotDiagnosticsSummary = { + stats: SnapshotTimingStats; + warning?: string; +}; + +export function recordSnapshotTiming( + session: { snapshotDiagnostics?: SnapshotDiagnosticsState } | undefined, + sample: SnapshotTimingSample, +): void { + if (!session) return; + const diagnostics = (session.snapshotDiagnostics ??= { samples: [] }); + diagnostics.samples.push({ + ...sample, + durationMs: Math.max(0, Math.round(sample.durationMs)), + }); +} + +export function summarizeSnapshotDiagnostics( + session: { snapshotDiagnostics?: SnapshotDiagnosticsState } | undefined, +): SnapshotDiagnosticsSummary | undefined { + const samples = session?.snapshotDiagnostics?.samples; + if (!samples || samples.length === 0) return undefined; + return summarizeSnapshotTimingSamples(samples); +} + +export function summarizeSnapshotTimingSamples( + samples: SnapshotTimingSample[], +): SnapshotDiagnosticsSummary | undefined { + if (samples.length === 0) return undefined; + const stats = buildSnapshotTimingStats(samples); + return { + stats, + ...(stats.p95Ms >= SLOW_SNAPSHOT_P95_WARNING_MS + ? { warning: formatSlowSnapshotWarning(stats) } + : {}), + }; +} + +export function mergeSnapshotDiagnostics( + summaries: Array, +): SnapshotDiagnosticsSummary | undefined { + const samples = summaries.flatMap((summary) => samplesFromStats(summary?.stats)); + if (samples.length === 0) return undefined; + const stats = buildSnapshotTimingStats(samples); + return { + stats, + ...(stats.p95Ms >= SLOW_SNAPSHOT_P95_WARNING_MS + ? { warning: formatSlowSnapshotWarning(stats) } + : {}), + }; +} + +export function readSnapshotDiagnosticsSummary( + value: unknown, +): SnapshotDiagnosticsSummary | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + const stats = readSnapshotTimingStats(record.stats); + if (!stats) return undefined; + const warning = typeof record.warning === 'string' ? record.warning : undefined; + return { stats, ...(warning ? { warning } : {}) }; +} + +function buildSnapshotTimingStats(samples: SnapshotTimingSample[]): SnapshotTimingStats { + const durations = samples.map((sample) => sample.durationMs).sort((a, b) => a - b); + return { + count: durations.length, + p50Ms: percentileNearestRank(durations, 50), + p95Ms: percentileNearestRank(durations, 95), + maxMs: durations[durations.length - 1] ?? 0, + slowThresholdMs: SLOW_SNAPSHOT_P95_WARNING_MS, + ...singlePlatform(samples), + ...backendCounts(samples), + }; +} + +function percentileNearestRank(values: number[], percentile: number): number { + if (values.length === 0) return 0; + const index = Math.max(0, Math.ceil((percentile / 100) * values.length) - 1); + return values[Math.min(index, values.length - 1)] ?? 0; +} + +function singlePlatform(samples: SnapshotTimingSample[]): Pick { + const platforms = samples + .map((sample) => sample.platform) + .filter((platform): platform is Platform => Boolean(platform)); + const uniquePlatforms = new Set(platforms); + return uniquePlatforms.size === 1 ? { platform: platforms[0] } : {}; +} + +function backendCounts(samples: SnapshotTimingSample[]): Pick { + const backends: Record = {}; + for (const sample of samples) { + if (!sample.backend) continue; + backends[sample.backend] = (backends[sample.backend] ?? 0) + 1; + } + return Object.keys(backends).length > 0 ? { backends } : {}; +} + +function formatSlowSnapshotWarning(stats: SnapshotTimingStats): string { + const platform = stats.platform ? `${stats.platform} ` : ''; + return `Warning: ${platform}snapshots are slow in this run: p95 ${stats.p95Ms}ms over ${stats.count} captures. Possible causes: device load, app or dev server stuck, helper fallback, or stale daemon.`; +} + +function readSnapshotTimingStats(value: unknown): SnapshotTimingStats | undefined { + const record = readRecord(value); + if (!record) return undefined; + const required = readRequiredSnapshotTimingStats(record); + if (!required) return undefined; + return { + ...required, + ...readOptionalSnapshotTimingStats(record), + }; +} + +function readRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function readRequiredSnapshotTimingStats( + record: Record, +): + | Pick + | undefined { + const entries = { + count: record.count, + p50Ms: record.p50Ms, + p95Ms: record.p95Ms, + maxMs: record.maxMs, + slowThresholdMs: record.slowThresholdMs, + }; + if (Object.values(entries).some((value) => typeof value !== 'number')) return undefined; + return entries as Pick< + SnapshotTimingStats, + 'count' | 'p50Ms' | 'p95Ms' | 'maxMs' | 'slowThresholdMs' + >; +} + +function readBackendCounts(value: Record): Record | undefined { + const entries = Object.entries(value).filter((entry): entry is [string, number] => { + return typeof entry[1] === 'number'; + }); + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +function readOptionalSnapshotTimingStats( + record: Record, +): Pick { + const platform = typeof record.platform === 'string' ? record.platform : undefined; + const backendRecord = readRecord(record.backends); + const backends = backendRecord ? readBackendCounts(backendRecord) : undefined; + return { + ...(platform ? { platform: platform as SnapshotTimingStats['platform'] } : {}), + ...(backends ? { backends } : {}), + }; +} + +function samplesFromStats(stats: SnapshotTimingStats | undefined): SnapshotTimingSample[] { + if (!stats || stats.count <= 0) return []; + const platform = stats.platform; + if (stats.count === 1) return [{ durationMs: stats.maxMs, platform }]; + if (stats.count === 2) { + return [ + { durationMs: stats.p50Ms, platform }, + { durationMs: stats.maxMs, platform }, + ]; + } + return [ + ...Array.from({ length: stats.count - 3 }, () => ({ durationMs: stats.p50Ms, platform })), + { durationMs: stats.p50Ms, platform }, + { durationMs: stats.p95Ms, platform }, + { durationMs: stats.maxMs, platform }, + ]; +}