diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 7460bf48f..bf2c354de 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -49,10 +49,9 @@ extension RunnerTests { return nil } - // The public windows query backing safeSnapshotViewport can fail on the same apps that - // need this fallback, degrading to an infinite viewport that marks off-screen content - // (e.g. closed drawer menus at negative x) as visible and tappable. The private root's - // own frame is the reliable screen rect here. + // If the app frame is unavailable, the private root's own frame is the reliable screen + // rect here. Avoid public window queries: stale transient windows can record XCTest + // failures after the runner already returned a successful command response. var viewport = safeSnapshotViewport(app: app) let rootFrame = privateAXRect(root["frame"]) if viewport.isInfinite || viewport.isNull || viewport.isEmpty, !rootFrame.isEmpty { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 24d7347b9..61c1a7cea 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -13,7 +13,7 @@ extension RunnerTests { return (gestureStartUptimeMs, currentUptimeMs()) } - private func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval { + func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval { min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120) } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 15050c2ed..2341e3d07 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -157,8 +157,8 @@ struct SequenceStep: Codable { let y2: Double? let durationMs: Double? let pauseMs: Double? - /// For `tap` steps on iOS non-tv: use the synthesized HID tap fast path (synthesizedTapAt) - /// instead of the drag-based XCUICoordinate tapAt, matching the individual `tap` command. + /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the + /// drag-based XCUICoordinate path, matching the individual command behavior. let synthesized: Bool? } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 9192b9e59..9796e2e87 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -164,6 +164,50 @@ extension RunnerTests { } // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } + if step.kind == "drag", step.synthesized == true { + let dragPoints = keyboardAvoidingDragPoints( + app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) + let durationMs = min(max(step.durationMs ?? 250, 16), 10000) + let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + synthesizedDragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + durationMs: durationMs + ) + } + if case .performed = outcome { + if let pauseMs = step.pauseMs, pauseMs > 0 { + sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) + } + return SequenceStepOutcome( + outcome: outcome, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs + ) + } + let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250) + let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) { + dragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + holdDuration: fallbackHoldDuration + ) + } + if case .performed = fallbackOutcome, let pauseMs = step.pauseMs, pauseMs > 0 { + sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) + } + return SequenceStepOutcome( + outcome: fallbackOutcome, + gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs, + gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs + ) + } let (timing, outcome) = performGesture(activeApp) { switch step.kind { case "doubleTap": @@ -173,9 +217,9 @@ extension RunnerTests { let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0 return longPressAt(app: activeApp, x: x, y: y, duration: duration) case "drag": - // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command - // (RunnerTests+CommandExecution.swift). durationMs is intentionally ignored on this - // coordinate-drag path, matching that command's non-synthesized branch. + // Route through keyboardAvoidingDragPoints for parity with the individual `.drag` command. + // The non-synthesized coordinate-drag path ignores durationMs, matching that command's + // non-synthesized branch. let dragPoints = keyboardAvoidingDragPoints( app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) return dragAt( diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 63221ca52..13e305ea9 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -739,15 +739,6 @@ extension RunnerTests { } private func snapshotViewport(app: XCUIApplication) -> CGRect { - let windows = app.windows.allElementsBoundByIndex - let windowFrames = windows - .filter { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty } - .map(\.frame) - if let largestWindowFrame = windowFrames.max(by: { left, right in - left.width * left.height < right.width * right.height - }) { - return largestWindowFrame - } let appFrame = app.frame if !appFrame.isNull && !appFrame.isEmpty { return appFrame diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts index 6e84248e2..91c7800cd 100644 --- a/src/core/__tests__/dispatch-interactions.test.ts +++ b/src/core/__tests__/dispatch-interactions.test.ts @@ -197,8 +197,25 @@ test('handleSwipeCommand fuses repeated swipes into sequence drag steps with pin steps: [ // Ping-pong is unrolled daemon-side: odd indices swap endpoints, replacing the // runner-side pattern handling of the retired dragSeries command. - { kind: 'drag', x: 100, y: 650, x2: 100, y2: 450, durationMs: 120, pauseMs: 50 }, - { kind: 'drag', x: 100, y: 450, x2: 100, y2: 650, durationMs: 120 }, + { + kind: 'drag', + x: 100, + y: 650, + x2: 100, + y2: 450, + durationMs: 120, + synthesized: true, + pauseMs: 50, + }, + { + kind: 'drag', + x: 100, + y: 450, + x2: 100, + y2: 650, + durationMs: 120, + synthesized: true, + }, ], appBundleId: 'com.example.App', }); diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 47e76dec4..cd2b9a2df 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -427,10 +427,10 @@ function buildPressSequenceSteps( // Unrolls a swipe series into `sequence` drag steps, replacing the retired `dragSeries` runner // command. Ping-pong becomes per-step endpoint swapping (odd indices reversed), matching the -// runner-side performDragSeries the daemon no longer invokes. durationMs is carried for wire -// fidelity and budget estimation; the runner's coordinate-drag path ignores it, exactly as the -// daemon-sent (non-synthesized) dragSeries did. +// runner-side performDragSeries the daemon no longer invokes. iOS touch targets request the same +// synthesized, duration-aware drag path as one-shot swipe; macOS/tvOS keep coordinate drag. function buildSwipeSequenceSteps(params: { + device: DeviceInfo; x1: number; y1: number; x2: number; @@ -440,7 +440,8 @@ function buildSwipeSequenceSteps(params: { pattern: string; effectiveDurationMs: number; }): RunnerSequenceStep[] { - const { x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs } = params; + const { device, x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs } = params; + const synthesized = device.platform === 'ios' && device.target !== 'tv'; return Array.from({ length: count }, (_, index) => { const reverse = pattern === 'ping-pong' && index % 2 === 1; const isLast = index === count - 1; @@ -451,6 +452,7 @@ function buildSwipeSequenceSteps(params: { x2: reverse ? x1 : x2, y2: reverse ? y1 : y2, durationMs: effectiveDurationMs, + ...(synthesized ? { synthesized: true } : {}), ...(!isLast && pauseMs > 0 ? { pauseMs } : {}), }; }); @@ -608,7 +610,17 @@ async function runSwipeCoordinates(params: { if (shouldUseIosDragSeries(device, count)) { const aggregated = await runIosSequenceChunks( device, - buildSwipeSequenceSteps({ x1, y1, x2, y2, count, pauseMs, pattern, effectiveDurationMs }), + buildSwipeSequenceSteps({ + device, + x1, + y1, + x2, + y2, + count, + pauseMs, + pattern, + effectiveDurationMs, + }), context, ); return { diff --git a/src/platforms/ios/__tests__/index.test.ts b/src/platforms/ios/__tests__/index.test.ts index d749c01ea..a6318c213 100644 --- a/src/platforms/ios/__tests__/index.test.ts +++ b/src/platforms/ios/__tests__/index.test.ts @@ -184,7 +184,7 @@ test('resolveMacOsHelperPackageRootFrom finds helper package from source and dis } }); -test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async () => { +test('iosRunnerOverrides maps iOS fling duration to synthesized drag', async () => { mockRunIosRunnerCommand.mockResolvedValue({}); const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, { @@ -200,6 +200,7 @@ test('iosRunnerOverrides gives fling a short default XCUITest drag hold', async x2: 180, y2: 200, durationMs: 16, + synthesized: true, appBundleId: 'com.example.App', }); }); @@ -252,7 +253,7 @@ for (const [name, device] of [ }); } -test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => { +test('iosRunnerOverrides maps iOS swipe and pan durations to synthesized drag', async () => { mockRunIosRunnerCommand.mockResolvedValue({}); const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, { @@ -270,6 +271,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => { x2: 180, y2: 200, durationMs: 300, + synthesized: true, appBundleId: 'com.example.App', }); assert.deepEqual(mockRunIosRunnerCommand.mock.calls[1]?.[1], { @@ -279,6 +281,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => { x2: 180, y2: 200, durationMs: 250, + synthesized: true, appBundleId: 'com.example.App', }); assert.deepEqual(mockRunIosRunnerCommand.mock.calls[2]?.[1], { @@ -288,6 +291,7 @@ test('iosRunnerOverrides maps swipe to XCTest iOS drag duration', async () => { x2: 180, y2: 200, durationMs: 300, + synthesized: true, appBundleId: 'com.example.App', }); }); @@ -296,7 +300,7 @@ for (const [name, device] of [ ['macOS', MACOS_TEST_DEVICE], ['tvOS', TVOS_TEST_SIMULATOR], ] as const) { - test(`iosRunnerOverrides keeps ${name} swipes on the standard drag path`, async () => { + test(`iosRunnerOverrides keeps ${name} drag gestures on the standard path`, async () => { mockRunIosRunnerCommand.mockResolvedValue({}); const { overrides } = iosRunnerOverrides(device, { @@ -304,6 +308,8 @@ for (const [name, device] of [ }); await overrides.swipe(100, 200, 180, 200, 300); + await overrides.pan(100, 200, 180, 200, 300); + await overrides.fling(100, 200, 180, 200, 300); assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], { command: 'drag', @@ -314,6 +320,24 @@ for (const [name, device] of [ durationMs: 300, appBundleId: 'com.example.App', }); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[1]?.[1], { + command: 'drag', + x: 100, + y: 200, + x2: 180, + y2: 200, + durationMs: 300, + appBundleId: 'com.example.App', + }); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[2]?.[1], { + command: 'drag', + x: 100, + y: 200, + x2: 180, + y2: 200, + durationMs: 300, + appBundleId: 'com.example.App', + }); }); } diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index 5324c26da..3565da6a8 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -29,6 +29,12 @@ type NormalizedScrollOptions = { preferProvidedPixels?: boolean; }; +type IosDragCommandOptions = { + defaultDurationMs: number; + legacyDefaultDurationMs?: number; + synthesized?: boolean; +}; + type IosRunnerOverrides = Pick< Interactor, | 'tap' @@ -101,6 +107,7 @@ export function iosRunnerOverrides( device, iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, { defaultDurationMs: IOS_SWIPE_DEFAULT_DURATION_MS, + synthesized: shouldUseSynthesizedIosGesture(device), }), runnerOpts, ); @@ -111,6 +118,7 @@ export function iosRunnerOverrides( iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, { defaultDurationMs: 500, legacyDefaultDurationMs: 500, + synthesized: shouldUseSynthesizedIosGesture(device), }), runnerOpts, ); @@ -118,15 +126,11 @@ export function iosRunnerOverrides( fling: async (x1, y1, x2, y2, durationMs) => { return await runIosRunnerCommand( device, - { - command: 'drag', - x: x1, - y: y1, - x2, - y2, - durationMs: durationMs ?? 16, - appBundleId: ctx.appBundleId, - }, + iosDragCommand(device, ctx, x1, y1, x2, y2, durationMs, { + defaultDurationMs: 16, + legacyDefaultDurationMs: 16, + synthesized: shouldUseSynthesizedIosGesture(device), + }), runnerOpts, ); }, @@ -252,11 +256,15 @@ function iosTapCommand( command: 'tap', x, y, - ...(device.platform === 'ios' && device.target !== 'tv' ? { synthesized: true } : {}), + ...(shouldUseSynthesizedIosGesture(device) ? { synthesized: true } : {}), appBundleId: ctx.appBundleId, }; } +function shouldUseSynthesizedIosGesture(device: DeviceInfo): boolean { + return device.platform === 'ios' && device.target !== 'tv'; +} + function iosDragCommand( device: DeviceInfo, ctx: RunnerContext, @@ -265,10 +273,7 @@ function iosDragCommand( x2: number, y2: number, durationMs: number | undefined, - options: { - defaultDurationMs: number; - legacyDefaultDurationMs?: number; - }, + options: IosDragCommandOptions, ): RunnerCommand { const normalizedDurationMs = device.platform === 'ios' && device.target !== 'tv' @@ -281,6 +286,7 @@ function iosDragCommand( x2, y2, ...(normalizedDurationMs !== undefined ? { durationMs: normalizedDurationMs } : {}), + ...(options.synthesized === true ? { synthesized: true } : {}), appBundleId: ctx.appBundleId, }; } diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index 7c859f7f3..ae1f95d08 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -76,6 +76,7 @@ export async function createIosSettingsWorld(): Promise { x2: 276, y2: 122, durationMs: 500, + synthesized: true, appBundleId: 'com.apple.Preferences', }, result: { dragged: true }, @@ -91,6 +92,7 @@ export async function createIosSettingsWorld(): Promise { x2: 376, y2: 122, durationMs: 50, + synthesized: true, appBundleId: 'com.apple.Preferences', }, result: { flung: true },