import { Mobilecli } from "./mobilecli"; import { ActionableError, Button, FoldPosture, InstalledApp, Orientation, Robot, ScreenElement, ScreenSize, ScreenshotOptions, SwipeDirection } from "./robot"; interface InstalledAppsResponse { status: "ok", data: Array<{ packageName: string; appName?: string; // ios version?: string; // ios }>; } interface DeviceInfoResponse { status: "ok", data: { device: { id: string; name: string; platform: string; type: string; version: string; state: string; screenSize?: { width: number; height: number; scale: number; }; }; }; } interface UIElementResponse { ref?: string; type: string; label?: string; text?: string; name?: string; value?: string; identifier?: string; rect: { x: number; y: number; width: number; height: number; }; focused?: boolean; selected?: boolean; checked?: boolean; enabled?: boolean; children?: UIElementResponse[]; } interface DumpUIResponse { status: "ok", // mobilecli answers "dump ui" with status "ok" but an empty payload when the // device screen is off or locked, so neither level can be relied on. data?: { elements?: UIElementResponse[]; }; } interface ForegroundAppResponse { status: "ok", data: { packageName: string; appName?: string; }; } interface OrientationResponse { status: "ok", data: { orientation: Orientation; }; } interface ClipboardResponse { data: { text: string; }; } const flattenUIElement = (element: UIElementResponse): ScreenElement[] => { const screenElement: ScreenElement = { type: element.type, label: element.label, text: element.text, name: element.name, value: element.value, identifier: element.identifier, rect: element.rect, ref: element.ref, focused: element.focused, selected: element.selected, checked: element.checked, enabled: element.enabled, }; if (!element.children) { return [screenElement]; } return [screenElement, ...element.children.flatMap(child => flattenUIElement(child))]; }; export class MobileDevice implements Robot { private mobilecli: Mobilecli; public constructor(private deviceId: string) { this.mobilecli = new Mobilecli(); } private runCommand(args: string[]): string { const fullArgs = [...args, "--device", this.deviceId]; return this.mobilecli.executeCommand(fullArgs); } private runJsonCommand(args: string[]): T { const output = this.runCommand(args); try { return JSON.parse(output) as T; } catch { throw new ActionableError(`Failed to parse JSON response from mobilecli ${args.join(" ")}`); } } public async getScreenSize(): Promise { const response = this.runJsonCommand(["device", "info"]); if (response.data.device.screenSize) { return response.data.device.screenSize; } return { width: 0, height: 0, scale: 1.0 }; } public async swipe(direction: SwipeDirection): Promise { const screenSize = await this.getScreenSize(); const centerX = Math.floor(screenSize.width / 2); const centerY = Math.floor(screenSize.height / 2); const distance = 400; // Default distance in pixels let startX = centerX; let startY = centerY; let endX = centerX; let endY = centerY; switch (direction) { case "up": startY = centerY + distance / 2; endY = centerY - distance / 2; break; case "down": startY = centerY - distance / 2; endY = centerY + distance / 2; break; case "left": startX = centerX + distance / 2; endX = centerX - distance / 2; break; case "right": startX = centerX - distance / 2; endX = centerX + distance / 2; break; } this.runCommand(["io", "swipe", `${startX},${startY},${endX},${endY}`]); } public async swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance?: number): Promise { const swipeDistance = distance || 400; let endX = x; let endY = y; switch (direction) { case "up": endY = y - swipeDistance; break; case "down": endY = y + swipeDistance; break; case "left": endX = x - swipeDistance; break; case "right": endX = x + swipeDistance; break; } this.runCommand(["io", "swipe", `${Math.round(x)},${Math.round(y)},${Math.round(endX)},${Math.round(endY)}`]); } public async getScreenshot(options?: ScreenshotOptions): Promise { const format = options?.format || "png"; const fullArgs = ["screenshot", "--device", this.deviceId, "--format", format, "--output", "-"]; if (format === "jpeg" && options?.quality !== undefined) { fullArgs.push("--quality", `${options.quality}`); } if (options?.maxSize !== undefined) { fullArgs.push("--max-size", `${options.maxSize}`); } if (options?.scale !== undefined && options.scale !== 1) { fullArgs.push("--scale", `${options.scale}`); } return this.mobilecli.executeCommandBuffer(fullArgs); } public async listApps(): Promise { const response = this.runJsonCommand(["apps", "list"]); return response.data.map(app => ({ appName: app.appName || app.packageName, packageName: app.packageName, })) as InstalledApp[]; } public async getForegroundApp(): Promise { const response = this.runJsonCommand(["apps", "foreground"]); return { appName: response.data.appName || response.data.packageName, packageName: response.data.packageName, }; } public async launchApp(packageName: string, locale?: string): Promise { const args = ["apps", "launch", packageName]; if (locale) { args.push("--locale", locale); } this.runCommand(args); } public async terminateApp(packageName: string): Promise { this.runCommand(["apps", "terminate", packageName]); } public async installApp(path: string): Promise { this.runCommand(["apps", "install", path]); } public async uninstallApp(bundleId: string): Promise { this.runCommand(["apps", "uninstall", bundleId]); } public async openUrl(url: string): Promise { this.runCommand(["url", url]); } public async sendKeys(text: string): Promise { this.runCommand(["io", "text", text]); } public async pressButton(button: Button): Promise { this.runCommand(["io", "button", button]); } public async tap(x: number, y: number): Promise { // mobilecli rejects fractional coordinates ("x and y must be integers") this.runCommand(["io", "tap", `${Math.round(x)},${Math.round(y)}`]); } public async tapByRef(ref: string): Promise { this.runCommand(["io", "tap", ref]); } public async doubleTap(x: number, y: number): Promise { // TODO: should move into mobilecli itself as "io doubletap" await this.tap(x, y); await this.tap(x, y); } public async doubleTapByRef(ref: string): Promise { // resolve the ref once: two "io tap @ref" calls would each dump the ui again and miss the double-tap window const elements = await this.getElementsOnScreen(); const element = elements.find(candidate => candidate.ref === ref); if (!element) { throw new ActionableError(`Element ${ref} was not found on the current screen. Call mobile_list_elements_on_screen again to get fresh refs`); } const { x, y, width, height } = element.rect; await this.doubleTap(x + width / 2, y + height / 2); } public async longPress(x: number, y: number, duration: number): Promise { this.runCommand(["io", "longpress", `${Math.round(x)},${Math.round(y)}`, "--duration", `${duration}`]); } public async longPressByRef(ref: string, duration: number): Promise { this.runCommand(["io", "longpress", ref, "--duration", `${duration}`]); } public async getElementsOnScreen(): Promise { const response = this.runJsonCommand(["dump", "ui"]); const elements = response.data?.elements; if (!elements) { throw new ActionableError("No UI hierarchy was returned for this device - its screen is most likely off or locked. Wake it up (for example with mobile_press_button using HOME, or POWER when that is supported) and try again"); } return elements.flatMap(element => flattenUIElement(element)); } public async setOrientation(orientation: Orientation): Promise { this.runCommand(["device", "orientation", "set", orientation]); } public async getOrientation(): Promise { const response = this.runJsonCommand(["device", "orientation", "get"]); return response.data.orientation; } public async fold(posture: FoldPosture | number): Promise { // parse the response, older mobilecli versions print help and exit 0 on unknown commands this.runJsonCommand(["device", "fold", `${posture}`]); } public async setLocation(latitude: number, longitude: number): Promise { this.runCommand(["device", "location", "set", `${latitude},${longitude}`]); } public async clearLocation(): Promise { this.runCommand(["device", "location", "clear"]); } public async getClipboard(): Promise { const response = this.runJsonCommand(["io", "clipboard", "get"]); return response.data.text; } public async setClipboard(text: string): Promise { this.runCommand(["io", "clipboard", "set", text]); } public getLogs(limit: number, filters: string[], timeoutMs: number): Promise { const args = ["device", "logs", "--limit", String(limit), ...filters.flatMap(filter => ["--filter", filter]), "--device", this.deviceId]; const child = this.mobilecli.spawnCommand(args, true); return new Promise((resolve, reject) => { const stdout: Buffer[] = []; const stderr: Buffer[] = []; let silenceTimer: NodeJS.Timeout; // kill the stream once no log entry arrived for timeoutMs, then return what was captured const restartSilenceTimer = () => { clearTimeout(silenceTimer); silenceTimer = setTimeout(() => child.kill(), timeoutMs); }; child.stdout?.on("data", (chunk: Buffer) => { stdout.push(chunk); restartSilenceTimer(); }); child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk)); child.on("error", err => { clearTimeout(silenceTimer); reject(err); }); child.on("close", code => { clearTimeout(silenceTimer); if (code !== 0 && !child.killed) { reject(new Error(`mobilecli device logs failed with code ${code}: ${Buffer.concat(stderr).toString().trim()}`)); return; } resolve(Buffer.concat(stdout).toString().trim()); }); restartSilenceTimer(); }); } }