diff --git a/.github/workflows/ios_e2e.sh b/.github/workflows/ios_e2e.sh index a2a16550..c3627d3d 100755 --- a/.github/workflows/ios_e2e.sh +++ b/.github/workflows/ios_e2e.sh @@ -22,4 +22,4 @@ xcrun simctl install $UDID ./.github/workflows/fakeStore.app xcrun simctl launch $UDID $APPID mkdir -p report -npx flashlight-ios-poc ios-test --appId $APPID --simulatorId $UDID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json' \ No newline at end of file +PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId $APPID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json' --iterationCount 2 diff --git a/packages/commands/test/src/PerformanceMeasurer.ts b/packages/commands/test/src/PerformanceMeasurer.ts index 23be4c72..31770512 100644 --- a/packages/commands/test/src/PerformanceMeasurer.ts +++ b/packages/commands/test/src/PerformanceMeasurer.ts @@ -19,9 +19,12 @@ export class PerformanceMeasurer { // noop by default } ) { + // Hack to make sure the profiler is ready to receive measures + await profiler.waitUntilReady(this.bundleId); this.polling = profiler.pollPerformanceMeasures(this.bundleId, { onMeasure: (measure) => { - if (this.shouldStop) { + // The ios-instruments profiler yields measures at the end of the test when the polling is already stopped + if (this.shouldStop && process.env.PLATFORM !== "ios-instruments") { this.polling?.stop(); } @@ -60,6 +63,8 @@ export class PerformanceMeasurer { // Ensure polling has stopped this.polling?.stop(); + // Hack for ios-instruments to get the measures at the end of the test + await profiler.getMeasures(); return { time: time ?? 0, diff --git a/packages/core/types/index.ts b/packages/core/types/index.ts index 6d4e484a..f4930032 100644 --- a/packages/core/types/index.ts +++ b/packages/core/types/index.ts @@ -85,4 +85,6 @@ export interface Profiler { cleanup: () => void; getScreenRecorder: (videoPath: string) => ScreenRecorder | undefined; stopApp: (bundleId: string) => Promise; + waitUntilReady: (bundleId: string) => Promise; + getMeasures: () => Promise; } diff --git a/packages/platforms/android/src/index.ts b/packages/platforms/android/src/index.ts index 6d4f5127..085cdde1 100644 --- a/packages/platforms/android/src/index.ts +++ b/packages/platforms/android/src/index.ts @@ -12,6 +12,12 @@ export { waitFor } from "./utils/waitFor"; export { executeAsync, executeCommand } from "./commands/shell"; export class AndroidProfiler implements Profiler { + waitUntilReady = (bundleId: string) => { + return new Promise((resolve) => resolve()); + }; + getMeasures = () => { + return new Promise((resolve) => resolve()); + }; pollPerformanceMeasures = pollPerformanceMeasures; detectCurrentBundleId = profiler.detectCurrentBundleId; installProfilerOnDevice = ensureCppProfilerIsInstalled; diff --git a/packages/platforms/ios-instruments/README.md b/packages/platforms/ios-instruments/README.md index b0d531bd..42e169c2 100644 --- a/packages/platforms/ios-instruments/README.md +++ b/packages/platforms/ios-instruments/README.md @@ -10,7 +10,7 @@ Requirements: - Get a running simulator id with `xcrun simctl list devices` - Create template Flashlight in Xcode Instruments (with cpu-profile and memory usage) - Add your own test in `test.yaml` -- `flashlight-ios-poc ios-test --appId --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` +- `PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"` - Check the results in the web-reporter `yarn workspace @perf-profiler/web-reporter build` @@ -19,5 +19,5 @@ Requirements: ## Next steps - run several iterations -- add more metrics (RAM, FPS, CPU per thread) +- add more metrics (RAM, FPS) - Unify API with flashlight test diff --git a/packages/platforms/ios-instruments/src/XcodePerfParser.ts b/packages/platforms/ios-instruments/src/XcodePerfParser.ts new file mode 100644 index 00000000..57d44361 --- /dev/null +++ b/packages/platforms/ios-instruments/src/XcodePerfParser.ts @@ -0,0 +1,112 @@ +import { XMLParser } from "fast-xml-parser"; +import { Result, Row, Thread, isRefField } from "./utils/xmlTypes"; +import fs from "fs"; +import { CpuMeasure, Measure } from "@perf-profiler/types"; + +const FAKE_RAM = 200; +const FAKE_FPS = 60; +const TIME_INTERVAL = 500; +const NANOSEC_TO_MILLISEC = 1_000_000; +const CPU_TIME_INTERVAL = 10; + +const initThreadMap = (row: Row[]): { [id: number]: string } => { + const threadRef: { [id: number]: Thread } = {}; + row.forEach((row: Row) => { + if (!isRefField(row.thread)) { + threadRef[row.thread.id] = row.thread; + } + }); + return Object.values(threadRef).reduce((acc: { [id: number]: string }, thread) => { + const currentThreadName = thread.fmt + .split(" ") + .slice(0, thread.fmt.split(" ").indexOf("")) + .join(" "); + const currentTid = thread.tid.value; + const numberOfThread = Object.values(threadRef).filter((thread: Thread) => { + return thread.fmt.includes(currentThreadName) && thread.tid.value < currentTid; + }).length; + acc[thread.id] = + numberOfThread > 0 ? `${currentThreadName} (${numberOfThread})` : currentThreadName; + return acc; + }, {}); +}; + +const getMeasures = (row: Row[]): Map> => { + const sampleTimeRef: { [id: number]: number } = {}; + const threadRef: { [id: number]: string } = initThreadMap(row); + const classifiedMeasures = row.reduce((acc: Map>, row: Row) => { + const sampleTime = isRefField(row.sampleTime) + ? sampleTimeRef[row.sampleTime.ref] + : row.sampleTime.value / NANOSEC_TO_MILLISEC; + if (!isRefField(row.sampleTime)) { + sampleTimeRef[row.sampleTime.id] = sampleTime; + } + + const threadName = isRefField(row.thread) + ? threadRef[row.thread.ref] + : threadRef[row.thread.id]; + + const correspondingTimeInterval = + parseInt((sampleTime / TIME_INTERVAL).toFixed(0), 10) * TIME_INTERVAL; + + const timeIntervalMap = acc.get(correspondingTimeInterval) ?? new Map(); + + const numberOfPointsIn = timeIntervalMap.get(threadName) ?? 0; + + timeIntervalMap.set(threadName, numberOfPointsIn + 1); + + acc.set(correspondingTimeInterval, timeIntervalMap); + + return acc; + }, new Map>()); + return classifiedMeasures; +}; + +export const computeMeasures = (inputFileName: string) => { + const xml = fs.readFileSync(inputFileName, "utf8"); + const options = { + attributeNamePrefix: "", + ignoreAttributes: false, + parseAttributeValue: true, + textNodeName: "value", + updateTag(tagName: string, jPath: string, attrs: { [x: string]: string | number }) { + switch (tagName) { + case "trace-query-result": { + return "result"; + } + case "sample-time": { + return "sampleTime"; + } + default: { + return tagName; + } + } + }, + }; + const parser = new XMLParser(options); + const jsonObject: Result = parser.parse(xml); + if (!jsonObject.result.node.row) { + throw new Error("No rows in the xml file"); + } + const measures: Map> = getMeasures(jsonObject.result.node.row); + const formattedMeasures: Measure[] = Array.from(measures.entries()).map( + (classifiedMeasures: [number, Map]) => { + const timeInterval = classifiedMeasures[0]; + const timeIntervalMap = classifiedMeasures[1]; + const cpuMeasure: CpuMeasure = { + perName: {}, + perCore: {}, + }; + timeIntervalMap.forEach((value: number, key: string) => { + cpuMeasure.perName[key] = (value * 10) / (TIME_INTERVAL / CPU_TIME_INTERVAL); + }); + return { + cpu: cpuMeasure, + ram: FAKE_RAM, + fps: FAKE_FPS, + time: timeInterval, + }; + } + ); + return formattedMeasures; +}; diff --git a/packages/platforms/ios-instruments/src/index.ts b/packages/platforms/ios-instruments/src/index.ts index 92b13684..ac04babf 100644 --- a/packages/platforms/ios-instruments/src/index.ts +++ b/packages/platforms/ios-instruments/src/index.ts @@ -1 +1,120 @@ +import { Measure, Profiler, ProfilerPollingOptions, ScreenRecorder } from "@perf-profiler/types"; +import { ChildProcess } from "child_process"; +// TODO: refactor so that these functions are not in android +// eslint-disable-next-line import/no-extraneous-dependencies +import { executeAsync, executeCommand } from "@perf-profiler/android"; +import { IdbDevice, getConnectedDevice, killApp, launchApp } from "./utils/DeviceManager"; +import { computeMeasures } from "./XcodePerfParser"; +import { getTmpFilePath, removeTmpFiles } from "./utils/tmpFileManager"; export { killApp } from "./utils/DeviceManager"; + +const startRecord = async ( + deviceUdid: string, + appPid: number, + traceFile: string +): Promise => { + const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`; + const recordingProcess = executeAsync( + `xcrun xctrace record --device ${deviceUdid} --template ${templateFilePath} --attach ${appPid} --output ${traceFile}` + ); + await new Promise((resolve) => { + recordingProcess.stdout?.on("data", (data) => { + if (data.toString().includes("Ctrl-C to stop")) { + resolve(); + } + }); + }); + return recordingProcess; +}; + +const saveTraceFile = (traceFile: string): string => { + const xmlOutputFile = getTmpFilePath("report.xml"); + executeCommand( + `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` + ); + return xmlOutputFile; +}; + +const stopPerfRecord = async ( + recordingProcess: ChildProcess, + traceFile: string, + onMeasure: (measure: Measure) => void +) => { + try { + await new Promise((resolve) => { + recordingProcess.stdout?.on("data", (data) => { + if (data.toString().includes("Output file saved as")) { + resolve(); + } + }); + }); + } catch (e) { + console.log("Error while recording: ", e); + } + const xmlFile = saveTraceFile(traceFile); + const measures = computeMeasures(xmlFile); + measures.forEach((measure) => { + onMeasure(measure); + }); + removeTmpFiles(); +}; + +export class IOSInstrumentsProfiler implements Profiler { + connectedDevice: IdbDevice | undefined; + recordingProcess: ChildProcess | undefined; + traceFile: string | undefined; + pid: number | undefined; + bundleId: string | undefined; + onMeasure: ((measure: Measure) => void) | undefined; + pollPerformanceMeasures(bundleId: string, options: ProfilerPollingOptions): { stop: () => void } { + if (!this.pid) throw new Error("Profiler is not ready, app is not running"); + this.onMeasure = options.onMeasure; + return { + stop: () => { + return; + }, + }; + } + + detectCurrentBundleId(): string { + throw new Error("App Id detection is not implemented on iOS with Instruments"); + } + + installProfilerOnDevice() { + // Do we need anything here? + } + + getScreenRecorder(videoPath: string): ScreenRecorder | undefined { + return undefined; + } + + cleanup: () => void = () => { + // Do we need anything here? + }; + + async waitUntilReady(bundleId: string): Promise { + this.connectedDevice = getConnectedDevice(); + if (!this.connectedDevice) { + throw new Error("No device connected"); + } + this.bundleId = bundleId; + this.pid = launchApp(bundleId); + const traceFile = `report_${new Date().getTime()}.trace`; + this.traceFile = traceFile; + this.recordingProcess = await startRecord(this.connectedDevice.udid, this.pid, traceFile); + } + + async getMeasures(): Promise { + if (!this.recordingProcess || !this.traceFile || !this.pid || !this.onMeasure || !this.bundleId) + throw new Error("Profiler is not ready to get measures"); + const recordingProcess = this.recordingProcess; + const traceFile = this.traceFile; + killApp(this.bundleId); + await stopPerfRecord(recordingProcess, traceFile, this.onMeasure); + } + + async stopApp(bundleId: string): Promise { + killApp(bundleId); + return new Promise((resolve) => resolve()); + } +} diff --git a/packages/platforms/ios-instruments/src/launchIOS.ts b/packages/platforms/ios-instruments/src/launchIOS.ts deleted file mode 100644 index 8607a4bf..00000000 --- a/packages/platforms/ios-instruments/src/launchIOS.ts +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env node - -// TODO: refactor so that these functions are not in android -// eslint-disable-next-line import/no-extraneous-dependencies -import { executeAsync, executeCommand } from "@perf-profiler/android/dist/src/commands/shell"; -import fs from "fs"; -import { writeReport } from "./writeReport"; -import { program } from "commander"; -import { execSync, ChildProcess } from "child_process"; -import os from "os"; - -const tmpFiles: string[] = []; -const removeTmpFiles = () => { - for (const tmpFile of tmpFiles) { - fs.rmSync(tmpFile, { recursive: true }); - } -}; - -const getTmpFilePath = (fileName: string) => { - const filePath = `${os.tmpdir()}/${fileName}`; - tmpFiles.push(filePath); - - return filePath; -}; - -const writeTmpFile = (fileName: string, content: string): string => { - const tmpPath = getTmpFilePath(fileName); - fs.writeFileSync(tmpPath, content); - return tmpPath; -}; - -const startRecord = (simulatorId: string, traceFile: string): ChildProcess => { - const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`; - return executeAsync( - `xcrun xctrace record --device ${simulatorId} --template ${templateFilePath} --attach fakeStore --output ${traceFile}` - ); -}; - -const save = (traceFile: string, resultsFilePath: string) => { - const xmlOutputFile = getTmpFilePath("report.xml"); - executeCommand( - `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}` - ); - writeReport(xmlOutputFile, resultsFilePath); -}; - -const launchTest = async ({ - testCommand, - appId, - simulatorId, - resultsFilePath, -}: { - testCommand: string; - appId: string; - simulatorId: string; - resultsFilePath: string; -}) => { - const traceFile = `report_${new Date().getTime()}.trace`; - const lauchAppFile = writeTmpFile( - "./launch.yaml", - `appId: ${appId} ---- -- launchApp -` - ); - execSync(`maestro test ${lauchAppFile} --no-ansi`, { - stdio: "inherit", - }); - const recordingProcess = startRecord(simulatorId, traceFile); - await new Promise((resolve) => { - recordingProcess.stdout?.on("data", (data) => { - if (data.toString().includes("Ctrl-C to stop")) { - resolve(); - } - }); - }); - execSync(`${testCommand} --no-ansi`, { - stdio: "inherit", - }); - const stopAppFile = writeTmpFile( - "./stop.yaml", - `appId: ${appId} ---- -- stopApp -` - ); - execSync(`maestro test ${stopAppFile} --no-ansi`, { - stdio: "inherit", - }); - try { - await new Promise((resolve) => { - recordingProcess.stdout?.on("data", (data) => { - console.log(data.toString()); - if (data.toString().includes("Output file saved as")) { - resolve(); - } - }); - }); - } catch (e) { - console.log("Error while recording: ", e); - } - save(traceFile, resultsFilePath); - - removeTmpFiles(); -}; - -program - .command("ios-test") - .requiredOption("--appId ", "App ID (e.g. com.monapp)") - .requiredOption( - "--simulatorId ", - "Simulator ID (e.g. 12345678-1234-1234-1234-123456789012)" - ) - .requiredOption( - "--testCommand ", - "Test command (e.g. `maestro test flow.yml`). App performance during execution of this script will be measured over several iterations." - ) - .requiredOption( - "--resultsFilePath ", - "Path where the JSON of results will be written" - ) - .summary("Generate web report from performance measures for iOS.") - .description( - `Generate web report from performance measures. - -Examples: -flashlight ios-test --appId com.monapp --simulatorId 12345678-1234-1234-1234-123456789012 --testCommand "maestro test flow.yml" --resultsFilePath report.json -` - ) - .action((options) => { - launchTest(options); - }); - -program.parse(); diff --git a/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts b/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts index 3ab96330..4693ee5e 100644 --- a/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts +++ b/packages/platforms/ios-instruments/src/utils/tmpFileManager.ts @@ -11,7 +11,9 @@ export const writeTmpFile = (fileName: string, content: string): string => { export const getTmpFilePath = (fileName: string) => { const filePath = `${os.tmpdir()}/${fileName}`; - tmpFiles.push(filePath); + if (!tmpFiles.includes(filePath)) { + tmpFiles.push(filePath); + } return filePath; }; diff --git a/packages/platforms/ios-instruments/src/writeReport.ts b/packages/platforms/ios-instruments/src/writeReport.ts deleted file mode 100644 index b8fddbee..00000000 --- a/packages/platforms/ios-instruments/src/writeReport.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { XMLParser } from "fast-xml-parser"; -import fs from "fs"; -import { CpuMeasure, Measure, TestCaseIterationResult, TestCaseResult } from "@perf-profiler/types"; -import { Result, Row, Thread, isRefField } from "./utils/xmlTypes"; - -export const writeReport = (inputFileName: string, outputFileName: string) => { - const xml = fs.readFileSync(inputFileName, "utf8"); - const iterations: TestCaseIterationResult[] = []; - const FAKE_RAM = 200; - const FAKE_FPS = 60; - const TIME_INTERVAL = 500; - const NANOSEC_TO_MILLISEC = 1_000_000; - const CPU_TIME_INTERVAL = 10; - - const initThreadMap = (row: Row[]): { [id: number]: string } => { - const threadRef: { [id: number]: Thread } = {}; - row.forEach((row: Row) => { - if (!isRefField(row.thread)) { - threadRef[row.thread.id] = row.thread; - } - }); - return Object.values(threadRef).reduce((acc: { [id: number]: string }, thread) => { - const currentThreadName = thread.fmt - .split(" ") - .slice(0, thread.fmt.split(" ").indexOf("")) - .join(" "); - const currentTid = thread.tid.value; - const numberOfThread = Object.values(threadRef).filter((thread: Thread) => { - return thread.fmt.includes(currentThreadName) && thread.tid.value < currentTid; - }).length; - acc[thread.id] = - numberOfThread > 0 ? `${currentThreadName} (${numberOfThread})` : currentThreadName; - return acc; - }, {}); - }; - - const getMeasures = (row: Row[]): Map> => { - const sampleTimeRef: { [id: number]: number } = {}; - const threadRef: { [id: number]: string } = initThreadMap(row); - const classifiedMeasures = row.reduce((acc: Map>, row: Row) => { - const sampleTime = isRefField(row.sampleTime) - ? sampleTimeRef[row.sampleTime.ref] - : row.sampleTime.value / NANOSEC_TO_MILLISEC; - if (!isRefField(row.sampleTime)) { - sampleTimeRef[row.sampleTime.id] = sampleTime; - } - - const threadName = isRefField(row.thread) - ? threadRef[row.thread.ref] - : threadRef[row.thread.id]; - - const correspondingTimeInterval = - parseInt((sampleTime / TIME_INTERVAL).toFixed(0), 10) * TIME_INTERVAL; - - const timeIntervalMap = acc.get(correspondingTimeInterval) ?? new Map(); - - const numberOfPointsIn = timeIntervalMap.get(threadName) ?? 0; - - timeIntervalMap.set(threadName, numberOfPointsIn + 1); - - acc.set(correspondingTimeInterval, timeIntervalMap); - - return acc; - }, new Map>()); - return classifiedMeasures; - }; - - const options = { - attributeNamePrefix: "", - ignoreAttributes: false, - parseAttributeValue: true, - textNodeName: "value", - updateTag(tagName: string, jPath: string, attrs: { [x: string]: string | number }) { - switch (tagName) { - case "trace-query-result": { - return "result"; - } - case "sample-time": { - return "sampleTime"; - } - default: { - return tagName; - } - } - }, - }; - const parser = new XMLParser(options); - const jsonObject: Result = parser.parse(xml); - if (!jsonObject.result.node.row) { - throw new Error("No rows in the xml file"); - } - - const measures: Map> = getMeasures(jsonObject.result.node.row); - const formattedMeasures: Measure[] = Array.from(measures.entries()).map( - (classifiedMeasures: [number, Map]) => { - const timeInterval = classifiedMeasures[0]; - const timeIntervalMap = classifiedMeasures[1]; - const cpuMeasure: CpuMeasure = { - perName: {}, - perCore: {}, - }; - timeIntervalMap.forEach((value: number, key: string) => { - cpuMeasure.perName[key] = (value * 10) / (TIME_INTERVAL / CPU_TIME_INTERVAL); - }); - return { - cpu: cpuMeasure, - ram: FAKE_RAM, - fps: FAKE_FPS, - time: timeInterval, - }; - } - ); - - iterations.push({ - time: formattedMeasures[formattedMeasures.length - 1].time, - measures: formattedMeasures, - status: "SUCCESS", - }); - - const results: TestCaseResult = { - name: "iOS Measures", - status: "SUCCESS", - iterations: iterations, - type: "IOS_EXPERIMENTAL", - }; - - fs.writeFileSync(outputFileName, JSON.stringify(results, null, 2)); -}; diff --git a/packages/platforms/ios/src/index.ts b/packages/platforms/ios/src/index.ts index f715c8af..ccc8348e 100644 --- a/packages/platforms/ios/src/index.ts +++ b/packages/platforms/ios/src/index.ts @@ -104,4 +104,12 @@ export class IOSProfiler implements Profiler { killApp(bundleId); return new Promise((resolve) => resolve()); } + + async waitUntilReady(bundleId: string): Promise { + return new Promise((resolve) => resolve()); + } + + async getMeasures(): Promise { + return new Promise((resolve) => resolve()); + } } diff --git a/packages/platforms/profiler/package.json b/packages/platforms/profiler/package.json index 3ae4eb6c..6e9f4084 100644 --- a/packages/platforms/profiler/package.json +++ b/packages/platforms/profiler/package.json @@ -16,6 +16,7 @@ "dependencies": { "@perf-profiler/android": "^0.10.7", "@perf-profiler/ios": "^0.2.0", + "@perf-profiler/ios-instruments": "^0.2.0", "@perf-profiler/types": "^0.6.0" } } diff --git a/packages/platforms/profiler/src/index.ts b/packages/platforms/profiler/src/index.ts index c3bdc822..e9ab087e 100644 --- a/packages/platforms/profiler/src/index.ts +++ b/packages/platforms/profiler/src/index.ts @@ -1,9 +1,20 @@ import { AndroidProfiler } from "@perf-profiler/android"; import { IOSProfiler } from "@perf-profiler/ios"; +import { IOSInstrumentsProfiler } from "@perf-profiler/ios-instruments"; import { Profiler } from "@perf-profiler/types"; -export const profiler: Profiler = - process.env.PLATFORM === "ios" ? new IOSProfiler() : new AndroidProfiler(); +export const profiler: Profiler = (() => { + switch (process.env.PLATFORM) { + case "android": + return new AndroidProfiler(); + case "ios": + return new IOSProfiler(); + case "ios-instruments": + return new IOSInstrumentsProfiler(); + default: + return new AndroidProfiler(); + } +})(); // TODO move this to a separate package export { waitFor } from "@perf-profiler/android";