Add the iOS instrument Profiler

.github/workflows/ios_e2e.sh

@@ -22,4 +22,4 @@ xcrun simctl install $UDID ./.github/workflows/fakeStore.app
 xcrun simctl launch $UDID $APPID
 
 mkdir -p report
-npx flashlight-ios-poc ios-test --appId $APPID --simulatorId $UDID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json'
+PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId $APPID --testCommand 'maestro test ./packages/platforms/ios-instruments/test.yaml' --resultsFilePath './report/result.json' --iterationCount 2

packages/commands/test/src/PerformanceMeasurer.ts

@@ -19,9 +19,12 @@ export class PerformanceMeasurer {
       // noop by default
     }
   ) {
+    // Hack to make sure the profiler is ready to receive measures
+    await profiler.waitUntilReady(this.bundleId);
     this.polling = profiler.pollPerformanceMeasures(this.bundleId, {
       onMeasure: (measure) => {
-        if (this.shouldStop) {
+        // The ios-instruments profiler yields measures at the end of the test when the polling is already stopped
+        if (this.shouldStop && process.env.PLATFORM !== "ios-instruments") {
           this.polling?.stop();
         }
 
@@ -60,6 +63,8 @@ export class PerformanceMeasurer {
 
     // Ensure polling has stopped
     this.polling?.stop();
+    // Hack for ios-instruments to get the measures at the end of the test
+    await profiler.getMeasures();
 
     return {
       time: time ?? 0,

packages/core/types/index.ts

@@ -85,4 +85,6 @@ export interface Profiler {
   cleanup: () => void;
   getScreenRecorder: (videoPath: string) => ScreenRecorder | undefined;
   stopApp: (bundleId: string) => Promise<void>;
+  waitUntilReady: (bundleId: string) => Promise<void>;
+  getMeasures: () => Promise<void>;
 }

packages/platforms/android/src/index.ts

@@ -12,6 +12,12 @@ export { waitFor } from "./utils/waitFor";
 export { executeAsync, executeCommand } from "./commands/shell";
 
 export class AndroidProfiler implements Profiler {
+  waitUntilReady = (bundleId: string) => {
+    return new Promise<void>((resolve) => resolve());
+  };
+  getMeasures = () => {
+    return new Promise<void>((resolve) => resolve());
+  };
   pollPerformanceMeasures = pollPerformanceMeasures;
   detectCurrentBundleId = profiler.detectCurrentBundleId;
   installProfilerOnDevice = ensureCppProfilerIsInstalled;

packages/platforms/ios-instruments/README.md

@@ -10,7 +10,7 @@ Requirements:
 - Get a running simulator id with `xcrun simctl list devices`
 - Create template Flashlight in Xcode Instruments (with cpu-profile and memory usage)
 - Add your own test in `test.yaml`
-- `flashlight-ios-poc ios-test --appId <YOUR_APP_ID> --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"`
+- `PLATFORM=ios-instruments node packages/commands/test/dist/bin.js test --bundleId <YOUR_APP_ID> --simulatorId 9F852910-03AD-495A-8E16-7356B764284 --testCommand "maestro test test.yaml" --resultsFilePath "./result.json"`
 
 - Check the results in the web-reporter
   `yarn workspace @perf-profiler/web-reporter build`
@@ -19,5 +19,5 @@ Requirements:
 ## Next steps
 
 - run several iterations
-- add more metrics (RAM, FPS, CPU per thread)
+- add more metrics (RAM, FPS)
 - Unify API with flashlight test

packages/platforms/ios-instruments/src/XcodePerfParser.ts

@@ -0,0 +1,112 @@
+import { XMLParser } from "fast-xml-parser";
+import { Result, Row, Thread, isRefField } from "./utils/xmlTypes";
+import fs from "fs";
+import { CpuMeasure, Measure } from "@perf-profiler/types";
+
+const FAKE_RAM = 200;
+const FAKE_FPS = 60;
+const TIME_INTERVAL = 500;
+const NANOSEC_TO_MILLISEC = 1_000_000;
+const CPU_TIME_INTERVAL = 10;
+
+const initThreadMap = (row: Row[]): { [id: number]: string } => {
+  const threadRef: { [id: number]: Thread } = {};
+  row.forEach((row: Row) => {
+    if (!isRefField(row.thread)) {
+      threadRef[row.thread.id] = row.thread;
+    }
+  });
+  return Object.values(threadRef).reduce((acc: { [id: number]: string }, thread) => {
+    const currentThreadName = thread.fmt
+      .split(" ")
+      .slice(0, thread.fmt.split(" ").indexOf(""))
+      .join(" ");
+    const currentTid = thread.tid.value;
+    const numberOfThread = Object.values(threadRef).filter((thread: Thread) => {
+      return thread.fmt.includes(currentThreadName) && thread.tid.value < currentTid;
+    }).length;
+    acc[thread.id] =
+      numberOfThread > 0 ? `${currentThreadName} (${numberOfThread})` : currentThreadName;
+    return acc;
+  }, {});
+};
+
+const getMeasures = (row: Row[]): Map<number, Map<string, number>> => {
+  const sampleTimeRef: { [id: number]: number } = {};
+  const threadRef: { [id: number]: string } = initThreadMap(row);
+  const classifiedMeasures = row.reduce((acc: Map<number, Map<string, number>>, row: Row) => {
+    const sampleTime = isRefField(row.sampleTime)
+      ? sampleTimeRef[row.sampleTime.ref]
+      : row.sampleTime.value / NANOSEC_TO_MILLISEC;
+    if (!isRefField(row.sampleTime)) {
+      sampleTimeRef[row.sampleTime.id] = sampleTime;
+    }
+
+    const threadName = isRefField(row.thread)
+      ? threadRef[row.thread.ref]
+      : threadRef[row.thread.id];
+
+    const correspondingTimeInterval =
+      parseInt((sampleTime / TIME_INTERVAL).toFixed(0), 10) * TIME_INTERVAL;
+
+    const timeIntervalMap = acc.get(correspondingTimeInterval) ?? new Map<string, number>();
+
+    const numberOfPointsIn = timeIntervalMap.get(threadName) ?? 0;
+
+    timeIntervalMap.set(threadName, numberOfPointsIn + 1);
+
+    acc.set(correspondingTimeInterval, timeIntervalMap);
+
+    return acc;
+  }, new Map<number, Map<string, number>>());
+  return classifiedMeasures;
+};
+
+export const computeMeasures = (inputFileName: string) => {
+  const xml = fs.readFileSync(inputFileName, "utf8");
+  const options = {
+    attributeNamePrefix: "",
+    ignoreAttributes: false,
+    parseAttributeValue: true,
+    textNodeName: "value",
+    updateTag(tagName: string, jPath: string, attrs: { [x: string]: string | number }) {
+      switch (tagName) {
+        case "trace-query-result": {
+          return "result";
+        }
+        case "sample-time": {
+          return "sampleTime";
+        }
+        default: {
+          return tagName;
+        }
+      }
+    },
+  };
+  const parser = new XMLParser(options);
+  const jsonObject: Result = parser.parse(xml);
+  if (!jsonObject.result.node.row) {
+    throw new Error("No rows in the xml file");
+  }
+  const measures: Map<number, Map<string, number>> = getMeasures(jsonObject.result.node.row);
+  const formattedMeasures: Measure[] = Array.from(measures.entries()).map(
+    (classifiedMeasures: [number, Map<string, number>]) => {
+      const timeInterval = classifiedMeasures[0];
+      const timeIntervalMap = classifiedMeasures[1];
+      const cpuMeasure: CpuMeasure = {
+        perName: {},
+        perCore: {},
+      };
+      timeIntervalMap.forEach((value: number, key: string) => {
+        cpuMeasure.perName[key] = (value * 10) / (TIME_INTERVAL / CPU_TIME_INTERVAL);
+      });
+      return {
+        cpu: cpuMeasure,
+        ram: FAKE_RAM,
+        fps: FAKE_FPS,
+        time: timeInterval,
+      };
+    }
+  );
+  return formattedMeasures;
+};

packages/platforms/ios-instruments/src/index.ts

@@ -1 +1,120 @@
+import { Measure, Profiler, ProfilerPollingOptions, ScreenRecorder } from "@perf-profiler/types";
+import { ChildProcess } from "child_process";
+// TODO: refactor so that these functions are not in android
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { executeAsync, executeCommand } from "@perf-profiler/android";
+import { IdbDevice, getConnectedDevice, killApp, launchApp } from "./utils/DeviceManager";
+import { computeMeasures } from "./XcodePerfParser";
+import { getTmpFilePath, removeTmpFiles } from "./utils/tmpFileManager";
 export { killApp } from "./utils/DeviceManager";
+
+const startRecord = async (
+  deviceUdid: string,
+  appPid: number,
+  traceFile: string
+): Promise<ChildProcess> => {
+  const templateFilePath = `${__dirname}/../Flashlight.tracetemplate`;
+  const recordingProcess = executeAsync(
+    `xcrun xctrace record --device ${deviceUdid} --template ${templateFilePath} --attach ${appPid} --output ${traceFile}`
+  );
+  await new Promise<void>((resolve) => {
+    recordingProcess.stdout?.on("data", (data) => {
+      if (data.toString().includes("Ctrl-C to stop")) {
+        resolve();
+      }
+    });
+  });
+  return recordingProcess;
+};
+
+const saveTraceFile = (traceFile: string): string => {
+  const xmlOutputFile = getTmpFilePath("report.xml");
+  executeCommand(
+    `xctrace export --input ${traceFile} --xpath '/trace-toc/run[@number="1"]/data/table[@schema="time-profile"]' --output ${xmlOutputFile}`
+  );
+  return xmlOutputFile;
+};
+
+const stopPerfRecord = async (
+  recordingProcess: ChildProcess,
+  traceFile: string,
+  onMeasure: (measure: Measure) => void
+) => {
+  try {
+    await new Promise<void>((resolve) => {
+      recordingProcess.stdout?.on("data", (data) => {
+        if (data.toString().includes("Output file saved as")) {
+          resolve();
+        }
+      });
+    });
+  } catch (e) {
+    console.log("Error while recording: ", e);
+  }
+  const xmlFile = saveTraceFile(traceFile);
+  const measures = computeMeasures(xmlFile);
+  measures.forEach((measure) => {
+    onMeasure(measure);
+  });
+  removeTmpFiles();
+};
+
+export class IOSInstrumentsProfiler implements Profiler {
+  connectedDevice: IdbDevice | undefined;
+  recordingProcess: ChildProcess | undefined;
+  traceFile: string | undefined;
+  pid: number | undefined;
+  bundleId: string | undefined;
+  onMeasure: ((measure: Measure) => void) | undefined;
+  pollPerformanceMeasures(bundleId: string, options: ProfilerPollingOptions): { stop: () => void } {
+    if (!this.pid) throw new Error("Profiler is not ready, app is not running");
+    this.onMeasure = options.onMeasure;
+    return {
+      stop: () => {
+        return;
+      },
+    };
+  }
+
+  detectCurrentBundleId(): string {
+    throw new Error("App Id detection is not implemented on iOS with Instruments");
+  }
+
+  installProfilerOnDevice() {
+    // Do we need anything here?
+  }
+
+  getScreenRecorder(videoPath: string): ScreenRecorder | undefined {
+    return undefined;
+  }
+
+  cleanup: () => void = () => {
+    // Do we need anything here?
+  };
+
+  async waitUntilReady(bundleId: string): Promise<void> {
+    this.connectedDevice = getConnectedDevice();
+    if (!this.connectedDevice) {
+      throw new Error("No device connected");
+    }
+    this.bundleId = bundleId;
+    this.pid = launchApp(bundleId);
+    const traceFile = `report_${new Date().getTime()}.trace`;
+    this.traceFile = traceFile;
+    this.recordingProcess = await startRecord(this.connectedDevice.udid, this.pid, traceFile);
+  }
+
+  async getMeasures(): Promise<void> {
+    if (!this.recordingProcess || !this.traceFile || !this.pid || !this.onMeasure || !this.bundleId)
+      throw new Error("Profiler is not ready to get measures");
+    const recordingProcess = this.recordingProcess;
+    const traceFile = this.traceFile;
+    killApp(this.bundleId);
+    await stopPerfRecord(recordingProcess, traceFile, this.onMeasure);
+  }
+
+  async stopApp(bundleId: string): Promise<void> {
+    killApp(bundleId);
+    return new Promise<void>((resolve) => resolve());
+  }
+}