mikehardy · September 2, 2022 19:15 · Apr 24, 2022 · Apr 24, 2022 · Apr 23, 2022 · Apr 23, 2022
diff --git a/example_results.csv b/example_results.csv
@@ -1,4 +1,82 @@
-"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"
-"25","x86","default","0","106","362","468","2"
-"25","x86","default","180","113.33333333333333","326.3333333333333","439.6666666666667","0"
-"25","x86","default","600","112.5","328","440.5","1"
+"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average 
+Total Elapsed Seconds","Failure Count"
+"23","x86","default","0","NaN","NaN","NaN","3"
+"23","x86","default","600","NaN","NaN","NaN","3"
+"23","x86","google_apis","0","NaN","NaN","NaN","3"
+"23","x86","google_apis","600","NaN","NaN","NaN","3"
+"23","x86_64","default","0","NaN","NaN","NaN","3"
+"23","x86_64","default","600","NaN","NaN","NaN","3"
+"23","x86_64","google_apis","0","NaN","NaN","NaN","3"
+"23","x86_64","google_apis","600","NaN","NaN","NaN","3"
+"24","x86","default","0","107.33333333333333","400","507.3333333333333","0"
+"24","x86","default","600","115","343","458","0"
+"24","x86","google_apis","0","179","439.6666666666667","618.6666666666666","0"
+"24","x86","google_apis","600","164.33333333333334","385.6666666666667","550","0"
+"24","x86_64","default","0","109","337.6666666666667","446.6666666666667","0"
+"24","x86_64","default","600","109.33333333333333","293.6666666666667","403","0"
+"24","x86_64","google_apis","0","245.33333333333334","414","659.3333333333334","0"
+"24","x86_64","google_apis","600","226","340.6666666666667","566.6666666666666","0"
+"25","x86","default","0","119.5","313","432.5","1"
+"25","x86","default","600","115","357.5","472.5","1"
+"25","x86","google_apis","0","178.33333333333334","434.6666666666667","613","0"
+"25","x86","google_apis","600","163","322","485","1"
+"25","x86_64","default","0","117","275","392","1"
+"25","x86_64","default","600","109","242","351","2"
+"25","x86_64","google_apis","0","187","364","551","2"
+"25","x86_64","google_apis","600","171","368","539","2"
+"26","x86","default","0","NaN","NaN","NaN","3"
+"26","x86","default","600","NaN","NaN","NaN","3"
+"26","x86","google_apis","0","159.66666666666666","426.3333333333333","586","0"
+"26","x86","google_apis","600","151.66666666666666","317.6666666666667","469.3333333333333","0"
+"26","x86_64","default","0","NaN","NaN","NaN","3"
+"26","x86_64","default","600","NaN","NaN","NaN","3"
+"26","x86_64","google_apis","0","176","396.3333333333333","572.3333333333334","0"
+"26","x86_64","google_apis","600","154","339.3333333333333","493.3333333333333","0"
+"27","x86","default","0","132","290","422","1"
+"27","x86","default","600","112.33333333333333","300","412.3333333333333","0"
+"27","x86","google_apis","0","167","424","591","0"
+"27","x86","google_apis","600","131.33333333333334","280.6666666666667","412","0"
+"27","x86_64","default","0","116.66666666666667","272","388.6666666666667","0"
+"27","x86_64","default","600","121.66666666666667","313","434.6666666666667","0"
+"27","x86_64","google_apis","0","NaN","NaN","NaN","3"
+"27","x86_64","google_apis","600","NaN","NaN","NaN","3"
+"28","x86","default","0","146.66666666666666","378.6666666666667","525.3333333333334","0"
+"28","x86","default","600","119.33333333333333","303.3333333333333","422.6666666666667","0"
+"28","x86","google_apis","0","246.33333333333334","417.3333333333333","663.6666666666666","0"
+"28","x86","google_apis","600","163","321.3333333333333","484.3333333333333","0"
+"28","x86_64","default","0","122","338.5","460.5","1"
+"28","x86_64","default","600","135","312.3333333333333","447.3333333333333","0"
+"28","x86_64","google_apis","0","NaN","NaN","NaN","3"
+"28","x86_64","google_apis","600","265","327.3333333333333","592.3333333333334","0"
+"29","x86","default","0","119","407","526","0"
+"29","x86","default","600","131.33333333333334","296","427.3333333333333","0"
+"29","x86","google_apis","0","169.33333333333334","505","674.3333333333334","0"
+"29","x86","google_apis","600","177.33333333333334","288","465.3333333333333","0"
+"29","x86_64","default","0","138.5","413","551.5","1"
+"29","x86_64","default","600","133.66666666666666","281.6666666666667","415.3333333333333","0"
+"29","x86_64","google_apis","0","169.66666666666666","378","547.6666666666666","0"
+"29","x86_64","google_apis","600","158","226.66666666666666","384.6666666666667","0"
+"30","x86","default","0","NaN","NaN","NaN","3"
+"30","x86","default","600","NaN","NaN","NaN","3"
+"30","x86","google_apis","0","257.6666666666667","559.3333333333334","817","0"
+"30","x86","google_apis","600","219.66666666666666","328","547.6666666666666","0"
+"30","x86_64","default","0","246","452.5","698.5","1"
+"30","x86_64","default","600","209.33333333333334","321.3333333333333","530.6666666666666","0"
+"30","x86_64","google_apis","0","270","658.5","928.5","1"
+"30","x86_64","google_apis","600","269.3333333333333","371.3333333333333","640.6666666666666","0"
+"31","x86","default","0","NaN","NaN","NaN","3"
+"31","x86","default","600","NaN","NaN","NaN","3"
+"31","x86","google_apis","0","NaN","NaN","NaN","3"
+"31","x86","google_apis","600","NaN","NaN","NaN","3"
+"31","x86_64","default","0","231","639.3333333333334","870.3333333333334","0"
+"31","x86_64","default","600","218.66666666666666","612.6666666666666","831.3333333333334","0"
+"31","x86_64","google_apis","0","307.6666666666667","1060.3333333333333","1368","0"
+"31","x86_64","google_apis","600","351","766.3333333333334","1117.3333333333333","0"
+"32","x86","default","0","NaN","NaN","NaN","3"
+"32","x86","default","600","NaN","NaN","NaN","3"
+"32","x86","google_apis","0","NaN","NaN","NaN","3"
+"32","x86","google_apis","600","NaN","NaN","NaN","3"
+"32","x86_64","default","0","NaN","NaN","NaN","3"
+"32","x86_64","default","600","NaN","NaN","NaN","3"
+"32","x86_64","google_apis","0","350.6666666666667","1159.3333333333333","1510","0"
+"32","x86_64","google_apis","600","349","802","1151","0"
diff --git a/README.md b/README.md
@@ -39,6 +39,14 @@ I have attempted to separate performance in to the major components:
 - "create+cold boot time" - time taken to install/create/start emulator. Emulator likey still performing background first boot tasks
 - "test execution time" - time taken to execute tests. May be affected by background first-boot tasks if any, running concurrently
 
+
+**API29 appears stable and fast (a great combo) and is my choice for single API runs.**
+
+It looks like several APIs are flaky.
+API32 is useful to test the newest version but is incredibly slow.
+Lower APIs are also flaky unfortunately so it's tough to test low-end of a compatibility bracket (API21 etc)
+Testing any of the other APIs should be done in a re-try loop because the APIs suffer frequent startup failures
+
 ### Test Results extractor
 
 - Access GitHub REST API to fetch test run 

diff --git a/fetch_workflow_jobs.sh b/fetch_workflow_jobs.sh
@@ -5,12 +5,14 @@ echo "Fetching jobs JSON for workflow run $1"
 rm -f emulator_perf_results_page*.json
 
 REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android
+PER_PAGE=100
+PAGE=1
+curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
 
-PER_PAGE=4
 TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json)
 LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1))
 echo "$TOTAL_COUNT jobs so $LAST_PAGE pages"
-for ((PAGE=1; PAGE <= LAST_PAGE; PAGE++)); do
+for ((PAGE=2; PAGE <= LAST_PAGE; PAGE++)); do
   echo "On iteration $PAGE"
   curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
 done

diff --git a/README.md b/README.md
@@ -0,0 +1,54 @@
+# Performance characteristics of Android emulators on GitHub Actions
+
+Lots of projects need to test android apps, and use GitHub Actions infrastructure to do so.
+
+This document intends to show current timings for a sample workload, to inform what emulators are a good match for testing.
+
+## Test Conditions
+
+- We use the AnkiDroid androidTests 
+ - they are long enough to execute that they form a nice balance between cold start emulator time so neither dominates
+ - they are open source, feel free to inspect
+- We use the macos runners, specfically macos-11 (though we reference it as macos-latest at the moment)
+ - the macos runner family is the only family that enables [virtual machine hardware acceleration](https://github.com/actions/virtual-environments/issues?q=is%3Aissue+nested+virtualization), a hard requirement
+- Build time of app under test is removed from testing, to focus on emulator performance only
+- We iterate a few times for each emulator style, as there can be quite a bit of variance
+
+## Emulator Test Matrix
+
+There are a wide variety of emulators available from Google's Android project. We will focus on:
+
+- API 25 minimm: 21 would be better but I have problems with 21-24, and I want to limit matrix job expansion to < 256 limit
+- API 32 maximum: this is the current maximum Android API available on stable channel
+- Arches x86 and x86_64: they have different performance characteristics, and different per-system-image failure modes
+- Target default and google_apis: they have different performance characteristics, and some workloads require Google APIs
+
+No attempt has been made to test the matrix for different RAM or Disk sizes, but I would welcome further work from someone
+if they were interested in performing it. I imagine smaller disk, within reason, would speed up emulator creation and more RAM
+would speed up emulator execution up to a point.
+
+Performance hypothesis:
+- the x86 arch, on a default (non google_apis) target, somewhere in the high 20s (perhaps API28) will be the fastest emulator.
+- older APIs will have complete failures for various bitrot-related reasons that offer low-value with regard to diagnosis and are best ignored
+
+
+## Emulator Test Results
+
+I have attempted to separate performance in to the major components:
+
+- "create+cold boot time" - time taken to install/create/start emulator. Emulator likey still performing background first boot tasks
+- "test execution time" - time taken to execute tests. May be affected by background first-boot tasks if any, running concurrently
+
+### Test Results extractor
+
+- Access GitHub REST API to fetch test run 
+  - `fetch_workflow_jobs.sh <workflow run id>`
+- Parse out major component times from the logs and format as csv for analysis
+  - `node analyze_emulator_performance.js emulator_perf_results.json`
+
+## Further Work
+
+- Test different RAM sizes. Hypothesis: RAM may improves first boot + test velocity, until virtual runner memory is fully utilized
+- Test different disk sizes. Hypothesis: Smaller disk improves install velocity, until it is too small to contain the app+system
+- Test more targets. Hypothesis: play store images will be really slow, but there are also watch images people may be interested in
+- Examine AVD snapshot size: Hypothesis: RAM size or some other factor may affect snapshot size, which affects caching
diff --git a/analyze_emulator_performance.js b/analyze_emulator_performance.js
@@ -0,0 +1,101 @@
+// Fetch results to parse like this, with the workflow run id you want:
+// curl https://api.github.com/repos/mikehardy/Anki-Android/actions/runs/2210525974/jobs?per_page=100 > emulator_perf_results.json
+
+// Or if you have more than 100 results, you need to page through them and merge them, there is a script
+// ./fetch_workflow_jobs_json.sh 2212862357
+
+function main() {
+    // Read in the results
+    // console.log("Processing results in " + process.argv[2]);
+    var fs = require("fs");
+    var runLog = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+
+    console.log(
+        '"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"',
+    );
+
+    let averageTimings = {};
+
+    runLog.jobs.forEach(job => {
+        // console.log("analyzing job " + job.name);
+        const matrixVars = job.name.match(/.*\((.*)\)/)[1].split(", ");
+        // console.log("Job name: " + job.name);
+        // console.log("  Android API level: " + matrixVars[0]);
+        // console.log("  Emulator Architecture: " + matrixVars[1]);
+        // console.log("  Emulator Image: " + matrixVars[2]);
+
+        const startTime = new Date(job.started_at);
+        const endTime = new Date(job.completed_at);
+        let jobElapsed = endTime - startTime;
+        jobElapsed = jobElapsed > 0 ? jobElapsed : 0; // some are negative !?
+
+        // console.log("  conclusion: " + job.conclusion);
+        // console.log("  elapsed_time_seconds: " + jobElapsed / 1000);
+
+        let AVDCreateBootElapsedSeconds = -1;
+        let AVDRebootTestElapsedSeconds = -1;
+        let stepFailed = false;
+
+        job.steps.forEach(step => {
+            if (!["success", "skipped"].includes(step.conclusion)) {
+                stepFailed = true;
+                return;
+            }
+            const stepStart = new Date(step.started_at);
+            const stepEnd = new Date(step.completed_at);
+            let stepElapsedSeconds = (stepEnd - stepStart) / 1000;
+            stepElapsedSeconds = stepElapsedSeconds > 0 ? stepElapsedSeconds : 0; // some are negative !?
+
+            switch (step.name) {
+                case "AVD Boot and Snapshot Creation":
+                    AVDCreateBootElapsedSeconds = stepElapsedSeconds;
+                case "Run Emulator Tests":
+                    AVDRebootTestElapsedSeconds = stepElapsedSeconds;
+            }
+        });
+
+        // Get or create aggregate timing entry
+        timingKey = `${matrixVars[0]}_${matrixVars[1]}_${matrixVars[2]}_${matrixVars[3]}`;
+        let currentAverageTiming = averageTimings[timingKey];
+        if (currentAverageTiming === undefined) {
+            currentAverageTiming = {
+                api: matrixVars[0],
+                arch: matrixVars[1],
+                target: matrixVars[2],
+                warmtime: matrixVars[3],
+                totalCreateBootElapsedSecs: 0,
+                totalTestElapsedSecs: 0,
+                runs: 0,
+                failureCount: 0,
+            };
+            averageTimings[timingKey] = currentAverageTiming;
+        }
+
+        // If something failed, set status and skip timing aggregation
+        if (stepFailed) {
+            currentAverageTiming.failureCount++;
+            return;
+        }
+
+        // Update our aggregate timings
+        currentAverageTiming.totalCreateBootElapsedSecs += AVDCreateBootElapsedSeconds;
+        currentAverageTiming.totalTestElapsedSecs += AVDRebootTestElapsedSeconds;
+        currentAverageTiming.runs++;
+    });
+
+    // Print out averages for each non-iteration combo
+    Object.keys(averageTimings).forEach(key => {
+        // console.log("printing timings for key " + key);
+        const timing = averageTimings[key];
+        // console.log("entry is " + JSON.stringify(timing));
+        console.log(
+            `"${timing.api}","${timing.arch}","${timing.target}","${timing.warmtime}","${
+                timing.totalCreateBootElapsedSecs / timing.runs
+            }","${timing.totalTestElapsedSecs / timing.runs}","${
+                (timing.totalCreateBootElapsedSecs + timing.totalTestElapsedSecs) / timing.runs
+            }","${timing.failureCount}"`,
+        );
+    });
+}
+
+main();
diff --git a/example_results.csv b/example_results.csv
@@ -0,0 +1,4 @@
+"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"
+"25","x86","default","0","106","362","468","2"
+"25","x86","default","180","113.33333333333333","326.3333333333333","439.6666666666667","0"
+"25","x86","default","600","112.5","328","440.5","1"
diff --git a/fetch_workflow_jobs.sh b/fetch_workflow_jobs.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+echo "Fetching jobs JSON for workflow run $1"
+
+rm -f emulator_perf_results_page*.json
+
+REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android
+
+PER_PAGE=4
+TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json)
+LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1))
+echo "$TOTAL_COUNT jobs so $LAST_PAGE pages"
+for ((PAGE=1; PAGE <= LAST_PAGE; PAGE++)); do
+  echo "On iteration $PAGE"
+  curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
+done
+
+jq -s 'def deepmerge(a;b):
+  reduce b[] as $item (a;
+    reduce ($item | keys_unsorted[]) as $key (.;
+      $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then
+        deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val])
+      elif ($type == "array") then
+        (.[$key] + $val | unique)
+      else
+        $val
+      end)
+    );
+  deepmerge({}; .)' emulator_perf_results_page*.json > emulator_perf_results.json
+
+rm -f emulator_perf_results_page*.json
No results found