[HOLD] Osp/reboot emulator on e2e tests

src/libs/E2E/tests/appStartTimeTest.e2e.js

@@ -5,11 +5,11 @@ import E2EClient from '../client';
 
 const test = () => {
     // check for login (if already logged in the action will simply resolve)
-    E2ELogin().then((neededLogin) => {
-        if (neededLogin) {
-            // we don't want to submit the first login to the results
-            return E2EClient.submitTestDone();
-        }
+    E2ELogin().then(() => {
+        // if (neededLogin) {
+        //     // we don't want to submit the first login to the results
+        //     return E2EClient.submitTestDone();
+        // }
 
         console.debug('[E2E] Logged in, getting metrics and submitting them…');
 

tests/e2e/config.local.js

@@ -1,8 +1,6 @@
 module.exports = {
     APP_PACKAGE: 'com.expensify.chat.dev',
-
-    WARM_UP_RUNS: 1,
-    RUNS: 8,
+    RUNS: 30,
     APP_PATHS: {
         baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
         compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',

tests/e2e/measure/math.js

@@ -1,25 +1,25 @@
 const _ = require('underscore');
 
-const filterOutliersViaIQR = (data) => {
-    let q1;
-    let q3;
+// const filterOutliersViaIQR = (data) => {
+//     let q1;
+//     let q3;
 
-    const values = data.slice().sort((a, b) => a - b);
+//     const values = data.slice().sort((a, b) => a - b);
 
-    if ((values.length / 4) % 1 === 0) {
-        q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]);
-        q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]);
-    } else {
-        q1 = values[Math.floor(values.length / 4 + 1)];
-        q3 = values[Math.ceil(values.length * (3 / 4) + 1)];
-    }
+//     if ((values.length / 4) % 1 === 0) {
+//         q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]);
+//         q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]);
+//     } else {
+//         q1 = values[Math.floor(values.length / 4 + 1)];
+//         q3 = values[Math.ceil(values.length * (3 / 4) + 1)];
+//     }
 
-    const iqr = q3 - q1;
-    const maxValue = q3 + iqr * 1.5;
-    const minValue = q1 - iqr * 1.5;
+//     const iqr = q3 - q1;
+//     const maxValue = q3 + iqr * 1.5;
+//     const minValue = q1 - iqr * 1.5;
 
-    return _.filter(values, (x) => x >= minValue && x <= maxValue);
-};
+//     return _.filter(values, (x) => x >= minValue && x <= maxValue);
+// };
 
 const mean = (arr) => _.reduce(arr, (a, b) => a + b, 0) / arr.length;
 
@@ -34,15 +34,18 @@ const std = (arr) => {
 };
 
 const getStats = (entries) => {
-    const cleanedEntries = filterOutliersViaIQR(entries);
-    const meanDuration = mean(cleanedEntries);
-    const stdevDuration = std(cleanedEntries);
+    // TBD I don't think outliers should be discarded. On certain data sets such as user collection it might make sense.
+    // But in theory the app is a defined system. Discarting outliers will effectively hide extreme cases where app might hang
+    // or be stuck in a race condition
+    // const cleanedEntries = filterOutliersViaIQR(entries);
+    const meanDuration = mean(entries);
+    const stdevDuration = std(entries);
 
     return {
         mean: meanDuration,
         stdev: stdevDuration,
-        runs: cleanedEntries.length,
-        entries: cleanedEntries,
+        runs: entries.length,
+        entries,
     };
 };
 

tests/e2e/testRunner.js

@@ -13,7 +13,6 @@ const defaultConfig = require('./config');
 const compare = require('./compare/compare');
 const Logger = require('./utils/logger');
 const execAsync = require('./utils/execAsync');
-const killApp = require('./utils/killApp');
 const launchApp = require('./utils/launchApp');
 const createServerInstance = require('./server');
 const installApp = require('./utils/installApp');
@@ -22,6 +21,12 @@ const writeTestStats = require('./measure/writeTestStats');
 const withFailTimeout = require('./utils/withFailTimeout');
 const reversePort = require('./utils/androidReversePort');
 const getCurrentBranchName = require('./utils/getCurrentBranchName');
+const startEmulator = require('./utils/startEmulator');
+const killEmulator = require('./utils/killEmulator');
+const getDevices = require('./utils/getDevices');
+const sleep = require('./utils/sleep');
+const checkEmulatorIsBooted = require('./utils/checkEmulatorIsBooted');
+const listEmulators = require('./utils/listEmulators');
 
 const args = process.argv.slice(2);
 
@@ -71,9 +76,37 @@ if (isDevMode) {
     Logger.note(`Running in development mode. Set baseline branch to same as current ${baselineBranch}`);
 }
 
-const restartApp = async () => {
-    Logger.log('Killing app …');
-    await killApp('android', config.APP_PACKAGE);
+const restartApp = async (appPath) => {
+    let list = await listEmulators();
+    list = list.split('\n');
+
+    const emulatorName = list[0];
+
+    const devices = await getDevices();
+    Logger.log('DEVICES');
+    Logger.log(devices.trim().split('\n'));
+    if (devices.trim().split('\n').length > 1) {
+        killEmulator();
+        await sleep(10000);
+    }
+
+    if (!emulatorName) {
+        throw new Error('No emulator found');
+    }
+
+    await startEmulator(emulatorName);
+    // await sleep(5000);
+
+    let emulatorIsBooted = await checkEmulatorIsBooted();
+    while (emulatorIsBooted.trim() !== '1') {
+        await sleep(1000);
+        emulatorIsBooted = await checkEmulatorIsBooted();
+    }
+    // Give it a bit more time to finish booting
+    await sleep(5000);
+    Logger.log(`Installing app from ${appPath} …`);
+    await installApp('android', config.APP_PACKAGE, appPath);
+    await reversePort();
     Logger.log('Launching app …');
     await launchApp('android', config.APP_PACKAGE);
 };
@@ -127,8 +160,8 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => {
 
     // Install app and reverse port
     let progressLog = Logger.progressInfo('Installing app and reversing port');
-    await installApp('android', config.APP_PACKAGE, appPath);
-    await reversePort();
+    // await installApp('android', config.APP_PACKAGE, appPath);
+    // await reversePort();
     progressLog.done();
 
     // Start the HTTP server
@@ -169,33 +202,13 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => {
 
         server.setTestConfig(testConfig);
 
-        const warmupLogs = Logger.progressInfo(`Running test '${testConfig.name}'`);
-        for (let warmUpRuns = 0; warmUpRuns < config.WARM_UP_RUNS; warmUpRuns++) {
-            const progressText = `(${testIndex + 1}/${numOfTests}) Warmup for test '${testConfig.name}' (iteration ${warmUpRuns + 1}/${config.WARM_UP_RUNS})`;
-            warmupLogs.updateText(progressText);
-
-            await restartApp();
-
-            await withFailTimeout(
-                new Promise((resolve) => {
-                    const cleanup = server.addTestDoneListener(() => {
-                        Logger.log(`Warmup ${warmUpRuns + 1} done!`);
-                        cleanup();
-                        resolve();
-                    });
-                }),
-                progressText,
-            );
-        }
-        warmupLogs.done();
-
         // We run each test multiple time to average out the results
         const testLog = Logger.progressInfo('');
         for (let i = 0; i < config.RUNS; i++) {
             const progressText = `(${testIndex + 1}/${numOfTests}) Running test '${testConfig.name}' (iteration ${i + 1}/${config.RUNS})`;
             testLog.updateText(progressText);
 
-            await restartApp();
+            await restartApp(appPath);
 
             // Wait for a test to finish by waiting on its done call to the http server
             try {

tests/e2e/utils/checkEmulatorIsBooted.js

@@ -0,0 +1,6 @@
+const execAsync = require('./execAsync');
+
+module.exports = function () {
+    // Use adb to kill the app
+    return execAsync(`adb shell getprop sys.boot_completed | tr -d '\r'`);
+};

tests/e2e/utils/getDevices.js

@@ -0,0 +1,5 @@
+const execAsync = require('./execAsync');
+
+module.exports = function () {
+    return execAsync(`adb devices`);
+};

tests/e2e/utils/killEmulator.js

@@ -0,0 +1,5 @@
+const execAsync = require('./execAsync');
+
+module.exports = function () {
+    return execAsync(`adb emu kill`);
+};

tests/e2e/utils/listEmulators.js

@@ -0,0 +1,9 @@
+const execAsync = require('./execAsync');
+
+module.exports = function (platform = 'android') {
+    if (platform !== 'android') {
+        throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`);
+    }
+
+    return execAsync(`emulator -list-avds`);
+};

tests/e2e/utils/rebootEmulator.js

@@ -0,0 +1,10 @@
+const execAsync = require('./execAsync');
+
+module.exports = function (platform = 'android') {
+    if (platform !== 'android') {
+        throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`);
+    }
+
+    // Use adb to kill the app
+    return execAsync(`adb -e reboot`);
+};

tests/e2e/utils/sleep.js

@@ -0,0 +1,6 @@
+module.exports = function (ms) {
+    // Use adb to kill the app
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};

tests/e2e/utils/startEmulator.js

@@ -0,0 +1,5 @@
+const execAsync = require('./execAsync');
+
+module.exports = function (emulatorName) {
+    return execAsync(`emulator @${emulatorName} -no-snapstorage > /dev/null 2>&1 &`);
+};