diff --git a/src/libs/E2E/tests/appStartTimeTest.e2e.js b/src/libs/E2E/tests/appStartTimeTest.e2e.js index d44fd6ce1195..b0973a68f4d8 100644 --- a/src/libs/E2E/tests/appStartTimeTest.e2e.js +++ b/src/libs/E2E/tests/appStartTimeTest.e2e.js @@ -5,11 +5,11 @@ import E2EClient from '../client'; const test = () => { // check for login (if already logged in the action will simply resolve) - E2ELogin().then((neededLogin) => { - if (neededLogin) { - // we don't want to submit the first login to the results - return E2EClient.submitTestDone(); - } + E2ELogin().then(() => { + // if (neededLogin) { + // // we don't want to submit the first login to the results + // return E2EClient.submitTestDone(); + // } console.debug('[E2E] Logged in, getting metrics and submitting them…'); diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js index 0c38c3f1056f..f55139b5d301 100644 --- a/tests/e2e/config.local.js +++ b/tests/e2e/config.local.js @@ -1,8 +1,6 @@ module.exports = { APP_PACKAGE: 'com.expensify.chat.dev', - - WARM_UP_RUNS: 1, - RUNS: 8, + RUNS: 30, APP_PATHS: { baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', diff --git a/tests/e2e/measure/math.js b/tests/e2e/measure/math.js index 712e197d367b..6aee0b775614 100644 --- a/tests/e2e/measure/math.js +++ b/tests/e2e/measure/math.js @@ -1,25 +1,25 @@ const _ = require('underscore'); -const filterOutliersViaIQR = (data) => { - let q1; - let q3; +// const filterOutliersViaIQR = (data) => { +// let q1; +// let q3; - const values = data.slice().sort((a, b) => a - b); +// const values = data.slice().sort((a, b) => a - b); - if ((values.length / 4) % 1 === 0) { - q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]); - q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]); - } else { - q1 = values[Math.floor(values.length / 4 + 1)]; - q3 = values[Math.ceil(values.length * (3 / 4) + 1)]; - } +// if ((values.length / 4) % 1 === 0) { +// q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]); +// q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]); +// } else { +// q1 = values[Math.floor(values.length / 4 + 1)]; +// q3 = values[Math.ceil(values.length * (3 / 4) + 1)]; +// } - const iqr = q3 - q1; - const maxValue = q3 + iqr * 1.5; - const minValue = q1 - iqr * 1.5; +// const iqr = q3 - q1; +// const maxValue = q3 + iqr * 1.5; +// const minValue = q1 - iqr * 1.5; - return _.filter(values, (x) => x >= minValue && x <= maxValue); -}; +// return _.filter(values, (x) => x >= minValue && x <= maxValue); +// }; const mean = (arr) => _.reduce(arr, (a, b) => a + b, 0) / arr.length; @@ -34,15 +34,18 @@ const std = (arr) => { }; const getStats = (entries) => { - const cleanedEntries = filterOutliersViaIQR(entries); - const meanDuration = mean(cleanedEntries); - const stdevDuration = std(cleanedEntries); + // TBD I don't think outliers should be discarded. On certain data sets such as user collection it might make sense. + // But in theory the app is a defined system. Discarting outliers will effectively hide extreme cases where app might hang + // or be stuck in a race condition + // const cleanedEntries = filterOutliersViaIQR(entries); + const meanDuration = mean(entries); + const stdevDuration = std(entries); return { mean: meanDuration, stdev: stdevDuration, - runs: cleanedEntries.length, - entries: cleanedEntries, + runs: entries.length, + entries, }; }; diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 2a5aee78715f..4e0a08ad13df 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -13,7 +13,6 @@ const defaultConfig = require('./config'); const compare = require('./compare/compare'); const Logger = require('./utils/logger'); const execAsync = require('./utils/execAsync'); -const killApp = require('./utils/killApp'); const launchApp = require('./utils/launchApp'); const createServerInstance = require('./server'); const installApp = require('./utils/installApp'); @@ -22,6 +21,12 @@ const writeTestStats = require('./measure/writeTestStats'); const withFailTimeout = require('./utils/withFailTimeout'); const reversePort = require('./utils/androidReversePort'); const getCurrentBranchName = require('./utils/getCurrentBranchName'); +const startEmulator = require('./utils/startEmulator'); +const killEmulator = require('./utils/killEmulator'); +const getDevices = require('./utils/getDevices'); +const sleep = require('./utils/sleep'); +const checkEmulatorIsBooted = require('./utils/checkEmulatorIsBooted'); +const listEmulators = require('./utils/listEmulators'); const args = process.argv.slice(2); @@ -71,9 +76,37 @@ if (isDevMode) { Logger.note(`Running in development mode. Set baseline branch to same as current ${baselineBranch}`); } -const restartApp = async () => { - Logger.log('Killing app …'); - await killApp('android', config.APP_PACKAGE); +const restartApp = async (appPath) => { + let list = await listEmulators(); + list = list.split('\n'); + + const emulatorName = list[0]; + + const devices = await getDevices(); + Logger.log('DEVICES'); + Logger.log(devices.trim().split('\n')); + if (devices.trim().split('\n').length > 1) { + killEmulator(); + await sleep(10000); + } + + if (!emulatorName) { + throw new Error('No emulator found'); + } + + await startEmulator(emulatorName); + // await sleep(5000); + + let emulatorIsBooted = await checkEmulatorIsBooted(); + while (emulatorIsBooted.trim() !== '1') { + await sleep(1000); + emulatorIsBooted = await checkEmulatorIsBooted(); + } + // Give it a bit more time to finish booting + await sleep(5000); + Logger.log(`Installing app from ${appPath} …`); + await installApp('android', config.APP_PACKAGE, appPath); + await reversePort(); Logger.log('Launching app …'); await launchApp('android', config.APP_PACKAGE); }; @@ -127,8 +160,8 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { // Install app and reverse port let progressLog = Logger.progressInfo('Installing app and reversing port'); - await installApp('android', config.APP_PACKAGE, appPath); - await reversePort(); + // await installApp('android', config.APP_PACKAGE, appPath); + // await reversePort(); progressLog.done(); // Start the HTTP server @@ -169,33 +202,13 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { server.setTestConfig(testConfig); - const warmupLogs = Logger.progressInfo(`Running test '${testConfig.name}'`); - for (let warmUpRuns = 0; warmUpRuns < config.WARM_UP_RUNS; warmUpRuns++) { - const progressText = `(${testIndex + 1}/${numOfTests}) Warmup for test '${testConfig.name}' (iteration ${warmUpRuns + 1}/${config.WARM_UP_RUNS})`; - warmupLogs.updateText(progressText); - - await restartApp(); - - await withFailTimeout( - new Promise((resolve) => { - const cleanup = server.addTestDoneListener(() => { - Logger.log(`Warmup ${warmUpRuns + 1} done!`); - cleanup(); - resolve(); - }); - }), - progressText, - ); - } - warmupLogs.done(); - // We run each test multiple time to average out the results const testLog = Logger.progressInfo(''); for (let i = 0; i < config.RUNS; i++) { const progressText = `(${testIndex + 1}/${numOfTests}) Running test '${testConfig.name}' (iteration ${i + 1}/${config.RUNS})`; testLog.updateText(progressText); - await restartApp(); + await restartApp(appPath); // Wait for a test to finish by waiting on its done call to the http server try { diff --git a/tests/e2e/utils/checkEmulatorIsBooted.js b/tests/e2e/utils/checkEmulatorIsBooted.js new file mode 100644 index 000000000000..8bf9bbcda4d9 --- /dev/null +++ b/tests/e2e/utils/checkEmulatorIsBooted.js @@ -0,0 +1,6 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + // Use adb to kill the app + return execAsync(`adb shell getprop sys.boot_completed | tr -d '\r'`); +}; diff --git a/tests/e2e/utils/getDevices.js b/tests/e2e/utils/getDevices.js new file mode 100644 index 000000000000..984bb71e5a9a --- /dev/null +++ b/tests/e2e/utils/getDevices.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + return execAsync(`adb devices`); +}; diff --git a/tests/e2e/utils/killEmulator.js b/tests/e2e/utils/killEmulator.js new file mode 100644 index 000000000000..9209961461d5 --- /dev/null +++ b/tests/e2e/utils/killEmulator.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + return execAsync(`adb emu kill`); +}; diff --git a/tests/e2e/utils/listEmulators.js b/tests/e2e/utils/listEmulators.js new file mode 100644 index 000000000000..32a637a41264 --- /dev/null +++ b/tests/e2e/utils/listEmulators.js @@ -0,0 +1,9 @@ +const execAsync = require('./execAsync'); + +module.exports = function (platform = 'android') { + if (platform !== 'android') { + throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`); + } + + return execAsync(`emulator -list-avds`); +}; diff --git a/tests/e2e/utils/rebootEmulator.js b/tests/e2e/utils/rebootEmulator.js new file mode 100644 index 000000000000..b88f579e9a38 --- /dev/null +++ b/tests/e2e/utils/rebootEmulator.js @@ -0,0 +1,10 @@ +const execAsync = require('./execAsync'); + +module.exports = function (platform = 'android') { + if (platform !== 'android') { + throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`); + } + + // Use adb to kill the app + return execAsync(`adb -e reboot`); +}; diff --git a/tests/e2e/utils/sleep.js b/tests/e2e/utils/sleep.js new file mode 100644 index 000000000000..b73cf4947448 --- /dev/null +++ b/tests/e2e/utils/sleep.js @@ -0,0 +1,6 @@ +module.exports = function (ms) { + // Use adb to kill the app + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +}; diff --git a/tests/e2e/utils/startEmulator.js b/tests/e2e/utils/startEmulator.js new file mode 100644 index 000000000000..fa51a0db60ee --- /dev/null +++ b/tests/e2e/utils/startEmulator.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function (emulatorName) { + return execAsync(`emulator @${emulatorName} -no-snapstorage > /dev/null 2>&1 &`); +};