diff --git a/.github/actions/e2e-tests/action.yml b/.github/actions/e2e-tests/action.yml index 3c6cd53110b6..6f8f7f386f4b 100644 --- a/.github/actions/e2e-tests/action.yml +++ b/.github/actions/e2e-tests/action.yml @@ -5,9 +5,6 @@ inputs: e2e_test_token: description: The authentication token used for E2E test endpoints. required: true - slack_token: - description: The slack authentication token. - required: true environment: description: The value of the ENV environment variable to use for npm run env. required: true @@ -48,6 +45,13 @@ runs: lsof -ti:3000 | xargs kill -9 || true env: E2E_TEST_TOKEN: ${{ inputs.e2e_test_token }} - SLACK_TOKEN: ${{ inputs.slack_token }} ENV: ${{ inputs.environment == 'production' && 'prod' || inputs.environment }} - GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Upload failed test videos + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-failed-test-videos-${{ inputs.environment }}-${{ github.run_attempt }} + path: frontend/reports/screen-captures/*.mp4 + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/.reusable-deploy-ecs.yml b/.github/workflows/.reusable-deploy-ecs.yml index 3a9d9836b8b7..5ae843dbd9ce 100644 --- a/.github/workflows/.reusable-deploy-ecs.yml +++ b/.github/workflows/.reusable-deploy-ecs.yml @@ -134,5 +134,4 @@ jobs: uses: ./.github/actions/e2e-tests with: e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} - slack_token: ${{ secrets.SLACK_TOKEN }} environment: ${{ inputs.environment }} diff --git a/.github/workflows/.reusable-docker-e2e-tests.yml b/.github/workflows/.reusable-docker-e2e-tests.yml index 21a5b6a5db88..c7950ad821f0 100644 --- a/.github/workflows/.reusable-docker-e2e-tests.yml +++ b/.github/workflows/.reusable-docker-e2e-tests.yml @@ -31,9 +31,6 @@ on: GCR_TOKEN: description: A token to use for logging into Github Container Registry. If not provided, login does not occur. required: false - SLACK_TOKEN: - description: A token to use uploading test failures to slack. - required: false jobs: run-e2e: @@ -84,5 +81,12 @@ jobs: API_IMAGE: ${{ inputs.api-image }} E2E_IMAGE: ${{ inputs.e2e-image }} E2E_CONCURRENCY: ${{ inputs.concurrency }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Upload failed test videos + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-failed-test-videos-docker-${{ github.run_attempt }} + path: frontend/e2e-videos/*.mp4 + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/frontend-deploy-production.yml b/.github/workflows/frontend-deploy-production.yml index 56dce125277c..35bbe95727ce 100644 --- a/.github/workflows/frontend-deploy-production.yml +++ b/.github/workflows/frontend-deploy-production.yml @@ -40,7 +40,6 @@ jobs: uses: ./.github/actions/e2e-tests with: e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} - slack_token: ${{ secrets.SLACK_TOKEN }} environment: prod deploy-production: diff --git a/.github/workflows/frontend-test-staging.yml b/.github/workflows/frontend-test-staging.yml index 89af5d287275..114112a9c16f 100644 --- a/.github/workflows/frontend-test-staging.yml +++ b/.github/workflows/frontend-test-staging.yml @@ -35,5 +35,4 @@ jobs: uses: ./.github/actions/e2e-tests with: e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} - slack_token: ${{ secrets.SLACK_TOKEN }} environment: staging diff --git a/.github/workflows/manual-e2e-tests.yml b/.github/workflows/manual-e2e-tests.yml index 32c5626c79a5..fbb0c1853ab2 100644 --- a/.github/workflows/manual-e2e-tests.yml +++ b/.github/workflows/manual-e2e-tests.yml @@ -9,7 +9,7 @@ on: api-url: description: An API URL to run the E2E tests against default: 'https://api.flagsmith.com/api/v1/' - + jobs: run-e2e-tests: runs-on: depot-ubuntu-latest @@ -31,3 +31,12 @@ jobs: npm ci npm run env npm run test + + - name: Upload failed test videos + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-failed-test-videos-manual-${{ github.run_attempt }} + path: frontend/reports/screen-captures/*.mp4 + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/platform-docker-build-test-publish.yml b/.github/workflows/platform-docker-build-test-publish.yml index 5a273762636f..c0f1b82cf7db 100644 --- a/.github/workflows/platform-docker-build-test-publish.yml +++ b/.github/workflows/platform-docker-build-test-publish.yml @@ -88,7 +88,6 @@ jobs: args: ${{ matrix.args.args }} secrets: GCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} strategy: matrix: diff --git a/.github/workflows/platform-pull-request.yml b/.github/workflows/platform-pull-request.yml index 8edee7545fd2..e36f1be14bb1 100644 --- a/.github/workflows/platform-pull-request.yml +++ b/.github/workflows/platform-pull-request.yml @@ -154,7 +154,6 @@ jobs: args: --meta-filter category=oss secrets: GCR_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.GITHUB_TOKEN || '' }} - SLACK_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.SLACK_TOKEN || '' }} strategy: matrix: @@ -171,7 +170,6 @@ jobs: args: --meta-filter category=oss,category=enterprise secrets: GCR_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.GITHUB_TOKEN || '' }} - SLACK_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.SLACK_TOKEN || '' }} strategy: matrix: diff --git a/frontend/.testcaferc.js b/frontend/.testcaferc.js index 43d0102f3722..194cf43dcfca 100644 --- a/frontend/.testcaferc.js +++ b/frontend/.testcaferc.js @@ -1,4 +1,6 @@ -const isDev = process.env.E2E_DEV; +const environment = process.env.ENV || 'dev'; +const timestamp = new Date().toISOString().slice(0, 19).replace(/[:.]/g, '-'); + module.exports = { "browsers": "firefox:headless", "port1": 8080, @@ -13,7 +15,7 @@ module.exports = { "videoOptions": { "singleFile": true, "failedOnly": true, - "pathPattern": "./test-report-${FILE_INDEX}.mp4" + "pathPattern": environment + "-" + timestamp + "-test-${FILE_INDEX}.mp4" }, "videoEncodingOptions": { "r": 20, diff --git a/frontend/Makefile b/frontend/Makefile index f326515322d6..ea4277b9a47d 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -28,6 +28,11 @@ serve: .PHONY: test test: - docker compose run frontend \ - npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} npm run test -- $(opts) \ - || (docker compose logs flagsmith-api; exit 1) + mkdir -p e2e-videos + docker rm -f e2e-test-runner 2>/dev/null || true + docker compose run --rm=false --name e2e-test-runner frontend \ + npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} npm run test -- $(opts); \ + TEST_EXIT_CODE=$$?; \ + docker cp e2e-test-runner:/srv/flagsmith/reports/screen-captures/. e2e-videos/ 2>/dev/null || true; \ + docker rm e2e-test-runner 2>/dev/null || true; \ + if [ $$TEST_EXIT_CODE -ne 0 ]; then docker compose logs flagsmith-api; exit $$TEST_EXIT_CODE; fi diff --git a/frontend/bin/upload-file.js b/frontend/bin/upload-file.js deleted file mode 100644 index 95f62d490bd8..000000000000 --- a/frontend/bin/upload-file.js +++ /dev/null @@ -1,31 +0,0 @@ -require('dotenv').config() -const { WebClient } = require('@slack/web-api') -const fs = require('fs') - -const SLACK_TOKEN = process.env.SLACK_TOKEN - -module.exports = function uploadFile(path) { - if (!SLACK_TOKEN) { - // eslint-disable-next-line - console.log('Slack token not specified, skipping upload'); - return - } - - const title = 'Test Run' // Optional - const epoch = new Date().valueOf() - const filename = `e2e-record-${epoch}.mp4` - const channelId = 'C0102JZRG3G' // infra_tests channel ID - // eslint-disable-next-line - console.log(`Uploading ${path}`); - - const slackClient = new WebClient(SLACK_TOKEN) - - // Call the files.upload method using the WebClient - return slackClient.files.uploadV2({ - channel_id: channelId, - file: fs.createReadStream(path), - filename, - initial_comment: `✖ ${title} ${process.env.GITHUB_ACTION_URL || ''}`, - }) -} -new Date().valueOf() diff --git a/frontend/docker-compose-e2e-tests.yml b/frontend/docker-compose-e2e-tests.yml index 4dd26cc6ff63..d3fa9734ec37 100644 --- a/frontend/docker-compose-e2e-tests.yml +++ b/frontend/docker-compose-e2e-tests.yml @@ -47,14 +47,11 @@ services: E2E_TEST_TOKEN_DEV: some-token DISABLE_ANALYTICS_FEATURES: 'true' FLAGSMITH_API: flagsmith-api:8000/api/v1/ - SLACK_TOKEN: ${SLACK_TOKEN} - GITHUB_ACTION_URL: ${GITHUB_ACTION_URL} ports: - 3000:3000 depends_on: flagsmith-api: condition: service_healthy - links: - flagsmith-api:flagsmith-api command: [npm, run, test] diff --git a/frontend/e2e/README.md b/frontend/e2e/README.md new file mode 100644 index 000000000000..d966f452a44b --- /dev/null +++ b/frontend/e2e/README.md @@ -0,0 +1,48 @@ +# E2E Test Video Recording + +This document explains how to access video recordings of failed E2E tests from GitHub Actions. + +## Overview + +The E2E tests are configured to automatically record videos of **failed tests only** and upload them as GitHub Actions artifacts. + +## How it works + +**TestCafe Configuration**: The `.testcaferc.js` file is configured with: +- `failedOnly: true` - Only records videos when tests fail. +- Videos are saved to `reports/screen-captures/`. +- Files are named with environment, timestamp, and test index for easy identification. + +**GitHub Actions Artifacts**: +- Videos are automatically uploaded as artifacts when tests fail. +- Artifacts are retained for 30 days. +- Different workflows use different artifact names: + - `e2e-failed-test-videos-{environment}-{run_attempt}` for direct e2e-tests action + - `e2e-failed-test-videos-docker-{run_attempt}` for Docker-based tests + - `e2e-failed-test-videos-manual-{run_attempt}` for manual test runs + +## Accessing Video Recordings + +1. Go to the failed GitHub Actions run. +2. Scroll down to the "Artifacts" section at the bottom of the run page. +3. Download the video artifact(s) to view the recordings of failed tests. + +## Configuration + +The video recording behavior is controlled by: +- TestCafe configuration in `.testcaferc.js` - controls video quality, path, and naming +- GitHub Actions workflow files - handle the artifact upload + +## Video Settings + +- **Format**: MP4 +- **Frame rate**: 20 FPS +- **Aspect ratio**: 4:3 +- **Recording**: Only when tests fail +- **GitHub Actions Retention**: 30 days + +## Troubleshooting + +- If no videos appear in artifacts, check that tests actually failed (videos are only recorded on failure). +- For Docker-based tests, ensure the video extraction step completed successfully. +- Check the test logs for any video-related error messages. diff --git a/frontend/e2e/index.cafe.js b/frontend/e2e/index.cafe.js index cc83f2fb479e..53b9bef6ad13 100644 --- a/frontend/e2e/index.cafe.js +++ b/frontend/e2e/index.cafe.js @@ -3,7 +3,6 @@ const fs = require('fs'); const path = require('path'); const { fork } = require('child_process'); const _options = require("../.testcaferc") -const upload = require('../bin/upload-file'); const minimist = require('minimist'); const options = { ..._options, @@ -66,15 +65,10 @@ createTestCafe() .run(options) }) .then(async (v) => { - // Upload files console.log(`Test failures ${v} in ${Date.now().valueOf() - start}ms`); - if (fs.existsSync(dir) && !process.env.E2E_DEV) { - try { - const files = fs.readdirSync(dir); - await Promise.all(files.map(f => upload(path.join(dir, f)))); - } catch (e) { console.log('error uploading files', e); } - } else { - console.log('No files to upload'); + if (fs.existsSync(dir)) { + const files = fs.readdirSync(dir); + console.log(`${files.length} video file(s) saved for GitHub Actions artifacts`); } // Shut down server and testcafe server.kill('SIGINT');