From df22788e4556f3261f517055d3f67da067256430 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 13:15:37 +0000 Subject: [PATCH 1/2] ci: shard e2e full suite across 6 runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add 6-way vitest sharding to the cdk-source matrix (2 → 12 parallel runners) - Isolate import test resources per run via RESOURCE_SUFFIX to prevent concurrent conflicts --- .github/workflows/e2e-tests-full.yml | 5 +++-- e2e-tests/fixtures/import/cleanup_resources.py | 8 ++++++-- e2e-tests/fixtures/import/common.py | 6 +++++- e2e-tests/import-gateway.test.ts | 4 +++- e2e-tests/import-resources.test.ts | 4 +++- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 14809a587..e1ff4acab 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -27,6 +27,7 @@ jobs: fail-fast: false matrix: cdk-source: [npm, main] + shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] steps: - uses: actions/checkout@v6 with: @@ -70,7 +71,7 @@ jobs: CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}) + - name: Run E2E tests (${{ matrix.cdk-source }}, shard ${{ matrix.shard }}) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -78,7 +79,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - run: npm run test:e2e + run: npx vitest run --project e2e --shard=${{ matrix.shard }} browser-tests: runs-on: ubuntu-latest environment: e2e-testing diff --git a/e2e-tests/fixtures/import/cleanup_resources.py b/e2e-tests/fixtures/import/cleanup_resources.py index 0728b711e..c74da3e67 100644 --- a/e2e-tests/fixtures/import/cleanup_resources.py +++ b/e2e-tests/fixtures/import/cleanup_resources.py @@ -22,8 +22,12 @@ def cleanup_s3_code_objects(): account_id = get_account_id() bucket_name = f"bugbash-agentcore-code-{account_id}-{REGION}" s3 = boto3.client("s3", region_name=REGION) + prefix = f"bugbash-{RESOURCE_SUFFIX}/" if RESOURCE_SUFFIX else "" try: - resp = s3.list_objects_v2(Bucket=bucket_name) + list_args = {"Bucket": bucket_name} + if prefix: + list_args["Prefix"] = prefix + resp = s3.list_objects_v2(**list_args) objects = resp.get("Contents", []) if not objects: return @@ -31,7 +35,7 @@ def cleanup_s3_code_objects(): Bucket=bucket_name, Delete={"Objects": [{"Key": o["Key"]} for o in objects]}, ) - print(f"Deleted {len(objects)} object(s) from s3://{bucket_name}") + print(f"Deleted {len(objects)} object(s) from s3://{bucket_name}/{prefix}") except Exception as e: print(f"Could not clean up S3 objects: {e}") diff --git a/e2e-tests/fixtures/import/common.py b/e2e-tests/fixtures/import/common.py index 369ec0bb0..c786c0c01 100644 --- a/e2e-tests/fixtures/import/common.py +++ b/e2e-tests/fixtures/import/common.py @@ -8,9 +8,11 @@ import boto3 REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" +RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "") SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) APP_DIR = os.path.join(SCRIPT_DIR, "app") -RESOURCES_FILE = os.path.join(SCRIPT_DIR, "bugbash-resources.json") +_resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json" +RESOURCES_FILE = os.path.join(SCRIPT_DIR, _resources_name) INLINE_POLICY_NAME = "bugbash-agentcore-permissions" @@ -35,6 +37,8 @@ def upload_code(prefix="bugbash"): """Zip APP_DIR and upload to S3. Returns (bucket, s3_key).""" bucket_name = get_code_bucket() s3 = boto3.client("s3", region_name=REGION) + if RESOURCE_SUFFIX: + prefix = f"{prefix}-{RESOURCE_SUFFIX}" # Create zip of app directory with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: diff --git a/e2e-tests/import-gateway.test.ts b/e2e-tests/import-gateway.test.ts index 2aea04f02..fd80ae967 100644 --- a/e2e-tests/import-gateway.test.ts +++ b/e2e-tests/import-gateway.test.ts @@ -43,6 +43,7 @@ describe.sequential('e2e: import gateway', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'setup_gateway.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -50,7 +51,7 @@ describe.sequential('e2e: import gateway', () => { ); } - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; gatewayArn = resources.gateway!.arn; @@ -80,6 +81,7 @@ describe.sequential('e2e: import gateway', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ diff --git a/e2e-tests/import-resources.test.ts b/e2e-tests/import-resources.test.ts index d51cbffac..72d9c253a 100644 --- a/e2e-tests/import-resources.test.ts +++ b/e2e-tests/import-resources.test.ts @@ -54,6 +54,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', script], fixtureDir, { AWS_REGION: region, DEFAULT_EVALUATOR_MODEL, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -63,7 +64,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { } // 2. Read resource ARNs from bugbash-resources.json - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; runtimeArn = resources['runtime-basic']!.arn; memoryArn = resources['memory-full']!.arn; @@ -102,6 +103,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ From 28c60db9aa378bcae815f5da7f1e3603c5b5c281 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 15:46:26 +0000 Subject: [PATCH 2/2] fix: import RESOURCE_SUFFIX in cleanup_resources.py --- e2e-tests/fixtures/import/cleanup_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/fixtures/import/cleanup_resources.py b/e2e-tests/fixtures/import/cleanup_resources.py index c74da3e67..429a36a1f 100644 --- a/e2e-tests/fixtures/import/cleanup_resources.py +++ b/e2e-tests/fixtures/import/cleanup_resources.py @@ -12,7 +12,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from common import REGION, RESOURCES_FILE, get_control_client, get_account_id +from common import REGION, RESOURCE_SUFFIX, RESOURCES_FILE, get_control_client, get_account_id import boto3