From fb0b3dc86d15e09c3a202d63ba73406108446513 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 17:53:04 +0800 Subject: [PATCH 01/22] Optimize GitHub Actions workflow --- .github/workflows/pipeline.yaml | 199 ++++++++++++++++++++++++++------ Makefile | 6 +- template-arm64.yaml | 19 ++- template-x86_64.yaml | 21 +++- 4 files changed, 201 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 05df10d5..38727235 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -4,9 +4,15 @@ on: push: branches: - main + paths-ignore: + - "docs" + - "examples" pull_request: branches: - main + paths-ignore: + - "docs" + - "examples" release: types: - released @@ -31,20 +37,38 @@ env: PROD_IMAGE_REPOSITORY: 373534280245.dkr.ecr.us-east-1.amazonaws.com/aws-sam-cli-managed-prod-ecr-pipeline-resources-imagerepository-fhpoty0tapro PROD_ECR_REGION: us-east-1 + RUST_BACKTRACE: full + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + CARGO_INCREMENTAL: 0 + jobs: test: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable with: - toolchain: stable components: clippy, rustfmt + targets: x86_64-unknown-linux-musl,aarch64-unknown-linux-musl + + - uses: korandoru/setup-zig@v1 + with: + zig-version: 0.11.0 + + - name: Configure cache + uses: mozilla-actions/sccache-action@v0.0.3 + - uses: taiki-e/install-action@nextest + - name: linting run: | - cargo fmt -- --check + cargo fmt --all -- --check cargo clippy -- -Dwarnings + - name: run unit and integration tests run: cargo nextest run --profile ci @@ -54,13 +78,31 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.8" + - uses: aws-actions/setup-sam@v2 + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-unknown-linux-musl,aarch64-unknown-linux-musl + + - uses: korandoru/setup-zig@v1 + with: + zig-version: 0.11.0 + + - name: Configure cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - name: Build x86_64 Layer - run: sam build --template ${SAM_TEMPLATE_X86_64} -b build-x86_64 + run: sam build --template ${SAM_TEMPLATE_X86_64} --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} -b build-x86_64 - name: Tar files run: tar -cvf build-x86_64.tar build-x86_64 @@ -71,7 +113,7 @@ jobs: path: build-x86_64.tar - name: Build arm64 Layer - run: sam build --template ${SAM_TEMPLATE_ARM64} -b build-arm64 + run: sam build --template ${SAM_TEMPLATE_ARM64} --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} -b build-arm64 - name: Tar files run: tar -cvf build-arm64.tar build-arm64 @@ -149,13 +191,13 @@ jobs: - name: Create and push the x86_64 docker image to beta ecr repo run: | - tar -c -C build-x86_64/LambdaAdapterLayerX86/extensions . | docker import - 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-x86_64 + tar -c -C build-x86_64/LambdaAdapterLayerX86/extensions . | docker import --platform linux/amd64 - 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-x86_64 aws ecr get-login-password --region ap-northeast-1 | docker login --username AWS --password-stdin 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com docker push 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-x86_64 - name: Create and push the arm64 docker image to beta ecr repo run: | - tar -c -C build-arm64/LambdaAdapterLayerArm64/extensions . | docker import - 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-aarch64 + tar -c -C build-arm64/LambdaAdapterLayerArm64/extensions . | docker import --platform linux/arm64 - 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-aarch64 aws ecr get-login-password --region ap-northeast-1 | docker login --username AWS --password-stdin 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com docker push 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-aarch64 @@ -164,8 +206,6 @@ jobs: docker manifest create 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest \ 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-x86_64 \ 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-aarch64 - docker manifest annotate --arch arm64 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest \ - 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest-aarch64 docker manifest push 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest deploy-beta: @@ -198,6 +238,7 @@ jobs: run: | sam deploy --stack-name ${BETA_STACK_NAME}-x86 \ --template packaged-beta-x86_64.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${BETA_REGION} \ --s3-bucket ${BETA_ARTIFACTS_BUCKET} \ @@ -213,6 +254,7 @@ jobs: run: | sam deploy --stack-name ${BETA_STACK_NAME}-arm64 \ --template packaged-beta-arm64.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${BETA_REGION} \ --s3-bucket ${BETA_ARTIFACTS_BUCKET} \ @@ -220,21 +262,32 @@ jobs: --no-fail-on-empty-changeset \ --role-arn ${BETA_CLOUDFORMATION_EXECUTION_ROLE} - e2e-test: + e2e-test-zip: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [deploy-beta] runs-on: ubuntu-20.04 steps: - - uses: dtolnay/rust-toolchain@master + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable with: - toolchain: stable components: clippy, rustfmt + targets: x86_64-unknown-linux-musl + + - uses: korandoru/setup-zig@v1 + with: + zig-version: 0.11.0 + + - name: Configure cache + uses: mozilla-actions/sccache-action@v0.0.3 + - uses: taiki-e/install-action@nextest - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.8" + - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role @@ -248,12 +301,11 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true - - name: deploy the oci x86 integration test stacks for the beta environment - working-directory: ./tests/e2e_tests/fixtures/go-httpbin + - name: deploy the zip x86 integration test stacks for the beta environment + working-directory: ./tests/e2e_tests/fixtures/go-httpbin-zip run: | - aws ecr get-login-password --region ap-northeast-1 | docker login --username AWS --password-stdin 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com sam build - sam deploy --stack-name ${BETA_STACK_NAME}-oci-x86 \ + sam deploy --stack-name ${BETA_STACK_NAME}-zip-x86 \ --capabilities CAPABILITY_IAM \ --region ${BETA_REGION} \ --s3-bucket ${BETA_ARTIFACTS_BUCKET} \ @@ -261,11 +313,70 @@ jobs: --no-fail-on-empty-changeset \ --role-arn ${BETA_CLOUDFORMATION_EXECUTION_ROLE} - - name: deploy the zip x86 integration test stacks for the beta environment + - uses: dkershner6/aws-ssm-getparameters-action@v1 + with: + parameterPairs: "/lambda-web-adapter/e2e/httpbin/zip/rest-api-endpoint = HTTPBIN_ZIP_REST_ENDPOINT, + /lambda-web-adapter/e2e/httpbin/zip/http-api-endpoint = HTTPBIN_ZIP_HTTP_ENDPOINT, + /lambda-web-adapter/e2e/httpbin/zip/alb-endpoint = HTTPBIN_ZIP_ALB_ENDPOINT, + /lambda-web-adapter/e2e/httpbin/zip/function-url = HTTPBIN_ZIP_FURL_ENDPOINT" + + - name: run e2e tests + run: | + API_ENDPOINT=${HTTPBIN_ZIP_REST_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci + API_ENDPOINT=${HTTPBIN_ZIP_HTTP_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci + API_ENDPOINT=${HTTPBIN_ZIP_ALB_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci + API_ENDPOINT=${HTTPBIN_ZIP_FURL_ENDPOINT} API_AUTH_TYPE="iam" cargo nextest run --run-ignored ignored-only --profile ci + + - name: remove the zip x86 integration test stacks working-directory: ./tests/e2e_tests/fixtures/go-httpbin-zip run: | + sam delete --no-prompts --region ${BETA_REGION} --stack-name ${BETA_STACK_NAME}-zip-x86 + + e2e-test-oci: + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} + needs: [deploy-beta] + runs-on: ubuntu-20.04 + steps: + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + targets: x86_64-unknown-linux-musl + + - uses: korandoru/setup-zig@v1 + with: + zig-version: 0.11.0 + + - name: Configure cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - uses: taiki-e/install-action@nextest + + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - uses: aws-actions/setup-sam@v2 + + - name: Assume the beta pipeline user role + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} + role-session-name: beta-deployment + role-duration-seconds: 3600 + role-skip-session-tagging: true + + - name: deploy the oci x86 integration test stacks for the beta environment + working-directory: ./tests/e2e_tests/fixtures/go-httpbin + run: | + aws ecr get-login-password --region ap-northeast-1 | docker login --username AWS --password-stdin 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com sam build - sam deploy --stack-name ${BETA_STACK_NAME}-zip-x86 \ + sam deploy --stack-name ${BETA_STACK_NAME}-oci-x86 \ --capabilities CAPABILITY_IAM \ --region ${BETA_REGION} \ --s3-bucket ${BETA_ARTIFACTS_BUCKET} \ @@ -278,11 +389,7 @@ jobs: parameterPairs: "/lambda-web-adapter/e2e/httpbin/oci/rest-api-endpoint = HTTPBIN_OCI_REST_ENDPOINT, /lambda-web-adapter/e2e/httpbin/oci/http-api-endpoint = HTTPBIN_OCI_HTTP_ENDPOINT, /lambda-web-adapter/e2e/httpbin/oci/alb-endpoint = HTTPBIN_OCI_ALB_ENDPOINT, - /lambda-web-adapter/e2e/httpbin/oci/function-url = HTTPBIN_OCI_FURL_ENDPOINT, - /lambda-web-adapter/e2e/httpbin/zip/rest-api-endpoint = HTTPBIN_ZIP_REST_ENDPOINT, - /lambda-web-adapter/e2e/httpbin/zip/http-api-endpoint = HTTPBIN_ZIP_HTTP_ENDPOINT, - /lambda-web-adapter/e2e/httpbin/zip/alb-endpoint = HTTPBIN_ZIP_ALB_ENDPOINT, - /lambda-web-adapter/e2e/httpbin/zip/function-url = HTTPBIN_ZIP_FURL_ENDPOINT" + /lambda-web-adapter/e2e/httpbin/oci/function-url = HTTPBIN_OCI_FURL_ENDPOINT" - name: run e2e tests run: | @@ -290,25 +397,16 @@ jobs: API_ENDPOINT=${HTTPBIN_OCI_HTTP_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci API_ENDPOINT=${HTTPBIN_OCI_ALB_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci API_ENDPOINT=${HTTPBIN_OCI_FURL_ENDPOINT} API_AUTH_TYPE="iam" cargo nextest run --run-ignored ignored-only --profile ci - API_ENDPOINT=${HTTPBIN_ZIP_REST_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci - API_ENDPOINT=${HTTPBIN_ZIP_HTTP_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci - API_ENDPOINT=${HTTPBIN_ZIP_ALB_ENDPOINT} API_AUTH_TYPE="open" cargo nextest run --run-ignored ignored-only --profile ci - API_ENDPOINT=${HTTPBIN_ZIP_FURL_ENDPOINT} API_AUTH_TYPE="iam" cargo nextest run --run-ignored ignored-only --profile ci - name: remove the oci x86 integration test stacks working-directory: ./tests/e2e_tests/fixtures/go-httpbin run: | sam delete --no-prompts --region ${BETA_REGION} --stack-name ${BETA_STACK_NAME}-oci-x86 - - name: remove the zip x86 integration test stacks - working-directory: ./tests/e2e_tests/fixtures/go-httpbin-zip - run: | - sam delete --no-prompts --region ${BETA_REGION} --stack-name ${BETA_STACK_NAME}-zip-x86 - load-gamma-matrix: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test ] + needs: [ e2e-test-zip, e2e-test-oci] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -914,10 +1012,35 @@ jobs: run: | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws - - name: build OCI images for x86_64 and aarch64 + - uses: actions/download-artifact@v3 + with: + name: aws-sam-build-x86_64 + + - name: extract build_x86_64 + run: | + tar -xvf build-x86_64.tar + + - uses: actions/download-artifact@v3 + with: + name: aws-sam-build-arm64 + + - name: extract build_arm64 + run: | + tar -xvf build-arm64.tar + + - name: Create and push the x86_64 docker image to prod ecr public repo + run: | + tar -c -C build-x86_64/LambdaAdapterLayerX86/extensions . | docker import --platform linux/amd64 - public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-x86_64 + docker push public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-x86_64 + + - name: Create and push the arm64 docker image to prod ecr public repo run: | - make build + tar -c -C build-arm64/LambdaAdapterLayerArm64/extensions . | docker import --platform linux/arm64 - public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-aarch64 + docker push public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-aarch64 - - name: publish OCI images to ECR public repository + - name: create and push the multi-arch manifest to prod ecr public repo run: | - make publish + docker manifest create public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION) \ + public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-x86_64 \ + public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION)-aarch64 + docker manifest push public.ecr.aws/awsguru/aws-lambda-adapter:$(CARGO_PKG_VERSION) \ No newline at end of file diff --git a/Makefile b/Makefile index ba7127eb..0cabf0ad 100644 --- a/Makefile +++ b/Makefile @@ -45,11 +45,13 @@ build-mac: build-LambdaAdapterLayerX86: cp layer/* $(ARTIFACTS_DIR)/ - DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --build-arg TARGET_PLATFORM=linux/amd64 --build-arg ARCH=x86_64 -o $(ARTIFACTS_DIR)/extensions . + LAMBDA_RUNTIME_USER_AGENT=aws-lambda-rust/aws-lambda-adapter/$(CARGO_PKG_VERSION) \ + cargo lambda build --release --extension --target x86_64-unknown-linux-musl --lambda-dir $(ARTIFACTS_DIR) build-LambdaAdapterLayerArm64: cp layer/* $(ARTIFACTS_DIR)/ - DOCKER_BUILDKIT=1 docker build --platform linux/arm64 --build-arg TARGET_PLATFORM=linux/arm64 --build-arg ARCH=aarch64 -o $(ARTIFACTS_DIR)/extensions . + LAMBDA_RUNTIME_USER_AGENT=aws-lambda-rust/aws-lambda-adapter/$(CARGO_PKG_VERSION) \ + cargo lambda build --release --extension --target aarch64-unknown-linux-musl --lambda-dir $(ARTIFACTS_DIR) fmt: cargo fmt --all \ No newline at end of file diff --git a/template-arm64.yaml b/template-arm64.yaml index ffe77565..9b0af7b3 100644 --- a/template-arm64.yaml +++ b/template-arm64.yaml @@ -3,6 +3,11 @@ Transform: AWS::Serverless-2016-10-31 Description: > AWS Lambda Adapter (Arm64) - Run web applications on AWS Lambda +Parameters: + CargoPkgVersion: + Type: String + Default: latest + Resources: LambdaAdapterLayerArm64: Type: AWS::Serverless::LayerVersion @@ -10,7 +15,7 @@ Resources: ContentUri: . CompatibleArchitectures: - arm64 - Description: 'Layer for AWS Lambda Adapter arm64' + Description: !Sub 'Layer for AWS Lambda Adapter arm64: ${CargoPkgVersion}' LicenseInfo: 'Available under the Apache-2.0 license.' RetentionPolicy: Retain Metadata: @@ -30,7 +35,17 @@ Resources: Type: AWS::SSM::Parameter Properties: Name: /lambda-web-adapter/layer/arm64/latest - Description: 'Layer ARN for the latest Lambda Web Adapter Arm64 Layer' + Description: 'Layer ARN for the latest Lambda Web Adapter Arm64 Layer: latest' + Type: String + Value: !Ref LambdaAdapterLayerArm64 + + LambdaAdapterLayerArm64VersionParameter: + Type: AWS::SSM::Parameter + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + Name: !Sub '/lambda-web-adapter/layer/arm64/${CargoPkgVersion}' + Description: !Sub 'Layer ARN for the latest Lambda Web Adapter Arm64 Layer: ${CargoPkgVersion}' Type: String Value: !Ref LambdaAdapterLayerArm64 diff --git a/template-x86_64.yaml b/template-x86_64.yaml index 4027d84a..2fa465ba 100644 --- a/template-x86_64.yaml +++ b/template-x86_64.yaml @@ -3,12 +3,19 @@ Transform: AWS::Serverless-2016-10-31 Description: > AWS Lambda Adapter (x86_64) - Run web applications on AWS Lambda +Parameters: + CargoPkgVersion: + Type: String + Default: latest + Resources: LambdaAdapterLayerX86: Type: AWS::Serverless::LayerVersion Properties: ContentUri: . - Description: 'Layer for AWS Lambda Adapter x86_64' + CompatibleArchitectures: + - x86_64 + Description: !Sub 'Layer for AWS Lambda Adapter x86_64: ${CargoPkgVersion}' LicenseInfo: 'Available under the Apache-2.0 license.' RetentionPolicy: Retain Metadata: @@ -28,7 +35,17 @@ Resources: Type: AWS::SSM::Parameter Properties: Name: /lambda-web-adapter/layer/x86_64/latest - Description: 'Layer ARN for the latest Lambda Web Adapter X86_64 Layer' + Description: 'Layer ARN for the latest Lambda Web Adapter X86_64 Layer: latest' + Type: String + Value: !Ref LambdaAdapterLayerX86 + + LambdaAdapterLayerArm64VersionParameter: + Type: AWS::SSM::Parameter + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + Name: !Sub '/lambda-web-adapter/layer/x86_64/${CargoPkgVersion}' + Description: !Sub 'Layer ARN for the latest Lambda Web Adapter X86_64 Layer: ${CargoPkgVersion}' Type: String Value: !Ref LambdaAdapterLayerX86 From 2e7c3f4748ef30295d0df5656645510515410145 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 18:01:46 +0800 Subject: [PATCH 02/22] Update the pipeline.yaml --- .github/workflows/pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 38727235..542d1740 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -417,7 +417,7 @@ jobs: load-prod-matrix: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test ] + needs: [ e2e-test-zip, e2e-test-oci ] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -428,7 +428,7 @@ jobs: load-china-prod-matrix: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test ] + needs: [ e2e-test-zip, e2e-test-oci ] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -439,7 +439,7 @@ jobs: load-china-gamma-matrix: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test ] + needs: [ e2e-test-zip, e2e-test-oci ] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From caccd6910fa1539f169b853f901fdd3154a356f8 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 18:04:19 +0800 Subject: [PATCH 03/22] Update the pipeline.yaml --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 542d1740..cffb1618 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -723,7 +723,7 @@ jobs: load-gamma-matrix2: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test, package-gamma ] + needs: [ e2e-test-zip, e2e-test-oci, package-gamma ] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From f6f73eec0b8a9c3f4e4628aae48aaf76bb7760cf Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 18:05:55 +0800 Subject: [PATCH 04/22] Update the pipeline.yaml --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index cffb1618..5b1a5a6a 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -856,7 +856,7 @@ jobs: load-china-gamma-matrix2: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} - needs: [ e2e-test, package-china-gamma] + needs: [ e2e-test-zip, e2e-test-oci, package-china-gamma] runs-on: ubuntu-20.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From 591117dbe04a58c9f5bda2d6da6666f8b0e5372b Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 18:13:13 +0800 Subject: [PATCH 05/22] Update the pipeline.yaml --- .github/workflows/pipeline.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 5b1a5a6a..05a1e742 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -97,6 +97,14 @@ jobs: - name: Configure cache uses: mozilla-actions/sccache-action@v0.0.3 + - name: Install Cargo Lambda + uses: jaxxstorm/action-install-gh-release@v1.9.0 + with: + repo: cargo-lambda/cargo-lambda + tag: v1.0.1 + platform: linux + arch: x86_64 + - name: Add cargo pkg version to env vars run: | echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV From 26ed3125860cd639d942155d04c9e5c17223c250 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 18:48:18 +0800 Subject: [PATCH 06/22] Update the pipeline.yaml --- .github/workflows/pipeline.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 05a1e742..0414f779 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -238,6 +238,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - uses: actions/download-artifact@v3 with: name: packaged-beta-x86_64.yaml @@ -764,6 +768,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - uses: actions/download-artifact@v3 with: name: packaged-gamma-x86_64-${{ matrix.region }}.yaml @@ -772,6 +780,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-gamma-x86-${{ matrix.region }} \ --template packaged-gamma-x86_64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -788,6 +797,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-gamma-arm64-${{ matrix.region }} \ --template packaged-gamma-arm64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -831,6 +841,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - uses: actions/download-artifact@v3 with: name: packaged-prod-x86_64-${{ matrix.region }}.yaml @@ -839,6 +853,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-prod-x86-${{ matrix.region }} \ --template packaged-prod-x86_64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -855,6 +870,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-prod-arm64-${{ matrix.region }} \ --template packaged-prod-arm64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -898,6 +914,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - uses: actions/download-artifact@v3 with: name: packaged-china-gamma-x86_64-${{ matrix.region }}.yaml @@ -906,6 +926,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-gamma-x86-${{ matrix.region }} \ --template packaged-china-gamma-x86_64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -922,6 +943,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-gamma-arm64-${{ matrix.region }} \ --template packaged-china-gamma-arm64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -966,6 +988,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - uses: actions/download-artifact@v3 with: name: packaged-china-prod-x86_64-${{ matrix.region }}.yaml @@ -974,6 +1000,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-prod-x86-${{ matrix.region }} \ --template packaged-china-prod-x86_64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -990,6 +1017,7 @@ jobs: run: | sam deploy --stack-name lambda-adapter-prod-arm64-${{ matrix.region }} \ --template packaged-china-prod-arm64-${{ matrix.region }}.yaml \ + --parameter-overrides CargoPkgVersion=${CARGO_PKG_VERSION} \ --capabilities CAPABILITY_IAM \ --region ${{ matrix.region }} \ --s3-bucket ${{ matrix.artifacts_bucket }} \ @@ -1016,6 +1044,10 @@ jobs: role-duration-seconds: 3600 role-skip-session-tagging: true + - name: Add cargo pkg version to env vars + run: | + echo "CARGO_PKG_VERSION=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')" >> $GITHUB_ENV + - name: login ECR Public Registry run: | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws From ff36f326c0a333006526d492a81f071933ad7044 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 19:02:27 +0800 Subject: [PATCH 07/22] Run to e2e for pull requests --- .github/workflows/pipeline.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 0414f779..0a6aa0fb 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -132,7 +132,6 @@ jobs: path: build-arm64.tar package-beta: - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [ build ] runs-on: ubuntu-20.04 steps: @@ -217,7 +216,6 @@ jobs: docker manifest push 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest deploy-beta: - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [package-beta] runs-on: ubuntu-20.04 steps: @@ -275,7 +273,6 @@ jobs: --role-arn ${BETA_CLOUDFORMATION_EXECUTION_ROLE} e2e-test-zip: - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [deploy-beta] runs-on: ubuntu-20.04 steps: @@ -345,7 +342,6 @@ jobs: sam delete --no-prompts --region ${BETA_REGION} --stack-name ${BETA_STACK_NAME}-zip-x86 e2e-test-oci: - if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [deploy-beta] runs-on: ubuntu-20.04 steps: From e62b82c0fff12b3d9944084d63b86315c8cdf6a4 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 19:23:01 +0800 Subject: [PATCH 08/22] Update aws-actions/configure-aws-credentials to v4 --- .github/workflows/pipeline.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 0a6aa0fb..dba9d69d 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -142,7 +142,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -226,7 +226,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -300,7 +300,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -369,7 +369,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -470,7 +470,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the gamma pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -538,7 +538,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the prod pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -606,7 +606,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the china pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} @@ -675,7 +675,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the china pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} @@ -754,7 +754,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the gamma pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -827,7 +827,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the prod pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} @@ -900,7 +900,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the china pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} @@ -974,7 +974,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the china pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} @@ -1030,7 +1030,7 @@ jobs: - uses: actions/checkout@v3 - name: Assume the prod pipeline user role - uses: aws-actions/configure-aws-credentials@v1-node16 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} From cb64c703c8450d56697b7cb270ab18baa7792ce2 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 19:28:25 +0800 Subject: [PATCH 09/22] Update aws-actions/configure-aws-credentials to v4 --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index dba9d69d..4dfc77d4 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -142,7 +142,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@@v4 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} From 218703b91c6b7349d5f866d8462bc22a8d0011e4 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 21:34:15 +0800 Subject: [PATCH 10/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 64 ++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 4dfc77d4..b61495b4 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -19,10 +19,10 @@ on: env: CARGO_TERM_COLOR: always - PIPELINE_USER_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - PIPELINE_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - PIPELINE_CN_USER_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} - PIPELINE_CN_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} +# PIPELINE_USER_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# PIPELINE_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# PIPELINE_CN_USER_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} +# PIPELINE_CN_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} SAM_TEMPLATE_X86_64: template-x86_64.yaml SAM_TEMPLATE_ARM64: template-arm64.yaml BETA_STACK_NAME: lambda-adapter-beta @@ -42,6 +42,10 @@ env: RUSTC_WRAPPER: "sccache" CARGO_INCREMENTAL: 0 +permissions: + id-token: write + contents: read + jobs: test: runs-on: ubuntu-20.04 @@ -144,8 +148,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} role-session-name: beta-packaging @@ -228,8 +232,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} role-session-name: beta-deployment @@ -302,8 +306,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} role-session-name: beta-deployment @@ -472,8 +476,8 @@ jobs: - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: gamma-packaging @@ -540,8 +544,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: prod-packaging @@ -608,8 +612,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} + audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: china-gamma-packaging @@ -677,8 +682,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} + audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: china-prod-packaging @@ -756,8 +762,8 @@ jobs: - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: gamma-deployment @@ -829,8 +835,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: prod-deployment @@ -902,8 +908,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} + audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: china-deployment @@ -976,8 +983,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} + audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} role-session-name: china-deployment @@ -1032,8 +1040,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} +# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.PROD_ECR_REGION }} role-to-assume: ${{ env.PROD_ECR_PIPELINE_EXECUTION_ROLE }} role-session-name: prod-deployment From e0dd7d3523b735fbc694988565a930623a62c1b7 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 21:49:01 +0800 Subject: [PATCH 11/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index b61495b4..f0ade76d 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -17,6 +17,10 @@ on: types: - released +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + env: CARGO_TERM_COLOR: always # PIPELINE_USER_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -42,10 +46,6 @@ env: RUSTC_WRAPPER: "sccache" CARGO_INCREMENTAL: 0 -permissions: - id-token: write - contents: read - jobs: test: runs-on: ubuntu-20.04 From dcf0ac0fa831ebe0ec5982b41abdfc6a472daf71 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 22:03:33 +0800 Subject: [PATCH 12/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 69 --------------------------------- 1 file changed, 69 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f0ade76d..f0afadba 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -23,10 +23,6 @@ permissions: env: CARGO_TERM_COLOR: always -# PIPELINE_USER_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# PIPELINE_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# PIPELINE_CN_USER_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} -# PIPELINE_CN_USER_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} SAM_TEMPLATE_X86_64: template-x86_64.yaml SAM_TEMPLATE_ARM64: template-arm64.yaml BETA_STACK_NAME: lambda-adapter-beta @@ -148,13 +144,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - role-session-name: beta-packaging - role-duration-seconds: 3600 - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -232,13 +223,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - role-session-name: beta-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | @@ -306,13 +292,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - role-session-name: beta-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: deploy the zip x86 integration test stacks for the beta environment working-directory: ./tests/e2e_tests/fixtures/go-httpbin-zip @@ -375,13 +356,8 @@ jobs: - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - role-session-name: beta-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: deploy the oci x86 integration test stacks for the beta environment working-directory: ./tests/e2e_tests/fixtures/go-httpbin @@ -476,13 +452,8 @@ jobs: - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: gamma-packaging - role-duration-seconds: 3600 - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -544,13 +515,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: prod-packaging - role-duration-seconds: 3600 - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -612,14 +578,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: china-gamma-packaging - role-duration-seconds: 3600 - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -682,14 +643,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: china-prod-packaging - role-duration-seconds: 3600 - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -762,13 +718,8 @@ jobs: - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: gamma-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | @@ -835,13 +786,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: prod-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | @@ -908,14 +854,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: china-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | @@ -983,14 +924,9 @@ jobs: - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_CN_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_CN_USER_SECRET_ACCESS_KEY }} audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} - role-session-name: china-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | @@ -1040,13 +976,8 @@ jobs: - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: -# aws-access-key-id: ${{ env.PIPELINE_USER_ACCESS_KEY_ID }} -# aws-secret-access-key: ${{ env.PIPELINE_USER_SECRET_ACCESS_KEY }} aws-region: ${{ env.PROD_ECR_REGION }} role-to-assume: ${{ env.PROD_ECR_PIPELINE_EXECUTION_ROLE }} - role-session-name: prod-deployment - role-duration-seconds: 3600 - role-skip-session-tagging: true - name: Add cargo pkg version to env vars run: | From 00c91320790a68f85a1a24072baff1e3d627b9d5 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 22:14:59 +0800 Subject: [PATCH 13/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f0afadba..e960a877 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -134,6 +134,9 @@ jobs: package-beta: needs: [ build ] runs-on: ubuntu-20.04 + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 From 5bafeacc6efcf85069f21e5957f7e8fcf6ceab5f Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 22:29:24 +0800 Subject: [PATCH 14/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index e960a877..e0fbb63e 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -145,7 +145,7 @@ jobs: - uses: aws-actions/setup-sam@v2 - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v3 with: aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} From a4a4ead4b6a7d134d3ba2dc8c4bf50f3c97c3d82 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Mon, 15 Jan 2024 23:05:23 +0800 Subject: [PATCH 15/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index e0fbb63e..f7f071e6 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -18,13 +18,14 @@ on: - released permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + id-token: write + contents: read env: CARGO_TERM_COLOR: always SAM_TEMPLATE_X86_64: template-x86_64.yaml SAM_TEMPLATE_ARM64: template-arm64.yaml + GITHUB_RUNNER_ROLE: arn:aws:iam::238946506962:role/GitHubRunnerRole BETA_STACK_NAME: lambda-adapter-beta BETA_PIPELINE_EXECUTION_ROLE: arn:aws:iam::477159140107:role/aws-sam-cli-managed-beta-pip-PipelineExecutionRole-13NXRWTRTHDCJ BETA_CLOUDFORMATION_EXECUTION_ROLE: arn:aws:iam::477159140107:role/aws-sam-cli-managed-beta-CloudFormationExecutionR-132I77VBFOWQ2 @@ -144,8 +145,14 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the beta pipeline user role - uses: aws-actions/configure-aws-credentials@v3 + uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} From cc4c7ebcbb5ae6f20b5d6864d0b8819d201ed116 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Tue, 16 Jan 2024 19:16:05 +0800 Subject: [PATCH 16/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f7f071e6..816766a6 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -149,7 +149,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ env.BETA_REGION }} - role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 From b934c4233cb24bc9cfbad26454441fbbd5a8c74f Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Tue, 16 Jan 2024 19:27:51 +0800 Subject: [PATCH 17/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 816766a6..cbc344c7 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -149,11 +149,14 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ env.BETA_REGION }} - role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} From 8e903515b9d12a084d90c585c7d10a98e2c6d489 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Tue, 16 Jan 2024 19:45:40 +0800 Subject: [PATCH 18/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index cbc344c7..d8314c28 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -159,6 +159,7 @@ jobs: aws-session-token: ${{ env.AWS_SESSION_TOKEN }} aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} + role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: From 609f6b6cf02123eda638698a2e484dce8fd8bafd Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Tue, 16 Jan 2024 22:22:51 +0800 Subject: [PATCH 19/22] Use GitHub OIDC provider for auth --- .github/workflows/pipeline.yaml | 124 +++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index d8314c28..eb8682e5 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -26,6 +26,7 @@ env: SAM_TEMPLATE_X86_64: template-x86_64.yaml SAM_TEMPLATE_ARM64: template-arm64.yaml GITHUB_RUNNER_ROLE: arn:aws:iam::238946506962:role/GitHubRunnerRole + GITHUB_RUNNER_CHINA_ROLE: arn:aws-cn:iam::075528433517:role/GitHubRunnerRole BETA_STACK_NAME: lambda-adapter-beta BETA_PIPELINE_EXECUTION_ROLE: arn:aws:iam::477159140107:role/aws-sam-cli-managed-beta-pip-PipelineExecutionRole-13NXRWTRTHDCJ BETA_CLOUDFORMATION_EXECUTION_ROLE: arn:aws:iam::477159140107:role/aws-sam-cli-managed-beta-CloudFormationExecutionR-132I77VBFOWQ2 @@ -157,9 +158,9 @@ jobs: aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} - role-skip-session-tagging: true - uses: actions/download-artifact@v3 with: @@ -234,9 +235,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} @@ -303,9 +314,19 @@ jobs: - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} @@ -367,9 +388,19 @@ jobs: - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the beta pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ env.BETA_REGION }} role-to-assume: ${{ env.BETA_PIPELINE_EXECUTION_ROLE }} @@ -463,9 +494,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -526,9 +567,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -589,9 +640,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -654,9 +715,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -729,9 +800,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the gamma pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -797,9 +878,20 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + + - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -865,9 +957,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -935,9 +1037,19 @@ jobs: python-version: "3.8" - uses: aws-actions/setup-sam@v2 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true audience: sts.amazonaws.com.cn aws-region: ${{ matrix.region }} role-to-assume: ${{ matrix.pipeline_execution_role }} @@ -987,9 +1099,19 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Assume the github runner role + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.BETA_REGION }} + role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} + - name: Assume the prod pipeline user role uses: aws-actions/configure-aws-credentials@v4 with: + aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ env.AWS_SESSION_TOKEN }} + role-skip-session-tagging: true aws-region: ${{ env.PROD_ECR_REGION }} role-to-assume: ${{ env.PROD_ECR_PIPELINE_EXECUTION_ROLE }} From be6af7fb154a24a84c5d1ce08e97e33265f699f0 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Tue, 16 Jan 2024 23:13:00 +0800 Subject: [PATCH 20/22] Only run test and build jobs on a new pull request --- .github/workflows/pipeline.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index eb8682e5..9f4df9b7 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -134,6 +134,7 @@ jobs: path: build-arm64.tar package-beta: + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [ build ] runs-on: ubuntu-20.04 permissions: @@ -226,6 +227,7 @@ jobs: docker manifest push 477159140107.dkr.ecr.ap-northeast-1.amazonaws.com/awsguru/aws-lambda-adapter:latest deploy-beta: + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [package-beta] runs-on: ubuntu-20.04 steps: @@ -288,6 +290,7 @@ jobs: --role-arn ${BETA_CLOUDFORMATION_EXECUTION_ROLE} e2e-test-zip: + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [deploy-beta] runs-on: ubuntu-20.04 steps: @@ -362,6 +365,7 @@ jobs: sam delete --no-prompts --region ${BETA_REGION} --stack-name ${BETA_STACK_NAME}-zip-x86 e2e-test-oci: + if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [deploy-beta] runs-on: ubuntu-20.04 steps: From de5081e2b858db54ad05ad93ae241ca9b62381cc Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Wed, 17 Jan 2024 09:04:12 +0800 Subject: [PATCH 21/22] Correct the regions for assume roles --- .github/workflows/pipeline.yaml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 9f4df9b7..478d3013 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -137,9 +137,6 @@ jobs: if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }} needs: [ build ] runs-on: ubuntu-20.04 - permissions: - id-token: write - contents: read steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -501,7 +498,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} - name: Assume the gamma pipeline user role @@ -574,7 +571,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ env.PROD_ECR_REGION }} role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} - name: Assume the prod pipeline user role @@ -647,7 +644,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} - name: Assume the china pipeline user role @@ -722,7 +719,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} - name: Assume the china pipeline user role @@ -807,7 +804,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} - name: Assume the gamma pipeline user role @@ -885,7 +882,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} @@ -964,7 +961,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} - name: Assume the china pipeline user role @@ -1044,7 +1041,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} - name: Assume the china pipeline user role @@ -1106,7 +1103,7 @@ jobs: - name: Assume the github runner role uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: ${{ env.BETA_REGION }} + aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_ROLE }} - name: Assume the prod pipeline user role From 38206c472821f1111da1bd0983f4aaa904d9d8d2 Mon Sep 17 00:00:00 2001 From: Harold Sun Date: Wed, 17 Jan 2024 10:09:52 +0800 Subject: [PATCH 22/22] Correct the audience for China regions --- .github/workflows/pipeline.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 3216d02e..6fcaaed1 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -648,6 +648,7 @@ jobs: with: aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + audience: sts.amazonaws.com.cn - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 @@ -723,6 +724,7 @@ jobs: with: aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + audience: sts.amazonaws.com.cn - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 @@ -965,6 +967,7 @@ jobs: with: aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + audience: sts.amazonaws.com.cn - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4 @@ -1045,6 +1048,7 @@ jobs: with: aws-region: ${{ matrix.region }} role-to-assume: ${{ env.GITHUB_RUNNER_CHINA_ROLE }} + audience: sts.amazonaws.com.cn - name: Assume the china pipeline user role uses: aws-actions/configure-aws-credentials@v4