Convert Helm tests to use the new Python Breeze

.gitattributes

@@ -1,6 +1,3 @@
-breeze export-ignore
-breeze-legacy export-ignore
-breeze-complete export-ignore
 clients export-ignore
 clients export-ignore
 dev export-ignore

.github/CODEOWNERS

@@ -70,8 +70,6 @@
 
 # Dev tools
 /.github/workflows/ @potiuk @ashb @kaxil
-breeze @potiuk
-breeze-complete @potiuk
 Dockerfile @potiuk @ashb @mik-laj
 Dockerfile.ci @potiuk @ashb
 /dev/ @potiuk @ashb @jedcunningham

.github/workflows/ci.yml

@@ -133,13 +133,10 @@ jobs:
       all-python-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
       default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
-      min-max-python-versions-as-string: >-
-        ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }}
       kubernetes-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+      kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }}
       default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
-      min-max-kubernetes-versions-as-string: >-
-        ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }}
       postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
       default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
       mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
@@ -1441,7 +1438,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
 
   tests-kubernetes:
     timeout-minutes: 240
-    name: Helm Chart; ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+    name: "Helm: ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
     runs-on: ${{ needs.build-info.outputs.runs-on }}
     needs: [build-info, wait-for-prod-images]
     strategy:
@@ -1450,17 +1447,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       fail-fast: false
     env:
       RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
-      BACKEND: postgres
-      RUN_TESTS: "true"
-      RUNTIME: "kubernetes"
-      KUBERNETES_MODE: "image"
-      EXECUTOR: ${{matrix.executor}}
-      KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
-      HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
-      CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
-        ${{needs.build-info.outputs.python-versions-list-as-string}}
-      CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
-        ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
     if: >
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.needs-helm-tests == 'true' ) &&
@@ -1489,98 +1475,63 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       - name: "Cache bin folder with tools for kubernetes testing"
         uses: actions/cache@v3
         with:
-          path: ".build/kubernetes-bin"
-          key: "kubernetes-binaries
--${{ needs.build-info.outputs.default-kind-version }}\
--${{ needs.build-info.outputs.default-helm-version }}"
-          restore-keys: "kubernetes-binaries"
-      - name: "Kubernetes Tests"
-        run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
-        env:
-          PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
-      - name: "Upload KinD logs"
-        uses: actions/upload-artifact@v3
-        if: failure() || cancelled()
-        with:
-          name: kind-logs-${{matrix.executor}}
-          path: /tmp/kind_logs_*
-          retention-days: 7
-      - name: "Fix ownership"
-        run: breeze ci fix-ownership
-        if: always()
-
-  tests-helm-executor-upgrade:
-    timeout-minutes: 150
-    name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-    runs-on: ${{ needs.build-info.outputs.runs-on }}
-    needs: [build-info, wait-for-prod-images]
-    env:
-      RUNS_ON: ${{ needs.build-info.outputs.runs-on }}
-      BACKEND: postgres
-      RUN_TESTS: "true"
-      RUNTIME: "kubernetes"
-      KUBERNETES_MODE: "image"
-      EXECUTOR: "KubernetesExecutor"
-      KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
-      HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
-      CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >-
-        ${{needs.build-info.outputs.min-max-python-versions-as-string}}
-      CURRENT_KUBERNETES_VERSIONS_AS_STRING: >-
-        ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-    if: >
-      needs.build-info.outputs.run-kubernetes-tests == 'true' &&
-      needs.build-info.outputs.default-branch == 'main'
-    steps:
-      - name: Cleanup repo
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-      - name: "Setup python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ needs.build-info.outputs.default-python-version }}
-          cache: 'pip'
-          cache-dependency-path: ./dev/breeze/setup*
-      - run: ./scripts/ci/install_breeze.sh
-      - name: "Free space"
-        run: breeze ci free-space
-      - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-        run: breeze prod-image pull --run-in-parallel --tag-as-latest
+          path: ".build/.k8s-env"
+          key: "k8s-env"
+      - name: K8s Setup env
+        run: breeze k8s setup-env
+      - name: >
+          K8S Build images Python: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+        run: breeze k8s build-k8s-image --run-in-parallel
         env:
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
           IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
-      - name: "Cache virtualenv for kubernetes testing"
-        uses: actions/cache@v3
-        with:
-          path: ".build/.kubernetes_venv"
-          key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
-  -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
-  -${{needs.build-info.outputs.min-max-python-versions-as-string}}
-  -${{ hashFiles('setup.py','setup.cfg') }}"
-          restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
-  -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \
-  -${{needs.build-info.outputs.min-max-python-versions-as-string}}"
-      - name: "Cache bin folder with tools for kubernetes testing"
-        uses: actions/cache@v3
-        with:
-          path: ".build/kubernetes-bin"
-          key: "kubernetes-binaries
-  -${{ needs.build-info.outputs.default-kind-version }}\
-  -${{ needs.build-info.outputs.default-helm-version }}"
-          restore-keys: "kubernetes-binaries"
-      - name: "Kubernetes Helm Chart Executor Upgrade Tests"
-        run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
+      - name: K8S Create clusters ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s create-cluster --run-in-parallel
         env:
-          PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
-      - name: "Upload KinD logs"
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+      - name: K8S Upload images ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s upload-k8s-image --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Configure clusters ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s configure-cluster --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+      - name: K8S Deploy Airflow ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s deploy-airflow --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Tests ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s tests --run-in-parallel
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Upgrade ${{matrix.executor}} ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s deploy-airflow --run-in-parallel --upgrade --set config.logging.logging_level=INFO
+        env:
+          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+          KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+          EXECUTOR: ${{matrix.executor}}
+      - name: K8S Logs ${{needs.build-info.outputs.kubernetes-combos}}
+        run: breeze k8s logs --all
+        if: failure() || cancelled()
+      - name: Upload KinD logs ${{needs.build-info.outputs.kubernetes-combos}}
         uses: actions/upload-artifact@v3
         if: failure() || cancelled()
         with:
-          name: kind-logs-KubernetesExecutor
+          name: kind-logs-${{matrix.executor}}
           path: /tmp/kind_logs_*
           retention-days: 7
+      - name: "Delete clusters"
+        run: breeze k8s delete-cluster --all
+        if: always()
       - name: "Fix ownership"
         run: breeze ci fix-ownership
         if: always()

.gitpod.yml

@@ -20,7 +20,7 @@
 # Docs: https://www.gitpod.io/docs/config-gitpod-file/
 
 tasks:
-  - init: ./breeze-legacy -y
+  - init: ./script/ci/install_breeze.sh
   - name: Install pre-commit
     openMode: split-right
     command: |

.pre-commit-config.yaml

@@ -84,7 +84,7 @@ repos:
       - id: insert-license
         name: Add license for all shell files
         exclude: ^\.github/.*$|^airflow/_vendor/|^dev/breeze/autocomplete/.*$
-        files: ^breeze-legacy$|^breeze-complete$|\.bash$|\.sh$
+        files: \.bash$|\.sh$
         args:
           - --comment-style
           - "|#|"
@@ -301,7 +301,7 @@ repos:
           The word(s) should be in lowercase." && exec codespell "$@"' --
         language: python
         types: [text]
-        exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$
+        exclude: ^airflow/_vendor/|^RELEASE_NOTES\.txt$|^airflow/www/static/css/material-icons\.css$|^images/.*$|^.*package-lock.json$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
           - --skip=docs/*/commits.rst,airflow/providers/*/*.rst,*.lock,INTHEWILD.md,*.min.js,docs/apache-airflow/tutorial/pipeline_example.csv,airflow/www/*.log
@@ -556,12 +556,6 @@ repos:
         files: \.*example_dags.*\.py$
         exclude: ^airflow/_vendor/
         pass_filenames: true
-      - id: check-integrations-are-consistent
-        name: Check if integration list is consistent in various places
-        entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
-        language: system
-        pass_filenames: false
-        files: ^common/_common_values\.sh$|^breeze-complete$
       - id: check-apache-license-rat
         name: Check if licenses are OK for Apache
         entry: ./scripts/ci/pre_commit/pre_commit_check_license.py
@@ -597,16 +591,17 @@ repos:
         require_serial: true
       - id: lint-helm-chart
         name: Lint Helm Chart
-        entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
-        language: system
+        entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.py
+        language: python
         pass_filenames: false
         files: ^chart
         require_serial: true
+        additional_dependencies: ['rich>=12.4.4','requests']
       - id: run-shellcheck
         name: Check Shell scripts syntax correctness
         language: docker_image
         entry: koalaman/shellcheck:v0.8.0 -x -a
-        files: ^breeze-legacy$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$
+        files: ^.*\.sh$|^hooks/build$|^hooks/push$|\.bash$
         exclude: ^dev/breeze/autocomplete/.*$
       - id: lint-css
         name: stylelint