diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 195f7f7bbf16c..bd117e4ac4279 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: - name: Cache virtualenv for kubernetes testing uses: actions/cache@v2 env: - cache-name: cache-kubernetes-tests-virtualenv-v3 + cache-name: cache-kubernetes-tests-virtualenv-v4 with: path: .build/.kubernetes_venv key: "${{ env.cache-name }}-${{ github.job }}-\ @@ -306,6 +306,17 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh + helm-tests: + timeout-minutes: 5 + name: "Checks: Helm tests" + runs-on: ubuntu-latest + env: + CI_JOB_TYPE: "Tests" + steps: + - uses: actions/checkout@master + - name: "Helm Tests" + run: ./scripts/ci/ci_run_helm_testing.sh + requirements: timeout-minutes: 80 name: "Requirements" diff --git a/CI.rst b/CI.rst index c8986cc97fb84..210cf9026f0b2 100644 --- a/CI.rst +++ b/CI.rst @@ -33,7 +33,7 @@ environments we use. Most of our CI jobs are written as bash scripts which are e the CI jobs and we are mapping all the CI-specific environment variables to generic "CI" variables. The only two places where CI-specific code might be are: -- CI-specific declaration file (for example it is `<.github/workflow/ci.yml>`_ for GitHub Actions +- CI-specific declaration file (for example it is `<.github/workflows/ci.yml>`_ for GitHub Actions - The ``get_environment_for_builds_on_ci`` function in ``_ where mapping is performed from the CI-environment specific to generic values. Example for that is CI_EVENT_TYPE variable which determines whether we are running a ``push``. ``schedule`` or ``pull_request`` kind of CI job. For diff --git a/airflow/__init__.py b/airflow/__init__.py index c8dcd213fd5f4..287efe39e57a4 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -40,12 +40,14 @@ from airflow import utils from airflow import settings from airflow.configuration import conf -from airflow.models import DAG from flask_admin import BaseView from importlib import import_module from airflow.exceptions import AirflowException settings.initialize() +# Delay the import of airflow.models to be after the settings initialization to make sure that +# any reference to a settings' functions (e.g task_instance_mutation_hook) holds the expected implementation +from airflow.models import DAG # noqa: E402 login = None # type: Any log = logging.getLogger(__name__) diff --git a/airflow/kubernetes/worker_configuration.py b/airflow/kubernetes/worker_configuration.py index cf232a1591e6d..4eae2ef68e138 100644 --- a/airflow/kubernetes/worker_configuration.py +++ b/airflow/kubernetes/worker_configuration.py @@ -134,6 +134,10 @@ def _get_init_containers(self): name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh' ), + k8s.V1EnvVar( + name='GIT_SYNC_ADD_USER', + value='true' + ), k8s.V1EnvVar( name='GIT_SYNC_SSH', value='true' diff --git a/breeze b/breeze index 01927e507b6d8..3e8d2c3f8493c 100755 --- a/breeze +++ b/breeze @@ -509,7 +509,7 @@ function prepare_command_files() { export COMPOSE_PROD_FILE # Base python image for the build - export PYTHON_BASE_IMAGE=python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster + export PYTHON_BASE_IMAGE=python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" diff --git a/chart/README.md b/chart/README.md index 6cc361e859492..e7ef973ecdfcb 100644 --- a/chart/README.md +++ b/chart/README.md @@ -66,7 +66,7 @@ The command removes all the Kubernetes components associated with the chart and ## Updating DAGs -The recommended way to update your DAGs with this chart is to build a new docker image with the latest code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image: +The recommended way to update your DAGs with this chart is to build a new docker image with the latest DAG code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image: ```bash helm upgrade airflow . \ @@ -77,6 +77,42 @@ helm upgrade airflow . \ For local development purppose you can also u You can also build the image locally and use it via deployment method described by Breeze. +## Mounting DAGS using Git-Sync side car with Persistence enabled + +This option will use a Persistent Volume Claim with an accessMode of `ReadWriteMany`. The scheduler pod will sync DAGs from a git repository onto the PVC every configured number of seconds. The other pods will read the synced DAGs. Not all volume plugins have support for `ReadWriteMany` accessMode. Refer [Persistent Volume Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) for details + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=true \ + --set dags.gitSync.enabled=true + # you can also override the other persistence or gitSync values + # by setting the dags.persistence.* and dags.gitSync.* values + # Please refer to values.yaml for details +``` + +## Mounting DAGS using Git-Sync side car without Persistence +This option will use an always running Git-Sync side car on every scheduler,webserver and worker pods. The Git-Sync side car containers will sync DAGs from a git repository every configured number of seconds. If you are using the KubernetesExecutor, Git-sync will run as an initContainer on your worker pods. + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=false \ + --set dags.gitSync.enabled=true + # you can also override the other gitSync values + # by setting the dags.gitSync.* values + # Refer values.yaml for details +``` + +## Mounting DAGS from an externally populated PVC +In this approach, Airflow will read the DAGs from a PVC which has `ReadOnlyMany` or `ReadWriteMany` accessMode. You will have to ensure that the PVC is populated/updated with the required DAGs(this won't be handled by the chart). You can pass in the name of the volume claim to the chart + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=true \ + --set dags.persistence.existingClaim=my-volume-claim + --set dags.gitSync.enabled=false +``` + + ## Parameters The following tables lists the configurable parameters of the Airflow chart and their default values. @@ -160,6 +196,8 @@ The following tables lists the configurable parameters of the Airflow chart and | `webserver.resources.requests.cpu` | CPU Request of webserver | `~` | | `webserver.resources.requests.memory` | Memory Request of webserver | `~` | | `webserver.defaultUser` | Optional default airflow user information | `{}` | +| `dags.persistence.*` | Dag persistence configutation | Please refer to `values.yaml` | +| `dags.gitSync.*` | Git sync configuration | Please refer to `values.yaml` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index ac121a4260c40..898924f2ff930 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -93,6 +93,80 @@ {{ end }} {{- end }} +{{/* Git ssh key volume */}} +{{- define "git_sync_ssh_key_volume"}} +- name: git-sync-ssh-key + secret: + secretName: {{ .Values.dags.gitSync.sshKeySecret }} + defaultMode: 288 +{{- end }} + +{{/* Git sync container */}} +{{- define "git_sync_container"}} +- name: {{ .Values.dags.gitSync.containerName }} + image: "{{ .Values.dags.gitSync.containerRepository }}:{{ .Values.dags.gitSync.containerTag }}" + env: + {{- if .Values.dags.gitSync.sshKeySecret }} + - name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - name: GIT_SYNC_SSH + value: "true" + {{- if .Values.dags.gitSync.knownHosts }} + - name: GIT_KNOWN_HOSTS + value: "true" + - name: GIT_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + {{- else }} + - name: GIT_KNOWN_HOSTS + value: "false" + {{- end }} + {{ else if .Values.dags.gitSync.credentialsSecret }} + - name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GIT_SYNC_USERNAME + - name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GIT_SYNC_PASSWORD + {{- end }} + - name: GIT_SYNC_REV + value: {{ .Values.dags.gitSync.rev | quote }} + - name: GIT_SYNC_BRANCH + value: {{ .Values.dags.gitSync.branch | quote }} + - name: GIT_SYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} + - name: GIT_SYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} + - name: GIT_SYNC_ROOT + value: {{ .Values.dags.gitSync.root | quote }} + - name: GIT_SYNC_DEST + value: {{ .Values.dags.gitSync.dest | quote }} + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_SYNC_WAIT + value: {{ .Values.dags.gitSync.wait | quote }} + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} + volumeMounts: + - name: dags + mountPath: {{ .Values.dags.gitSync.root }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} + - name: git-sync-ssh-key + mountPath: /etc/git-secret/ssh + readOnly: true + subPath: gitSshKey + {{- if .Values.dags.gitSync.knownHosts }} + - name: config + mountPath: /etc/git-secret/known_hosts + readOnly: true + subPath: known_hosts + {{- end }} + {{- end }} +{{- end }} + # This helper will change when customers deploy a new image. {{ define "airflow_image" -}} {{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }} @@ -185,9 +259,30 @@ log_connections = {{ .Values.pgbouncer.logConnections }} {{ (printf "%s/logs" .Values.airflowHome) | quote }} {{- end }} +{{ define "airflow_dags" -}} +{{- if .Values.dags.gitSync.enabled -}} +{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subPath ) }} +{{- else -}} +{{ (printf "%s/dags" .Values.airflowHome) }} +{{- end -}} +{{- end -}} + +{{ define "airflow_dags_volume_claim" -}} +{{- if and .Values.dags.persistence.enabled .Values.dags.persistence.existingClaim -}} +{{ .Values.dags.persistence.existingClaim }} +{{- else -}} +{{ .Release.Name }}-dags +{{- end -}} +{{- end -}} + +{{ define "airflow_dags_mount_path" -}} +{{ (printf "%s/dags" .Values.airflowHome) }} +{{- end }} + {{ define "airflow_config_path" -}} {{ (printf "%s/airflow.cfg" .Values.airflowHome) | quote }} {{- end }} + {{ define "airflow_webserver_config_path" -}} {{ (printf "%s/webserver_config.py" .Values.airflowHome) | quote }} {{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 9a0c0008eebdc..b24e474db3cd4 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -30,83 +30,17 @@ metadata: heritage: {{ .Release.Service }} {{- with .Values.labels }} {{ toYaml . | indent 4 }} -{{- end }} +{{- end -}} +{{- $Global := . }} data: # These are system-specified config overrides. - airflow.cfg: | - [core] - load_examples = False - colored_console_log = False - executor = {{ .Values.executor }} -{{- if .Values.elasticsearch.enabled }} - remote_logging = True -{{- end }} - - [logging] - logging_level = DEBUG - [webserver] - enable_proxy_fix = True - expose_config = True - rbac = True - -{{- if eq .Values.executor "CeleryExecutor" }} - [celery] - default_queue = celery -{{- end }} - - [scheduler] - scheduler_heartbeat_sec = 5 -{{- if .Values.statsd.enabled }} - statsd_on = True - statsd_port = 9125 - statsd_prefix = airflow - statsd_host = {{ printf "%s-statsd" .Release.Name }} -{{- end }} - # Restart Scheduler every 41460 seconds (11 hours 31 minutes) - # The odd time is chosen so it is not always restarting on the same "hour" boundary - run_duration = 41460 - -{{- if .Values.elasticsearch.enabled }} - [elasticsearch] - # The elasticsearch variables were updated to the shorter names in v1.10.4 - write_stdout = True - json_format = True - log_id_template = {dag_id}_{task_id}_{execution_date}_{try_number} - - [elasticsearch_configs] - max_retries = 3 - timeout = 30 - retry_timeout = True -{{- end }} - -{{- if eq .Values.executor "KubernetesExecutor" }} - [kubernetes] - namespace = {{ .Release.Namespace }} - airflow_configmap = {{ include "airflow_config" . }} - airflow_local_settings_configmap = {{ include "airflow_config" . }} - worker_container_repository = {{ .Values.images.airflow.repository }} - worker_container_tag = {{ .Values.images.airflow.tag }} - worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} - worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount - image_pull_secrets = {{ template "registry_secret" . }} - dags_in_image = True - delete_worker_pods = True - - [kubernetes_secrets] - AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }} - AIRFLOW__CORE__FERNET_KEY = {{ printf "%s=fernet-key" (include "fernet_key_secret" .) }} -{{- if .Values.elasticsearch.enabled }} - AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST = {{ printf "%s=connection" (include "elasticsearch_secret" .) }} -{{- end }} - - [kubernetes_labels] - tier = airflow - component = worker - release = {{ .Release.Name }} - {{- range $key, $value := .Values.labels }} - {{ printf "%s = %s" $key $value }} + airflow.cfg: |- + {{- range $section, $settings := .Values.config }} + [{{ $section }}] + {{- range $key, $val := $settings }} + {{ $key }} = {{ tpl ($val | toString) $Global }} {{- end }} -{{- end }} + {{ end }} {{- if .Values.webserver.webserverConfig }} webserver_config.py: | @@ -117,3 +51,7 @@ data: airflow_local_settings.py: | {{ .Values.scheduler.airflowLocalSettings | nindent 4 }} {{- end }} +{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }} + known_hosts: | + {{ .Values.dags.gitSync.knownHosts | nindent 4 }} +{{- end }} diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml new file mode 100644 index 0000000000000..53ea550ddbc52 --- /dev/null +++ b/chart/templates/dags-persistent-volume-claim.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-dags + labels: + tier: airflow + component: dags-pvc + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +spec: + accessModes: [{{ .Values.dags.persistence.accessMode | quote }}] + resources: + requests: + storage: {{ .Values.dags.persistence.size | quote }} + {{- if .Values.dags.persistence.storageClass }} + {{- if (eq "-" .Values.dags.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.dags.persistence.storageClass }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index d5c3a0662bd50..933155687bf56 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -144,6 +144,11 @@ spec: mountPath: {{ template "airflow_local_setting_path" . }} subPath: airflow_local_settings.py readOnly: true +{{- end }} +{{- if .Values.dags.gitSync.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} + {{- include "git_sync_container" . | indent 8 }} {{- end }} # Always start the garbage collector sidecar. - name: scheduler-gc @@ -177,6 +182,17 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {} + {{- end }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} {{- if not $stateful }} - name: logs emptyDir: {} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index b4c9714bec9ed..9ea2bc11826f2 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -68,6 +68,7 @@ spec: restartPolicy: Always securityContext: runAsUser: {{ .Values.uid }} + fsGroup: {{ .Values.gid }} {{- if or .Values.registry.secretName .Values.registry.connection }} imagePullSecrets: - name: {{ template "registry_secret" . }} @@ -82,6 +83,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} +{{- include "git_sync_container" . | indent 8 }} +{{- end }} - name: webserver image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -105,6 +109,10 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} +{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} +{{- end }} {{- if .Values.webserver.extraVolumeMounts }} {{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }} {{- end }} @@ -134,6 +142,17 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ .Release.Name }}-dags + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {} + {{- if .Values.dags.gitSync.sshKeySecret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- end }} {{- if .Values.webserver.extraVolumes }} {{ toYaml .Values.webserver.extraVolumes | indent 8 }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index d5043a7735978..439843dac92e8 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -105,6 +105,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" . | indent 8 }} + {{- end }} - name: worker image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -126,6 +129,10 @@ spec: mountPath: {{ template "airflow_local_setting_path" . }} subPath: airflow_local_settings.py readOnly: true +{{- end }} +{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} env: {{- include "custom_airflow_environment" . | indent 10 }} @@ -142,6 +149,17 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if .Values.dags.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} + - name: dags + emptyDir: {} + {{- if .Values.dags.gitSync.sshKeySecret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- end }} {{- if not $persistence }} - name: logs emptyDir: {} diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml new file mode 100644 index 0000000000000..2e24615ec17c2 --- /dev/null +++ b/chart/tests/dags-persistent-volume-claim_test.yaml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - dags-persistent-volume-claim.yaml +tests: + - it: should not generate a document if persistence is disabled + set: + dags: + persistence: + enabled: false + asserts: + - hasDocuments: + count: 0 + - it: should not generate a document when using an existingClaim + set: + dags: + persistence: + enabled: true + existingClaim: test-claim + asserts: + - hasDocuments: + count: 0 + - it: should generate a document if persistence is enabled & not using an existingClaim + set: + dags: + persistence: + enabled: true + existingClaim: ~ + asserts: + - hasDocuments: + count: 1 + - it: should set PVC details correctly + set: + dags: + persistence: + enabled: true + size: 1G + existingClaim: ~ + storageClass: "MyStorageClass" + accessMode: ReadWriteMany + asserts: + - equal: + path: spec + value: + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: 1G + storageClassName: "MyStorageClass" diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml new file mode 100644 index 0000000000000..10d2288ed4876 --- /dev/null +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - scheduler/scheduler-deployment.yaml +tests: + - it: should add dags volume + set: + dags: + gitSync: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: validate the git sync container spec + set: + dags: + gitSync: + enabled: true + containerName: git-sync-test + containerTag: test-tag + containerRepository: test-registry/test-repo + wait: 66 + maxFailures: 70 + subPath: "path1/path2" + dest: "test-dest" + root: "/git-root" + rev: HEAD + depth: 1 + repo: https://github.com/apache/airflow.git + branch: test-branch + sshKeySecret: ~ + credentialsSecret: ~ + knownHosts: ~ + persistence: + enabled: true + asserts: + - equal: + path: spec.template.spec.containers[1] + value: + name: git-sync-test + image: test-registry/test-repo:test-tag + env: + - name: GIT_SYNC_REV + value: HEAD + - name: GIT_SYNC_BRANCH + value: test-branch + - name: GIT_SYNC_REPO + value: https://github.com/apache/airflow.git + - name: GIT_SYNC_DEPTH + value: "1" + - name: GIT_SYNC_ROOT + value: /git-root + - name: GIT_SYNC_DEST + value: test-dest + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_SYNC_WAIT + value: "66" + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: "70" + volumeMounts: + - mountPath: /git-root + name: dags + - it: validate if ssh params are added + set: + dags: + gitSync: + enabled: true + containerName: git-sync-test + sshKeySecret: ssh-secret + knownHosts: ~ + branch: test-branch + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_SSH + value: "true" + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_KNOWN_HOSTS + value: "false" + - contains: + path: spec.template.spec.volumes + content: + name: git-sync-ssh-key + secret: + secretName: ssh-secret + defaultMode: 288 + - it: should set username and pass env variables + set: + dags: + gitSync: + enabled: true + credentialsSecret: user-pass-secret + sshKeySecret: ~ + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: user-pass-secret + key: GIT_SYNC_USERNAME + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: user-pass-secret + key: GIT_SYNC_PASSWORD diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml new file mode 100644 index 0000000000000..bdbb7c66c2767 --- /dev/null +++ b/chart/tests/git-sync-webserver_test.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - webserver/webserver-deployment.yaml +tests: + - it: should add dags volume to the webserver if git sync & peristence is enabled + set: + dags: + gitSync: + enabled: true + persistence: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled + set: + dags: + gitSync: + enabled: true + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add git sync container to webserver if persistence is not enabled, but git sync is + set: + dags: + gitSync: + enabled: true + containerName: git-sync + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: git-sync + - it: should not add sync container to webserver if git sync and persistence are enabled + set: + dags: + gitSync: + enabled: true + container_name: git-sync + persistence: + enabled: true + asserts: + - notEqual: + path: spec.template.spec.containers[0].name + value: git-sync diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml new file mode 100644 index 0000000000000..847a4dc3baa00 --- /dev/null +++ b/chart/tests/git-sync-worker_test.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - workers/worker-deployment.yaml +tests: + - it: should add dags volume to the worker if git sync & peristence is enabled + set: + executor: CeleryExecutor + dags: + persistence: + enabled: true + gitSync: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add dags volume to the worker if git sync is enabled & peristence is disabled + set: + executor: CeleryExecutor + dags: + gitSync: + enabled: true + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add git sync container to worker if persistence is not enabled, but git sync is + set: + executor: CeleryExecutor + dags: + gitSync: + enabled: true + containerName: git-sync + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: git-sync + - it: should not add sync container to worker if git sync and persistence are enabled + set: + executor: CeleryExecutor + dags: + gitSync: + enabled: true + containerName: git-sync + persistence: + enabled: true + asserts: + - notEqual: + path: spec.template.spec.containers[0].name + value: git-sync diff --git a/chart/values.yaml b/chart/values.yaml index 19184198a547e..45ea84a0e74d7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +--- # Default values for airflow. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -434,3 +434,153 @@ postgresql: enabled: true postgresqlPassword: postgres postgresqlUsername: postgres + +# Config settings to go into the mounted airflow.cfg +# +# Please note that these values are passed through the `tpl` function, so are +# all subject to being rendered as go templates. If you need to include a +# litera `{{` in a value, it must be expessed like this: +# +# a: '{{ "{{ not a template }}" }}' +# +# yamllint disable rule:line-length +config: + core: + dags_folder: '{{ include "airflow_dags" . }}' + load_examples: 'False' + colored_console_log: 'False' + executor: '{{ .Values.executor }}' + remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}' + # Authentication backend used for the experimental API + api: + auth_backend: airflow.api.auth.backend.deny_all + logging: + logging_level: DEBUG + webserver: + enable_proxy_fix: 'True' + expose_config: 'True' + rbac: 'True' + + celery: + default_queue: celery + + scheduler: + scheduler_heartbeat_sec: 5 + statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' + statsd_port: 9125 + statsd_prefix: airflow + statsd_host: '{{ printf "%s-statsd" .Release.Name }}' + + # Restart Scheduler every 41460 seconds (11 hours 31 minutes) + # The odd time is chosen so it is not always restarting on the same "hour" boundary + run_duration: 41460 + elasticsearch: + json_format: 'True' + log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}" + elasticsearch_configs: + max_retries: 3 + timeout: 30 + retry_timeout: 'True' + + kubernetes: + namespace: '{{ .Release.Namespace }}' + airflow_configmap: '{{ include "airflow_config" . }}' + airflow_local_settings_configmap: '{{ include "airflow_config" . }}' + worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' + worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' + worker_container_image_pull_policy: '{{ .Values.images.airflow.pullPolicy }}' + worker_service_account_name: '{{ .Release.Name }}-worker-serviceaccount' + image_pull_secrets: '{{ template "registry_secret" . }}' + dags_in_image: '{{ ternary "False" "True" (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}' + delete_worker_pods: 'True' + run_as_user: '{{ .Values.uid }}' + fs_group: '{{ .Values.gid }}' + git_dags_folder_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{end}}' + dags_volume_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{ end }}' + dags_volume_claim: '{{- if .Values.dags.persistence.enabled }}{{ include "airflow_dags_volume_claim" . }}{{ end }}' + dags_volume_subpath: '{{- if .Values.dags.persistence.enabled }}{{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }}{{ end }}' + git_repo: '{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}{{ .Values.dags.gitSync.repo }}{{ end }}' + git_branch: '{{ .Values.dags.gitSync.branch }}' + git_sync_rev: '{{ .Values.dags.gitSync.rev }}' + git_sync_depth: '{{ .Values.dags.gitSync.depth }}' + git_sync_root: '{{ .Values.dags.gitSync.root }}' + git_sync_dest: '{{ .Values.dags.gitSync.dest }}' + git_sync_container_repository: '{{ .Values.dags.gitSync.containerRepository }}' + git_sync_container_tag: '{{ .Values.dags.gitSync.containerTag }}' + git_sync_init_container_name: '{{ .Values.dags.gitSync.containerName }}' + git_sync_run_as_user: '{{ .Values.uid }}' + git_ssh_known_hosts_configmap_name: '{{- if .Values.dags.gitSync.knownHosts }}{{ include "airflow_config" . }}{{ end }}' + git_ssh_key_secret_name: '{{- if .Values.dags.gitSync.sshKeySecret }}{{ .Values.dags.gitSync.sshKeySecret }}{{ end }}' + git_sync_credentials_secret: '{{- if .Values.dags.gitSync.credentialsSecret }}{{ .Values.dags.gitSync.credentialsSecret }}{{ end }}' + kubernetes_secrets: + AIRFLOW__CORE__SQL_ALCHEMY_CONN: '{{ printf "%s=connection" (include "airflow_metadata_secret" .) }}' + AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}' +# yamllint enable rule:line-length + +# Git sync +dags: + persistence: + # Enable persistent volume for storing dags + enabled: false + # Volume size for dags + size: 1Gi + # If using a custom storageClass, pass name here + storageClassName: + # access mode of the persistent volume + accessMode: ReadWriteMany + ## the name of an existing PVC to use + existingClaim: ~ + gitSync: + enabled: false + # git repo clone url + # ssh examples ssh://git@github.com/apache/airflow.git + # git@github.com:apache/airflow.git + # https example: https://github.com/apache/airflow.git + repo: https://github.com/apache/airflow.git + branch: v1-10-stable + rev: HEAD + root: "/git" + dest: "repo" + depth: 1 + # the number of consecutive failures allowed before aborting + maxFailures: 0 + # subpath within the repo where dags are located + # should be "" if dags are at repo root + subPath: "tests/dags" + # if your repo needs a user name password + # you can load them to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: git-credentials + # data: + # GIT_SYNC_USERNAME: + # GIT_SYNC_PASSWORD: + # and specify the name of the secret below + #credentialsSecret: git-credentials + # + # + # If you are using an ssh clone url, you can load + # the ssh private key to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: airflow-ssh-secret + # data: + # # key needs to be gitSshKey + # gitSshKey: + # and specify the name of the secret below + #sshKeySecret: airflow-ssh-secret + # If you are using an ssh private key, you can additionally + # specify the content of your known_hosts file, example: + #knownHosts: | + # , + # , + # interval between git sync attempts in seconds + wait: 60 + # git sync image details + containerRepository: k8s.gcr.io/git-sync + containerTag: v3.1.6 + containerName: git-sync diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt index ac06c016e4e92..311c764eb35bd 100644 --- a/requirements/requirements-python3.5.txt +++ b/requirements/requirements-python3.5.txt @@ -8,7 +8,7 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.3 +Flask-SQLAlchemy==2.4.4 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 @@ -25,7 +25,7 @@ PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 SQLAlchemy==1.3.18 -Sphinx==3.1.1 +Sphinx==3.1.2 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 @@ -38,7 +38,7 @@ ansiwrap==0.8.4 apipkg==1.5 apispec==3.3.1 appdirs==1.4.4 -argcomplete==1.11.1 +argcomplete==1.12.0 asn1crypto==1.3.0 aspy.yaml==1.3.0 astroid==2.4.2 @@ -47,10 +47,10 @@ attrs==19.3.0 aws-sam-translator==1.25.0 aws-xray-sdk==2.6.0 azure-common==1.1.25 -azure-cosmos==3.1.2 +azure-cosmos==3.2.0 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==10.0.0 +azure-mgmt-resource==10.1.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 @@ -60,9 +60,9 @@ bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 blinker==1.4 -boto3==1.14.14 +boto3==1.14.21 boto==2.49.0 -botocore==1.17.14 +botocore==1.17.21 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 @@ -71,7 +71,7 @@ celery==4.4.6 certifi==2020.6.20 cffi==1.14.0 cfgv==2.0.1 -cfn-lint==0.33.2 +cfn-lint==0.34.0 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -79,11 +79,11 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.1 +coverage==5.2 croniter==0.3.34 cryptography==2.9.2 cx-Oracle==8.0.0 -datadog==0.37.1 +datadog==0.38.0 decorator==4.4.2 defusedxml==0.6.0 dill==0.3.2 @@ -99,13 +99,13 @@ elasticsearch==5.5.3 email-validator==1.1.1 entrypoints==0.3 execnet==1.7.1 -fastavro==0.23.5 +fastavro==0.23.6 filelock==3.0.12 flake8-colors==0.1.6 flake8==3.8.3 -flaky==3.6.1 +flaky==3.7.0 flask-swagger==0.2.13 -flower==0.9.4 +flower==0.9.5 freezegun==0.3.15 fsspec==0.7.4 funcsigs==1.0.2 @@ -113,10 +113,10 @@ future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.2 google-api-core==1.21.0 -google-api-python-client==1.9.3 -google-auth-httplib2==0.0.3 +google-api-python-client==1.10.0 +google-auth-httplib2==0.0.4 google-auth-oauthlib==0.4.1 -google-auth==1.18.0 +google-auth==1.19.1 google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==1.0.1 @@ -133,7 +133,7 @@ google-cloud-videointelligence==1.15.0 google-cloud-vision==1.0.0 google-resumable-media==0.5.1 googleapis-common-protos==1.52.0 -graphviz==0.14 +graphviz==0.14.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 grpcio==1.30.0 @@ -141,9 +141,9 @@ gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 -humanize==0.5.1 +humanize==2.5.0 hvac==0.10.4 -identify==1.4.21 +identify==1.4.23 idna==2.10 imagesize==1.2.0 importlib-metadata==1.7.0 @@ -165,7 +165,7 @@ jsonpickle==1.4.1 jsonpointer==2.0 jsonschema==3.2.0 junit-xml==1.9 -jupyter-client==6.1.5 +jupyter-client==6.1.6 jupyter-core==4.6.3 kombu==4.6.11 kubernetes==11.0.0 @@ -214,9 +214,10 @@ pluggy==0.13.1 pre-commit==1.21.0 presto-python-client==0.7.0 prison==0.1.0 +prometheus-client==0.8.0 prompt-toolkit==2.0.10 protobuf==3.12.2 -psutil==5.7.0 +psutil==5.7.2 psycopg2-binary==2.8.5 ptyprocess==0.6.0 py==1.9.0 @@ -240,8 +241,8 @@ pytest-cov==2.10.0 pytest-forked==1.2.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.0 -pytest-timeout==1.4.1 -pytest-xdist==1.32.0 +pytest-timeout==1.4.2 +pytest-xdist==1.33.0 pytest==5.4.3 python-daemon==2.2.4 python-dateutil==2.8.1 @@ -253,7 +254,7 @@ python-nvd3==0.15.0 python-slugify==4.0.1 python3-openid==3.2.0 pytz==2020.1 -pytzdata==2019.3 +pytzdata==2020.1 pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 @@ -271,12 +272,12 @@ s3transfer==0.3.3 sasl==0.2.1 sendgrid==5.6.0 sentinels==1.0.0 -sentry-sdk==0.15.1 +sentry-sdk==0.16.1 setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.8 +snowflake-connector-python==2.2.9 snowflake-sqlalchemy==1.2.3 soupsieve==2.0.1 sphinx-argparse==0.2.5 @@ -307,19 +308,19 @@ tqdm==4.47.0 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 -typing==3.7.4.1 +typing==3.7.4.3 tzlocal==1.5.1 unicodecsv==0.14.1 uritemplate==3.0.1 urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 -virtualenv==20.0.25 +virtualenv==20.0.27 wcwidth==0.2.5 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 -yamllint==1.23.0 +yamllint==1.24.2 zdesk==2.7.1 zipp==1.2.0 zope.deprecation==4.4.0 diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt index 615b800413db6..cbfa08dbfc613 100644 --- a/requirements/requirements-python3.6.txt +++ b/requirements/requirements-python3.6.txt @@ -8,7 +8,7 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.3 +Flask-SQLAlchemy==2.4.4 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 @@ -24,9 +24,9 @@ PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.6 +SQLAlchemy-Utils==0.36.8 SQLAlchemy==1.3.18 -Sphinx==3.1.1 +Sphinx==3.1.2 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 @@ -39,7 +39,7 @@ ansiwrap==0.8.4 apipkg==1.5 apispec==1.3.3 appdirs==1.4.4 -argcomplete==1.11.1 +argcomplete==1.12.0 asn1crypto==1.3.0 astroid==2.4.2 async-generator==1.10 @@ -48,10 +48,10 @@ attrs==19.3.0 aws-sam-translator==1.25.0 aws-xray-sdk==2.6.0 azure-common==1.1.25 -azure-cosmos==3.1.2 +azure-cosmos==3.2.0 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==10.0.0 +azure-mgmt-resource==10.1.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 @@ -62,9 +62,9 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.14.14 +boto3==1.14.20 boto==2.49.0 -botocore==1.17.14 +botocore==1.17.20 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 @@ -81,11 +81,11 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.1 +coverage==5.2 croniter==0.3.34 cryptography==2.9.2 cx-Oracle==8.0.0 -datadog==0.37.1 +datadog==0.38.0 decorator==4.4.2 defusedxml==0.6.0 dill==0.3.2 @@ -101,13 +101,13 @@ elasticsearch==5.5.3 email-validator==1.1.1 entrypoints==0.3 execnet==1.7.1 -fastavro==0.23.5 +fastavro==0.23.6 filelock==3.0.12 flake8-colors==0.1.6 flake8==3.8.3 -flaky==3.6.1 +flaky==3.7.0 flask-swagger==0.2.13 -flower==0.9.4 +flower==0.9.5 freezegun==0.3.15 fsspec==0.7.4 funcsigs==1.0.2 @@ -116,9 +116,9 @@ future==0.18.2 gcsfs==0.6.2 google-api-core==1.21.0 google-api-python-client==1.9.3 -google-auth-httplib2==0.0.3 +google-auth-httplib2==0.0.4 google-auth-oauthlib==0.4.1 -google-auth==1.18.0 +google-auth==1.19.0 google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==1.0.1 @@ -135,7 +135,7 @@ google-cloud-videointelligence==1.15.0 google-cloud-vision==1.0.0 google-resumable-media==0.5.1 googleapis-common-protos==1.52.0 -graphviz==0.14 +graphviz==0.14.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 grpcio==1.30.0 @@ -143,13 +143,12 @@ gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 -humanize==0.5.1 +humanize==2.5.0 hvac==0.10.4 -identify==1.4.21 +identify==1.4.23 idna==2.10 imagesize==1.2.0 importlib-metadata==1.7.0 -importlib-resources==3.0.0 inflection==0.5.0 ipdb==0.13.3 ipython-genutils==0.2.0 @@ -167,7 +166,7 @@ jsonpickle==1.4.1 jsonpointer==2.0 jsonschema==3.2.0 junit-xml==1.9 -jupyter-client==6.1.5 +jupyter-client==6.1.6 jupyter-core==4.6.3 kombu==4.6.11 kubernetes==11.0.0 @@ -189,7 +188,7 @@ mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 natsort==7.0.1 -nbclient==0.4.0 +nbclient==0.4.1 nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 @@ -213,9 +212,10 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==2.5.1 +pre-commit==2.6.0 presto-python-client==0.7.0 prison==0.1.3 +prometheus-client==0.8.0 prompt-toolkit==3.0.5 protobuf==3.12.2 psutil==5.7.0 @@ -243,7 +243,7 @@ pytest-forked==1.2.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.0 pytest-timeout==1.4.1 -pytest-xdist==1.32.0 +pytest-xdist==1.33.0 pytest==5.4.3 python-daemon==2.2.4 python-dateutil==2.8.1 @@ -255,7 +255,7 @@ python-nvd3==0.15.0 python-slugify==4.0.1 python3-openid==3.2.0 pytz==2020.1 -pytzdata==2019.3 +pytzdata==2020.1 pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 @@ -274,7 +274,7 @@ s3transfer==0.3.3 sasl==0.2.1 sendgrid==5.6.0 sentinels==1.0.0 -sentry-sdk==0.15.1 +sentry-sdk==0.16.1 setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 @@ -310,14 +310,13 @@ tqdm==4.47.0 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 -typing==3.7.4.1 tzlocal==1.5.1 unicodecsv==0.14.1 uritemplate==3.0.1 urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 -virtualenv==20.0.25 +virtualenv==20.0.26 wcwidth==0.2.5 websocket-client==0.57.0 wrapt==1.12.1 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt index 7e9d467c230e2..f1c6c48bffdaf 100644 --- a/requirements/requirements-python3.7.txt +++ b/requirements/requirements-python3.7.txt @@ -8,7 +8,7 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.3 +Flask-SQLAlchemy==2.4.4 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 @@ -24,9 +24,9 @@ PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.6 +SQLAlchemy-Utils==0.36.8 SQLAlchemy==1.3.18 -Sphinx==3.1.1 +Sphinx==3.1.2 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 @@ -39,7 +39,7 @@ ansiwrap==0.8.4 apipkg==1.5 apispec==1.3.3 appdirs==1.4.4 -argcomplete==1.11.1 +argcomplete==1.12.0 asn1crypto==1.3.0 astroid==2.4.2 async-generator==1.10 @@ -48,10 +48,10 @@ attrs==19.3.0 aws-sam-translator==1.25.0 aws-xray-sdk==2.6.0 azure-common==1.1.25 -azure-cosmos==3.1.2 +azure-cosmos==3.2.0 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==10.0.0 +azure-mgmt-resource==10.1.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 @@ -62,9 +62,9 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.14.14 +boto3==1.14.20 boto==2.49.0 -botocore==1.17.14 +botocore==1.17.20 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 @@ -81,11 +81,11 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.1 +coverage==5.2 croniter==0.3.34 cryptography==2.9.2 cx-Oracle==8.0.0 -datadog==0.37.1 +datadog==0.38.0 decorator==4.4.2 defusedxml==0.6.0 dill==0.3.2 @@ -101,13 +101,13 @@ elasticsearch==5.5.3 email-validator==1.1.1 entrypoints==0.3 execnet==1.7.1 -fastavro==0.23.5 +fastavro==0.23.6 filelock==3.0.12 flake8-colors==0.1.6 flake8==3.8.3 -flaky==3.6.1 +flaky==3.7.0 flask-swagger==0.2.13 -flower==0.9.4 +flower==0.9.5 freezegun==0.3.15 fsspec==0.7.4 funcsigs==1.0.2 @@ -116,9 +116,9 @@ future==0.18.2 gcsfs==0.6.2 google-api-core==1.21.0 google-api-python-client==1.9.3 -google-auth-httplib2==0.0.3 +google-auth-httplib2==0.0.4 google-auth-oauthlib==0.4.1 -google-auth==1.18.0 +google-auth==1.19.0 google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==1.0.1 @@ -135,7 +135,7 @@ google-cloud-videointelligence==1.15.0 google-cloud-vision==1.0.0 google-resumable-media==0.5.1 googleapis-common-protos==1.52.0 -graphviz==0.14 +graphviz==0.14.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 grpcio==1.30.0 @@ -143,9 +143,9 @@ gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 -humanize==0.5.1 +humanize==2.5.0 hvac==0.10.4 -identify==1.4.21 +identify==1.4.23 idna==2.10 imagesize==1.2.0 importlib-metadata==1.7.0 @@ -166,7 +166,7 @@ jsonpickle==1.4.1 jsonpointer==2.0 jsonschema==3.2.0 junit-xml==1.9 -jupyter-client==6.1.5 +jupyter-client==6.1.6 jupyter-core==4.6.3 kombu==4.6.11 kubernetes==11.0.0 @@ -188,7 +188,7 @@ mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 natsort==7.0.1 -nbclient==0.4.0 +nbclient==0.4.1 nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 @@ -212,9 +212,10 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==2.5.1 +pre-commit==2.6.0 presto-python-client==0.7.0 prison==0.1.3 +prometheus-client==0.8.0 prompt-toolkit==3.0.5 protobuf==3.12.2 psutil==5.7.0 @@ -233,7 +234,6 @@ pydruid==0.5.8 pyflakes==2.2.0 pykerberos==1.2.1 pymongo==3.10.1 -pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 @@ -242,7 +242,7 @@ pytest-forked==1.2.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.0 pytest-timeout==1.4.1 -pytest-xdist==1.32.0 +pytest-xdist==1.33.0 pytest==5.4.3 python-daemon==2.2.4 python-dateutil==2.8.1 @@ -254,7 +254,7 @@ python-nvd3==0.15.0 python-slugify==4.0.1 python3-openid==3.2.0 pytz==2020.1 -pytzdata==2019.3 +pytzdata==2020.1 pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 @@ -273,7 +273,7 @@ s3transfer==0.3.3 sasl==0.2.1 sendgrid==5.6.0 sentinels==1.0.0 -sentry-sdk==0.15.1 +sentry-sdk==0.16.1 setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 @@ -315,7 +315,7 @@ uritemplate==3.0.1 urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 -virtualenv==20.0.25 +virtualenv==20.0.26 wcwidth==0.2.5 websocket-client==0.57.0 wrapt==1.12.1 diff --git a/requirements/requirements-python3.8.txt b/requirements/requirements-python3.8.txt index 747ae42adf9d6..f1c6c48bffdaf 100644 --- a/requirements/requirements-python3.8.txt +++ b/requirements/requirements-python3.8.txt @@ -8,7 +8,7 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.3 +Flask-SQLAlchemy==2.4.4 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 @@ -24,9 +24,9 @@ PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.6 +SQLAlchemy-Utils==0.36.8 SQLAlchemy==1.3.18 -Sphinx==3.1.1 +Sphinx==3.1.2 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 @@ -39,7 +39,7 @@ ansiwrap==0.8.4 apipkg==1.5 apispec==1.3.3 appdirs==1.4.4 -argcomplete==1.11.1 +argcomplete==1.12.0 asn1crypto==1.3.0 astroid==2.4.2 async-generator==1.10 @@ -48,10 +48,10 @@ attrs==19.3.0 aws-sam-translator==1.25.0 aws-xray-sdk==2.6.0 azure-common==1.1.25 -azure-cosmos==3.1.2 +azure-cosmos==3.2.0 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==10.0.0 +azure-mgmt-resource==10.1.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 @@ -62,9 +62,9 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.14.14 +boto3==1.14.20 boto==2.49.0 -botocore==1.17.14 +botocore==1.17.20 cached-property==1.5.1 cachetools==4.1.1 cassandra-driver==3.20.2 @@ -81,11 +81,11 @@ cloudant==0.5.10 colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 -coverage==5.1 +coverage==5.2 croniter==0.3.34 cryptography==2.9.2 cx-Oracle==8.0.0 -datadog==0.37.1 +datadog==0.38.0 decorator==4.4.2 defusedxml==0.6.0 dill==0.3.2 @@ -101,13 +101,13 @@ elasticsearch==5.5.3 email-validator==1.1.1 entrypoints==0.3 execnet==1.7.1 -fastavro==0.23.5 +fastavro==0.23.6 filelock==3.0.12 flake8-colors==0.1.6 flake8==3.8.3 -flaky==3.6.1 +flaky==3.7.0 flask-swagger==0.2.13 -flower==0.9.4 +flower==0.9.5 freezegun==0.3.15 fsspec==0.7.4 funcsigs==1.0.2 @@ -116,9 +116,9 @@ future==0.18.2 gcsfs==0.6.2 google-api-core==1.21.0 google-api-python-client==1.9.3 -google-auth-httplib2==0.0.3 +google-auth-httplib2==0.0.4 google-auth-oauthlib==0.4.1 -google-auth==1.18.0 +google-auth==1.19.0 google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==1.0.1 @@ -135,7 +135,7 @@ google-cloud-videointelligence==1.15.0 google-cloud-vision==1.0.0 google-resumable-media==0.5.1 googleapis-common-protos==1.52.0 -graphviz==0.14 +graphviz==0.14.1 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 grpcio==1.30.0 @@ -143,9 +143,9 @@ gunicorn==19.10.0 hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 -humanize==0.5.1 +humanize==2.5.0 hvac==0.10.4 -identify==1.4.21 +identify==1.4.23 idna==2.10 imagesize==1.2.0 importlib-metadata==1.7.0 @@ -166,7 +166,7 @@ jsonpickle==1.4.1 jsonpointer==2.0 jsonschema==3.2.0 junit-xml==1.9 -jupyter-client==6.1.5 +jupyter-client==6.1.6 jupyter-core==4.6.3 kombu==4.6.11 kubernetes==11.0.0 @@ -188,7 +188,7 @@ mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 natsort==7.0.1 -nbclient==0.4.0 +nbclient==0.4.1 nbformat==5.0.7 nest-asyncio==1.3.3 networkx==2.4 @@ -212,9 +212,10 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==2.5.1 +pre-commit==2.6.0 presto-python-client==0.7.0 prison==0.1.3 +prometheus-client==0.8.0 prompt-toolkit==3.0.5 protobuf==3.12.2 psutil==5.7.0 @@ -241,7 +242,7 @@ pytest-forked==1.2.0 pytest-instafail==0.4.2 pytest-rerunfailures==9.0 pytest-timeout==1.4.1 -pytest-xdist==1.32.0 +pytest-xdist==1.33.0 pytest==5.4.3 python-daemon==2.2.4 python-dateutil==2.8.1 @@ -253,7 +254,7 @@ python-nvd3==0.15.0 python-slugify==4.0.1 python3-openid==3.2.0 pytz==2020.1 -pytzdata==2019.3 +pytzdata==2020.1 pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 @@ -272,7 +273,7 @@ s3transfer==0.3.3 sasl==0.2.1 sendgrid==5.6.0 sentinels==1.0.0 -sentry-sdk==0.15.1 +sentry-sdk==0.16.1 setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 @@ -314,7 +315,7 @@ uritemplate==3.0.1 urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 -virtualenv==20.0.25 +virtualenv==20.0.26 wcwidth==0.2.5 websocket-client==0.57.0 wrapt==1.12.1 diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/ci_run_helm_testing.sh new file mode 100755 index 0000000000000..0a267d49aac5a --- /dev/null +++ b/scripts/ci/ci_run_helm_testing.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Running helm tests" + +CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/" + +echo "Chart directory is $CHART_DIR" + +docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \ + --entrypoint /bin/sh \ + aneeshkj/helm-unittest \ + -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ." diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 26ab1c45bc41f..ed894a127e1b7 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -334,14 +334,14 @@ function prepare_ci_build() { fi export CACHE_IMAGE_PREFIX=${CACHE_IMAGE_PREFX:=${GITHUB_ORGANISATION}/${GITHUB_REPO}} export CACHED_AIRFLOW_CI_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_CI_BASE_TAG}" - export CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" else export CACHED_AIRFLOW_CI_IMAGE="" export CACHED_PYTHON_BASE_IMAGE="" fi export AIRFLOW_BUILD_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}/${AIRFLOW_CI_BASE_TAG}" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci" - export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then export DEFAULT_IMAGE="${AIRFLOW_CI_IMAGE_DEFAULT}" @@ -621,7 +621,7 @@ function prepare_prod_build() { export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build" export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}" export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}" - export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then export DEFAULT_IMAGE="${AIRFLOW_PROD_IMAGE_DEFAULT}" else @@ -646,7 +646,7 @@ function prepare_prod_build() { export CACHE_IMAGE_PREFIX=${CACHE_IMAGE_PREFX:=${GITHUB_ORGANISATION}/${GITHUB_REPO}} export CACHED_AIRFLOW_PROD_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_PROD_BASE_TAG}" export CACHED_AIRFLOW_PROD_BUILD_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/${AIRFLOW_PROD_BASE_TAG}-build" - export CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export CACHED_PYTHON_BASE_IMAGE="${CACHE_REGISTRY}/${CACHE_IMAGE_PREFIX}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" else export CACHED_AIRFLOW_PROD_IMAGE="" export CACHED_AIRFLOW_PROD_BUILD_IMAGE="" diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 5f2a74251d350..c41dff995a484 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -21,6 +21,10 @@ function initialize_common_environment { # default python Major/Minor version PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"} + # python image version to use + # shellcheck disable=SC2034 + PYTHON_BASE_IMAGE_VERSION=${PYTHON_MAJOR_MINOR_VERSION} + # extra flags passed to docker run for CI image # shellcheck disable=SC2034 EXTRA_DOCKER_FLAGS=() diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index 4f5ebe522c490..f24b3aa2a564c 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -294,7 +294,8 @@ function deploy_airflow_with_helm() { --set "defaultAirflowRepository=${DOCKERHUB_USER}/${DOCKERHUB_REPO}" \ --set "images.airflow.repository=${DOCKERHUB_USER}/${DOCKERHUB_REPO}" \ --set "images.airflow.tag=${AIRFLOW_PROD_BASE_TAG}" -v 1 \ - --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}" -v 1 + --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}" -v 1 \ + --set "config.api.auth_backend=airflow.api.auth.backend.default" echo popd || exit 1 } diff --git a/tests/kubernetes/test_worker_configuration.py b/tests/kubernetes/test_worker_configuration.py index e57ae68c74eda..a94a112494164 100644 --- a/tests/kubernetes/test_worker_configuration.py +++ b/tests/kubernetes/test_worker_configuration.py @@ -216,6 +216,7 @@ def test_init_environment_using_git_sync_ssh_without_known_hosts(self): env = init_containers[0].env self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_SSH', value='true'), env) @@ -236,6 +237,7 @@ def test_init_environment_using_git_sync_ssh_with_known_hosts(self): env = init_containers[0].env self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env) self.assertIn(k8s.V1EnvVar( name='GIT_SSH_KNOWN_HOSTS_FILE', @@ -262,6 +264,7 @@ def test_init_environment_using_git_sync_user_without_known_hosts(self): env = init_containers[0].env self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env) @@ -290,6 +293,7 @@ def test_init_environment_using_git_sync_user_with_known_hosts(self): env = init_containers[0].env self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env)