From f1ba8959bc16018d72f4163113eac084c38c1dcc Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Thu, 18 Jun 2020 14:10:37 +0530 Subject: [PATCH 01/19] add git sync sidecars --- chart/templates/_helpers.yaml | 61 +++++++++++++++++++ chart/templates/configmap.yaml | 26 +++++++- .../dags-persistent-volume-claim.yaml | 41 +++++++++++++ .../scheduler/scheduler-deployment.yaml | 18 ++++++ .../webserver/webserver-deployment.yaml | 20 ++++++ .../templates/workers/worker-deployment.yaml | 20 ++++++ chart/values.yaml | 32 ++++++++++ 7 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 chart/templates/dags-persistent-volume-claim.yaml diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index ac121a4260c40..ead6e526e2e3f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -93,6 +93,54 @@ {{ end }} {{- end }} +{{/* Git ssh key volume */}} +{{- define "git_sync_ssh_key_volume"}} +- name: git-sync-ssh-key + secret: + secretName: {{ .Values.gitSync.ssh_key_secret }} + defaultMode: 288 +{{- end }} + +{{/* Git sync container */}} +{{- define "git_sync_container"}} +- name: {{ .Values.gitSync.container_name }} + image: "{{ .Values.gitSync.container_repository }}:{{ .Values.gitSync.container_tag }}" + env: + {{- if .Values.gitSync.ssh_key_secret }} + - name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - name: GIT_SYNC_SSH + value: "true" + {{- end }} + - name: GIT_SYNC_REV + value: {{ .Values.gitSync.rev | quote }} + - name: GIT_SYNC_BRANCH + value: {{ .Values.gitSync.branch | quote }} + - name: GIT_SYNC_REPO + value: {{ .Values.gitSync.repo | quote }} + - name: GIT_SYNC_DEPTH + value: {{ .Values.gitSync.depth | quote }} + - name: GIT_SYNC_ROOT + value: {{ .Values.gitSync.root | quote }} + - name: GIT_SYNC_DEST + value: {{ .Values.gitSync.dest | quote }} + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_KNOWN_HOSTS + value: "false" + - name: GIT_SYNC_WAIT + value: {{ .Values.gitSync.wait | quote }} + volumeMounts: + - name: dags + mountPath: {{ .Values.gitSync.root }} + {{- if and .Values.gitSync.enabled .Values.gitSync.ssh_key_secret }} + - name: git-sync-ssh-key + mountPath: /etc/git-secret/ssh + readOnly: true + subPath: gitSshKey + {{- end }} +{{- end }} + # This helper will change when customers deploy a new image. {{ define "airflow_image" -}} {{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }} @@ -185,9 +233,22 @@ log_connections = {{ .Values.pgbouncer.logConnections }} {{ (printf "%s/logs" .Values.airflowHome) | quote }} {{- end }} +{{ define "airflow_dags" -}} +{{- if .Values.gitSync.enabled -}} +{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.gitSync.dest .Values.gitSync.subpath ) }} +{{- else -}} +{{ (printf "%s/dags" .Values.airflowHome) }} +{{- end -}} +{{- end -}} + +{{ define "airflow_dags_mount_path" -}} +{{ (printf "%s/dags" .Values.airflowHome) }} +{{- end }} + {{ define "airflow_config_path" -}} {{ (printf "%s/airflow.cfg" .Values.airflowHome) | quote }} {{- end }} + {{ define "airflow_webserver_config_path" -}} {{ (printf "%s/webserver_config.py" .Values.airflowHome) | quote }} {{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 86affe8fcbda8..c4a0e2d2e8a5d 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -35,6 +35,7 @@ data: # These are system-specified config overrides. airflow.cfg: | [core] + dags_folder = {{ include "airflow_dags" . }} load_examples = False colored_console_log = False executor = {{ .Values.executor }} @@ -92,8 +93,31 @@ data: worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount image_pull_secrets = {{ template "registry_secret" . }} - dags_in_image = True + dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }} delete_worker_pods = True + run_as_user = {{ .Values.uid }} + {{- if .Values.gitSync.enabled }} + git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }} + dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }} + + {{- if .Values.gitSync.persistence.enabled }} + dags_volume_claim = {{ .Release.Name }}-dags + dags_volume_subpath = {{.Values.gitSync.dest }}/{{ .Values.gitSync.subpath }} + {{- else }} + git_repo = {{ .Values.gitSync.repo }} + git_branch = {{ .Values.gitSync.branch }} + git_sync_rev = {{ .Values.gitSync.rev }} + git_sync_depth = {{ .Values.gitSync.depth }} + git_sync_root = {{ .Values.gitSync.root }} + git_sync_dest = {{ .Values.gitSync.dest }} + git_sync_container_repository = {{ .Values.gitSync.container_repository }} + git_sync_container_tag = {{ .Values.gitSync.container_tag }} + git_sync_init_container_name = {{ .Values.gitSync.container_name }} + {{- if .Values.gitSync.ssh_key_secret }} + git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }} + {{- end }} + {{- end }} + {{- end }} [kubernetes_secrets] AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }} diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml new file mode 100644 index 0000000000000..12e7d7d5b74a4 --- /dev/null +++ b/chart/templates/dags-persistent-volume-claim.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if .Values.gitSync.persistence.enabled }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-dags + labels: + tier: airflow + component: dags-pvc + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +spec: + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: {{ .Values.gitSync.persistence.size | quote }} + {{- if .Values.gitSync.persistence.storageClass }} + {{- if (eq "-" .Values.gitSync.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.gitSync.persistence.storageClass }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index d5c3a0662bd50..c56342ca200e9 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -144,6 +144,11 @@ spec: mountPath: {{ template "airflow_local_setting_path" . }} subPath: airflow_local_settings.py readOnly: true +{{- end }} +{{- if .Values.gitSync.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} + {{- include "git_sync_container" . | indent 8 }} {{- end }} # Always start the garbage collector sidecar. - name: scheduler-gc @@ -177,6 +182,19 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if and .Values.gitSync.enabled .Values.gitSync.ssh_key_secret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + {{- if .Values.gitSync.enabled }} + {{- if .Values.gitSync.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ .Release.Name }}-dags + {{- else }} + - name: dags + emptyDir: {} + {{- end }} + {{- end }} {{- if not $stateful }} - name: logs emptyDir: {} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index b4c9714bec9ed..fd376be90891c 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -82,6 +82,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: +{{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }} +{{- include "git_sync_container" . | indent 8 }} +{{- end }} - name: webserver image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -105,6 +108,10 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} +{{- if .Values.gitSync.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} +{{- end }} {{- if .Values.webserver.extraVolumeMounts }} {{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }} {{- end }} @@ -134,6 +141,19 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if .Values.gitSync.enabled }} + {{- if .Values.gitSync.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ .Release.Name }}-dags + {{- else }} + {{- if .Values.gitSync.ssh_key_secret }} + {{- include "git_ssh_key_volume" . | indent 8 }} + {{- end }} + - name: dags + emptyDir: {} + {{- end }} + {{- end }} {{- if .Values.webserver.extraVolumes }} {{ toYaml .Values.webserver.extraVolumes | indent 8 }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index d5043a7735978..38edff7e54b80 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -105,6 +105,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: + {{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }} + {{- include "git_sync_container" . | indent 8 }} + {{- end }} - name: worker image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} @@ -126,6 +129,10 @@ spec: mountPath: {{ template "airflow_local_setting_path" . }} subPath: airflow_local_settings.py readOnly: true +{{- end }} +{{- if .Values.gitSync.enabled }} + - name: dags + mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} env: {{- include "custom_airflow_environment" . | indent 10 }} @@ -142,6 +149,19 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} + {{- if .Values.gitSync.enabled }} + {{- if .Values.gitSync.persistence.enabled }} + - name: dags + persistentVolumeClaim: + claimName: {{ .Release.Name }}-dags + {{- else }} + {{- if .Values.gitSync.ssh_key_secret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} + - name: dags + emptyDir: {} + {{- end }} + {{- end }} {{- if not $persistence }} - name: logs emptyDir: {} diff --git a/chart/values.yaml b/chart/values.yaml index dc13064927bfd..acf4ff7631242 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -438,3 +438,35 @@ postgresql: # Authentication backend used for the experimental API api: authBackend: airflow.api.auth.backend.deny_all + +# Git sync +gitSync: + enabled: true + persistence: + # Enable persistent volumes for storing synced dags + enabled: true + # Volume size for dags + size: 1Gi + # If using a custom storageClass, pass name here + storageClassName: + # git repo clone url + # ssh examples ssh://git@github.com/apache/airflow.git + # git@github.com:apache/airflow.git + # https example: https://github.com/apache/airflow.git + repo: https://github.com/apache/airflow.git + branch: v1-10-stable + rev: HEAD + root: "/git" + dest: "repo" + depth: 1 + # subpath within the repo where dags are located + # should be "" if dags are at repo root + subpath: "tests/dags" + #TODO Handle username password secrets + #TODO Handle known_hosts + #credentials_secret: + #ssh_key_secret: git-ssh + wait: 60 + container_repository: k8s.gcr.io/git-sync + container_tag: v3.1.6 + container_name: git-sync-clone From c8fb2ec850e8243b46c1ada44c54d8d7b5b6f316 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 10:41:13 +0530 Subject: [PATCH 02/19] add a helm test --- .github/workflows/ci.yml | 11 ++++ .gitignore | 1 + chart/requirements.lock | 6 -- .../dags-persistent-volume-claim.yaml | 2 +- .../dags-persistent-volume-claim_test.yaml | 63 +++++++++++++++++++ chart/tests/git-sync-scheduler_test.yaml | 26 ++++++++ scripts/ci/ci_run_helm_testing.sh | 28 +++++++++ 7 files changed, 130 insertions(+), 7 deletions(-) delete mode 100644 chart/requirements.lock create mode 100644 chart/tests/dags-persistent-volume-claim_test.yaml create mode 100644 chart/tests/git-sync-scheduler_test.yaml create mode 100755 scripts/ci/ci_run_helm_testing.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3b0e351b9a8b..2c7702e74e925 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -363,6 +363,17 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - name: "Tests" run: ./scripts/ci/ci_run_airflow_testing.sh + helm-tests: + timeout-minutes: 60 + name: "Checks: Helm tests" + runs-on: ubuntu-latest + env: + CI_JOB_TYPE: "Tests" + steps: + - uses: actions/checkout@master + - name: "Helm Tests" + run: ./scripts/ci/ci_run_helm_testing.sh + requirements: timeout-minutes: 80 name: "Requirements" diff --git a/.gitignore b/.gitignore index c7ae00a28ccfd..6699e29a3365a 100644 --- a/.gitignore +++ b/.gitignore @@ -201,4 +201,5 @@ log.txt* Chart.lock # Chart dependencies +requirements.lock **/charts/*.tgz diff --git a/chart/requirements.lock b/chart/requirements.lock deleted file mode 100644 index f86e6968adf29..0000000000000 --- a/chart/requirements.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://kubernetes-charts.storage.googleapis.com - version: 6.3.12 -digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af -generated: "2020-06-21T19:11:53.498134738+02:00" diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml index 12e7d7d5b74a4..79f9b7329b8f2 100644 --- a/chart/templates/dags-persistent-volume-claim.yaml +++ b/chart/templates/dags-persistent-volume-claim.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -{{- if .Values.gitSync.persistence.enabled }} +{{- if and .Values.gitSync.enabled .Values.gitSync.persistence.enabled }} kind: PersistentVolumeClaim apiVersion: v1 metadata: diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml new file mode 100644 index 0000000000000..ffc003934f2ff --- /dev/null +++ b/chart/tests/dags-persistent-volume-claim_test.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +templates: + - dags-persistent-volume-claim.yaml +tests: + - it: should not generate a document if persistence is disabled + set: + gitSync: + enabled: true + persistence: + enabled: false + asserts: + - hasDocuments: + count: 0 + - it: should not generate a document if git sync is disabled + set: + gitSync: + enabled: false + persistence: + enabled: true + asserts: + - hasDocuments: + count: 0 + - it: should generate a document if git sync & persistence is enabled + set: + gitSync: + enabled: true + persistence: + enabled: true + asserts: + - hasDocuments: + count: 1 + - it: should set PVC details correctly + set: + gitSync: + persistence: + enabled: true + size: 1G + storageClass: "MyStorageClass" + asserts: + - equal: + path: spec + value: + accessModes: ["ReadWriteMany"] + resources: + requests: + storage: 1G + storageClassName: "MyStorageClass" diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml new file mode 100644 index 0000000000000..483caf3a5b8d2 --- /dev/null +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +tests: + - it: should add dags volume + set: + gitSync: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + template: scheduler/scheduler-deployment.yaml diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/ci_run_helm_testing.sh new file mode 100755 index 0000000000000..43cc2d41961b8 --- /dev/null +++ b/scripts/ci/ci_run_helm_testing.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Running helm tests" + +CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/" + +echo "Chart directory is $CHART_DIR" + +docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \ + --entrypoint /bin/sh \ + divya12/helm-unittest \ + -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ." From e17430c585b8572aed10627ca7f87d43a856ba95 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 17:04:58 +0530 Subject: [PATCH 03/19] add more tests --- .../webserver/webserver-deployment.yaml | 6 +- .../dags-persistent-volume-claim_test.yaml | 2 +- chart/tests/git-sync-scheduler_test.yaml | 49 +++++++++++++- chart/tests/git-sync-webserver_test.yaml | 62 +++++++++++++++++ chart/tests/git-sync-worker_test.yaml | 66 +++++++++++++++++++ chart/values.yaml | 4 +- scripts/ci/ci_run_helm_testing.sh | 2 +- 7 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 chart/tests/git-sync-webserver_test.yaml create mode 100644 chart/tests/git-sync-worker_test.yaml diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index fd376be90891c..9aa6e078e8b18 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -147,11 +147,11 @@ spec: persistentVolumeClaim: claimName: {{ .Release.Name }}-dags {{- else }} - {{- if .Values.gitSync.ssh_key_secret }} - {{- include "git_ssh_key_volume" . | indent 8 }} - {{- end }} - name: dags emptyDir: {} + {{- if .Values.gitSync.ssh_key_secret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} {{- end }} {{- end }} {{- if .Values.webserver.extraVolumes }} diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml index ffc003934f2ff..f41e66dcdabf0 100644 --- a/chart/tests/dags-persistent-volume-claim_test.yaml +++ b/chart/tests/dags-persistent-volume-claim_test.yaml @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +--- templates: - dags-persistent-volume-claim.yaml tests: diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index 483caf3a5b8d2..85569df1677ac 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -14,6 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +--- +templates: + - scheduler/scheduler-deployment.yaml tests: - it: should add dags volume set: @@ -23,4 +26,48 @@ tests: - equal: path: spec.template.spec.volumes[1].name value: dags - template: scheduler/scheduler-deployment.yaml + - it: validate the git sync container spec + set: + gitSync: + enabled: true + container_name: git-sync-test + container_tag: test-tag + container_repository: test-registry/test-repo + wait: 66 + subpath: "path1/path2" + dest: "test-dest" + root: "/git-root" + rev: HEAD + depth: 1 + repo: https://github.com/apache/airflow.git + branch: test-branch + persistence: + enabled: true + asserts: + - equal: + path: spec.template.spec.containers[1] + value: + name: git-sync-test + image: test-registry/test-repo:test-tag + env: + - name: GIT_SYNC_REV + value: HEAD + - name: GIT_SYNC_BRANCH + value: test-branch + - name: GIT_SYNC_REPO + value: https://github.com/apache/airflow.git + - name: GIT_SYNC_DEPTH + value: "1" + - name: GIT_SYNC_ROOT + value: /git-root + - name: GIT_SYNC_DEST + value: test-dest + - name: GIT_SYNC_ADD_USER + value: "true" + - name: GIT_KNOWN_HOSTS + value: "false" + - name: GIT_SYNC_WAIT + value: "66" + volumeMounts: + - mountPath: /git-root + name: dags diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml new file mode 100644 index 0000000000000..b506e8700e815 --- /dev/null +++ b/chart/tests/git-sync-webserver_test.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - webserver/webserver-deployment.yaml +tests: + - it: should add dags volume to the webserver if git sync & peristence is enabled + set: + gitSync: + enabled: true + persistence: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled + set: + gitSync: + enabled: true + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add git sync container to webserver if persistence is not enabled, but git sync is + set: + gitSync: + enabled: true + container_name: git-sync + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: git-sync + - it: should not add sync container to webserver if git sync and persistence are enabled + set: + gitSync: + enabled: true + container_name: git-sync + persistence: + enabled: true + asserts: + - notEqual: + path: spec.template.spec.containers[0].name + value: git-sync diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml new file mode 100644 index 0000000000000..677d113379f3d --- /dev/null +++ b/chart/tests/git-sync-worker_test.yaml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +templates: + - workers/worker-deployment.yaml +tests: + - it: should add dags volume to the worker if git sync & peristence is enabled + set: + executor: CeleryExecutor + gitSync: + enabled: true + persistence: + enabled: true + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled + set: + executor: CeleryExecutor + gitSync: + enabled: true + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: dags + - it: should add git sync container to webserver if persistence is not enabled, but git sync is + set: + executor: CeleryExecutor + gitSync: + enabled: true + container_name: git-sync + persistence: + enabled: false + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: git-sync + - it: should not add sync container to webserver if git sync and persistence are enabled + set: + executor: CeleryExecutor + gitSync: + enabled: true + container_name: git-sync + persistence: + enabled: true + asserts: + - notEqual: + path: spec.template.spec.containers[0].name + value: git-sync diff --git a/chart/values.yaml b/chart/values.yaml index acf4ff7631242..d6df79d13fe24 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +--- # Default values for airflow. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -469,4 +469,4 @@ gitSync: wait: 60 container_repository: k8s.gcr.io/git-sync container_tag: v3.1.6 - container_name: git-sync-clone + container_name: git-sync diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/ci_run_helm_testing.sh index 43cc2d41961b8..0a267d49aac5a 100755 --- a/scripts/ci/ci_run_helm_testing.sh +++ b/scripts/ci/ci_run_helm_testing.sh @@ -24,5 +24,5 @@ echo "Chart directory is $CHART_DIR" docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \ --entrypoint /bin/sh \ - divya12/helm-unittest \ + aneeshkj/helm-unittest \ -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ." From 351d69ad42f609e8a74b2a7c2722347df218d1ad Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 20:26:26 +0530 Subject: [PATCH 04/19] allow users to provide git username and pass via a k8s secrets --- chart/templates/_helpers.yaml | 15 ++++++- chart/tests/git-sync-scheduler_test.yaml | 54 +++++++++++++++++++++++- chart/values.yaml | 32 ++++++++++++-- 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index ead6e526e2e3f..015c53ead5223 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -111,6 +111,19 @@ value: "/etc/git-secret/ssh" - name: GIT_SYNC_SSH value: "true" + - name: GIT_KNOWN_HOSTS + value: "false" + {{ else if .Values.gitSync.credentials_secret }} + - name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.gitSync.credentials_secret }} + key: GIT_SYNC_USERNAME + - name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.gitSync.credentials_secret }} + key: GIT_SYNC_PASSWORD {{- end }} - name: GIT_SYNC_REV value: {{ .Values.gitSync.rev | quote }} @@ -126,8 +139,6 @@ value: {{ .Values.gitSync.dest | quote }} - name: GIT_SYNC_ADD_USER value: "true" - - name: GIT_KNOWN_HOSTS - value: "false" - name: GIT_SYNC_WAIT value: {{ .Values.gitSync.wait | quote }} volumeMounts: diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index 85569df1677ac..cdb0925f8890c 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -64,10 +64,60 @@ tests: value: test-dest - name: GIT_SYNC_ADD_USER value: "true" - - name: GIT_KNOWN_HOSTS - value: "false" - name: GIT_SYNC_WAIT value: "66" volumeMounts: - mountPath: /git-root name: dags + - it: validate if ssh params are added + set: + gitSync: + enabled: true + container_name: git-sync-test + ssh_key_secret: ssh-secret + branch: test-branch + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SSH_KEY_FILE + value: "/etc/git-secret/ssh" + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_SSH + value: "true" + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_KNOWN_HOSTS + value: "false" + - contains: + path: spec.template.spec.volumes + content: + name: git-sync-ssh-key + secret: + secretName: ssh-secret + defaultMode: 288 + - it: should set username and pass env variables + set: + gitSync: + enabled: true + credentials_secret: user-pass-secret + asserts: + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_USERNAME + valueFrom: + secretKeyRef: + name: user-pass-secret + key: GIT_SYNC_USERNAME + - contains: + path: spec.template.spec.containers[1].env + content: + name: GIT_SYNC_PASSWORD + valueFrom: + secretKeyRef: + name: user-pass-secret + key: GIT_SYNC_PASSWORD diff --git a/chart/values.yaml b/chart/values.yaml index d6df79d13fe24..7ab4d5ad0853f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -462,11 +462,35 @@ gitSync: # subpath within the repo where dags are located # should be "" if dags are at repo root subpath: "tests/dags" - #TODO Handle username password secrets - #TODO Handle known_hosts - #credentials_secret: - #ssh_key_secret: git-ssh + # if your repo needs a user name password + # you can load them to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: git-credentials + # data: + # GIT_SYNC_USERNAME: + # GIT_SYNC_PASSWORD: + # and specify the name of the secret below + #credentials_secret: git-credentials + # + # + # If you are using an ssh clone url, you can load + # the ssh private key to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: airflow-ssh-secret + # data: + # # key needs to be gitSshKey + # gitSshKey: + # and specify the name of the secret below + #ssh_key_secret: airflow-ssh-secret + # interval between git sync attempts in seconds wait: 60 + # git sync image details container_repository: k8s.gcr.io/git-sync container_tag: v3.1.6 container_name: git-sync From dd08690da054ffe3addb2874e47fb52f6db13acf Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 20:54:59 +0530 Subject: [PATCH 05/19] set default values for airflow worker repository & tag --- chart/templates/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index c4a0e2d2e8a5d..c3ef69ce03996 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -88,8 +88,8 @@ data: namespace = {{ .Release.Namespace }} airflow_configmap = {{ include "airflow_config" . }} airflow_local_settings_configmap = {{ include "airflow_config" . }} - worker_container_repository = {{ .Values.images.airflow.repository }} - worker_container_tag = {{ .Values.images.airflow.tag }} + worker_container_repository = {{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }} + worker_container_tag = {{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }} worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount image_pull_secrets = {{ template "registry_secret" . }} From 6786d5e471a9d266758142681bdfc93d57722dd1 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 21:05:55 +0530 Subject: [PATCH 06/19] change ci timeout --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c7702e74e925..5714869f54d06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -364,7 +364,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') run: ./scripts/ci/ci_run_airflow_testing.sh helm-tests: - timeout-minutes: 60 + timeout-minutes: 5 name: "Checks: Helm tests" runs-on: ubuntu-latest env: From 784e2856dcc0be0dbe78d08c439dbbbcf5ffa1d0 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 21:35:50 +0530 Subject: [PATCH 07/19] fix link --- CI.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.rst b/CI.rst index 11a2f578329d3..a1a38ba9bd27a 100644 --- a/CI.rst +++ b/CI.rst @@ -33,7 +33,7 @@ environments we use. Most of our CI jobs are written as bash scripts which are e the CI jobs and we are mapping all the CI-specific environment variables to generic "CI" variables. The only two places where CI-specific code might be are: -- CI-specific declaration file (for example it is `<.github/workflow/ci.yml>`_ for GitHub Actions +- CI-specific declaration file (for example it is `<.github/workflows/ci.yml>`_ for GitHub Actions - The ``get_environment_for_builds_on_ci`` function in ``_ where mapping is performed from the CI-environment specific to generic values. Example for that is CI_EVENT_TYPE variable which determines whether we are running a ``push``. ``schedule`` or ``pull_request`` kind of CI job. For From 60f768f8b152ef0fce486060c0c0e938a217be91 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 21:47:24 +0530 Subject: [PATCH 08/19] add credentials_secret to airflow.cfg configmap --- chart/templates/configmap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index c3ef69ce03996..c5cc86d4063c8 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -115,6 +115,8 @@ data: git_sync_init_container_name = {{ .Values.gitSync.container_name }} {{- if .Values.gitSync.ssh_key_secret }} git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }} + {{- else if .Values.gitSync.credentials_secret }} + git_sync_credentials_secret = {{ .Values.gitSync.credentials_secret }} {{- end }} {{- end }} {{- end }} From 0e69d66e71739d27a7a9c36755c5418060e606be Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 22:48:07 +0530 Subject: [PATCH 09/19] set GIT_SYNC_ADD_USER on kubernetes worker pods, set uid --- airflow/kubernetes/worker_configuration.py | 4 ++++ chart/templates/configmap.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/airflow/kubernetes/worker_configuration.py b/airflow/kubernetes/worker_configuration.py index 732d5accbc56e..383a705bf4b4e 100644 --- a/airflow/kubernetes/worker_configuration.py +++ b/airflow/kubernetes/worker_configuration.py @@ -127,6 +127,10 @@ def _get_init_containers(self) -> List[k8s.V1Container]: name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh' ), + k8s.V1EnvVar( + name='GIT_SYNC_ADD_USER', + value='true' + ), k8s.V1EnvVar( name='GIT_SYNC_SSH', value='true' diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index c5cc86d4063c8..7a824729dea59 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -113,6 +113,7 @@ data: git_sync_container_repository = {{ .Values.gitSync.container_repository }} git_sync_container_tag = {{ .Values.gitSync.container_tag }} git_sync_init_container_name = {{ .Values.gitSync.container_name }} + git_sync_run_as_user = {{ .Values.uid }} {{- if .Values.gitSync.ssh_key_secret }} git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }} {{- else if .Values.gitSync.credentials_secret }} From 22b1b1584aa4f03aa1e90b3667d691ce6923a841 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Fri, 19 Jun 2020 23:12:17 +0530 Subject: [PATCH 10/19] add fsGroup to webserver and kubernete workers --- chart/templates/configmap.yaml | 1 + chart/templates/webserver/webserver-deployment.yaml | 1 + tests/kubernetes/test_worker_configuration.py | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 7a824729dea59..6ede82c24b1a4 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -96,6 +96,7 @@ data: dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }} delete_worker_pods = True run_as_user = {{ .Values.uid }} + fs_group = {{ .Values.gid }} {{- if .Values.gitSync.enabled }} git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }} dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index 9aa6e078e8b18..53db3bab275a9 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -68,6 +68,7 @@ spec: restartPolicy: Always securityContext: runAsUser: {{ .Values.uid }} + fsGroup: {{ .Values.gid }} {{- if or .Values.registry.secretName .Values.registry.connection }} imagePullSecrets: - name: {{ template "registry_secret" . }} diff --git a/tests/kubernetes/test_worker_configuration.py b/tests/kubernetes/test_worker_configuration.py index b223eefdf0d09..98dd0c706e952 100644 --- a/tests/kubernetes/test_worker_configuration.py +++ b/tests/kubernetes/test_worker_configuration.py @@ -244,6 +244,7 @@ def test_init_environment_using_git_sync_ssh_without_known_hosts(self): env = init_containers[0].env self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_SSH', value='true'), env) @@ -264,6 +265,7 @@ def test_init_environment_using_git_sync_ssh_with_known_hosts(self): env = init_containers[0].env self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env) self.assertIn(k8s.V1EnvVar( name='GIT_SSH_KNOWN_HOSTS_FILE', @@ -290,6 +292,7 @@ def test_init_environment_using_git_sync_user_without_known_hosts(self): env = init_containers[0].env self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env) @@ -318,6 +321,7 @@ def test_init_environment_using_git_sync_user_with_known_hosts(self): env = init_containers[0].env self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env) + self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env) self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env) self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env) From 8fdc1bd0c0a2a156c85aab7c9a60992c45343b4c Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sat, 20 Jun 2020 10:27:26 +0530 Subject: [PATCH 11/19] move gitSync to dags.gitSync --- chart/templates/_helpers.yaml | 44 ++++---- chart/templates/configmap.yaml | 35 +++--- .../dags-persistent-volume-claim.yaml | 12 +-- .../scheduler/scheduler-deployment.yaml | 12 +-- .../webserver/webserver-deployment.yaml | 12 +-- .../templates/workers/worker-deployment.yaml | 16 ++- .../dags-persistent-volume-claim_test.yaml | 19 ++-- chart/tests/git-sync-scheduler_test.yaml | 45 ++++---- chart/tests/git-sync-webserver_test.yaml | 32 +++--- chart/tests/git-sync-worker_test.yaml | 38 ++++--- chart/values.yaml | 101 +++++++++--------- 11 files changed, 192 insertions(+), 174 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 015c53ead5223..01840e0cc88b9 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -97,54 +97,54 @@ {{- define "git_sync_ssh_key_volume"}} - name: git-sync-ssh-key secret: - secretName: {{ .Values.gitSync.ssh_key_secret }} + secretName: {{ .Values.dags.gitSync.ssh_key_secret }} defaultMode: 288 {{- end }} {{/* Git sync container */}} {{- define "git_sync_container"}} -- name: {{ .Values.gitSync.container_name }} - image: "{{ .Values.gitSync.container_repository }}:{{ .Values.gitSync.container_tag }}" +- name: {{ .Values.dags.gitSync.container_name }} + image: "{{ .Values.dags.gitSync.container_repository }}:{{ .Values.dags.gitSync.container_tag }}" env: - {{- if .Values.gitSync.ssh_key_secret }} + {{- if .Values.dags.gitSync.ssh_key_secret }} - name: GIT_SSH_KEY_FILE value: "/etc/git-secret/ssh" - name: GIT_SYNC_SSH value: "true" - name: GIT_KNOWN_HOSTS value: "false" - {{ else if .Values.gitSync.credentials_secret }} + {{ else if .Values.dags.gitSync.credentials_secret }} - name: GIT_SYNC_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.gitSync.credentials_secret }} + name: {{ .Values.dags.gitSync.credentials_secret }} key: GIT_SYNC_USERNAME - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.gitSync.credentials_secret }} + name: {{ .Values.dags.gitSync.credentials_secret }} key: GIT_SYNC_PASSWORD {{- end }} - name: GIT_SYNC_REV - value: {{ .Values.gitSync.rev | quote }} + value: {{ .Values.dags.gitSync.rev | quote }} - name: GIT_SYNC_BRANCH - value: {{ .Values.gitSync.branch | quote }} + value: {{ .Values.dags.gitSync.branch | quote }} - name: GIT_SYNC_REPO - value: {{ .Values.gitSync.repo | quote }} + value: {{ .Values.dags.gitSync.repo | quote }} - name: GIT_SYNC_DEPTH - value: {{ .Values.gitSync.depth | quote }} + value: {{ .Values.dags.gitSync.depth | quote }} - name: GIT_SYNC_ROOT - value: {{ .Values.gitSync.root | quote }} + value: {{ .Values.dags.gitSync.root | quote }} - name: GIT_SYNC_DEST - value: {{ .Values.gitSync.dest | quote }} + value: {{ .Values.dags.gitSync.dest | quote }} - name: GIT_SYNC_ADD_USER value: "true" - name: GIT_SYNC_WAIT - value: {{ .Values.gitSync.wait | quote }} + value: {{ .Values.dags.gitSync.wait | quote }} volumeMounts: - name: dags - mountPath: {{ .Values.gitSync.root }} - {{- if and .Values.gitSync.enabled .Values.gitSync.ssh_key_secret }} + mountPath: {{ .Values.dags.gitSync.root }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.ssh_key_secret }} - name: git-sync-ssh-key mountPath: /etc/git-secret/ssh readOnly: true @@ -245,13 +245,21 @@ log_connections = {{ .Values.pgbouncer.logConnections }} {{- end }} {{ define "airflow_dags" -}} -{{- if .Values.gitSync.enabled -}} -{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.gitSync.dest .Values.gitSync.subpath ) }} +{{- if .Values.dags.gitSync.enabled -}} +{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subpath ) }} {{- else -}} {{ (printf "%s/dags" .Values.airflowHome) }} {{- end -}} {{- end -}} +{{ define "airflow_dags_volume_claim" -}} +{{- if and .Values.dags.persistence.enabled .Values.dags.persistence.existingClaim -}} +{{ .Values.dags.persistence.existingClaim }} +{{- else -}} +{{ .Release.Name }}-dags +{{- end -}} +{{- end -}} + {{ define "airflow_dags_mount_path" -}} {{ (printf "%s/dags" .Values.airflowHome) }} {{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 6ede82c24b1a4..acc82d29b84a6 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -93,32 +93,31 @@ data: worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount image_pull_secrets = {{ template "registry_secret" . }} - dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }} + dags_in_image = {{ if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}False{{ else }}True{{ end }} delete_worker_pods = True run_as_user = {{ .Values.uid }} fs_group = {{ .Values.gid }} - {{- if .Values.gitSync.enabled }} + {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }} dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }} - - {{- if .Values.gitSync.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} dags_volume_claim = {{ .Release.Name }}-dags - dags_volume_subpath = {{.Values.gitSync.dest }}/{{ .Values.gitSync.subpath }} + dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subpath }} {{- else }} - git_repo = {{ .Values.gitSync.repo }} - git_branch = {{ .Values.gitSync.branch }} - git_sync_rev = {{ .Values.gitSync.rev }} - git_sync_depth = {{ .Values.gitSync.depth }} - git_sync_root = {{ .Values.gitSync.root }} - git_sync_dest = {{ .Values.gitSync.dest }} - git_sync_container_repository = {{ .Values.gitSync.container_repository }} - git_sync_container_tag = {{ .Values.gitSync.container_tag }} - git_sync_init_container_name = {{ .Values.gitSync.container_name }} + git_repo = {{ .Values.dags.gitSync.repo }} + git_branch = {{ .Values.dags.gitSync.branch }} + git_sync_rev = {{ .Values.dags.gitSync.rev }} + git_sync_depth = {{ .Values.dags.gitSync.depth }} + git_sync_root = {{ .Values.dags.gitSync.root }} + git_sync_dest = {{ .Values.dags.gitSync.dest }} + git_sync_container_repository = {{ .Values.dags.gitSync.container_repository }} + git_sync_container_tag = {{ .Values.dags.gitSync.container_tag }} + git_sync_init_container_name = {{ .Values.dags.gitSync.container_name }} git_sync_run_as_user = {{ .Values.uid }} - {{- if .Values.gitSync.ssh_key_secret }} - git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }} - {{- else if .Values.gitSync.credentials_secret }} - git_sync_credentials_secret = {{ .Values.gitSync.credentials_secret }} + {{- if .Values.dags.gitSync.ssh_key_secret }} + git_ssh_key_secret_name = {{ .Values.dags.gitSync.ssh_key_secret }} + {{- else if .Values.dags.gitSync.credentials_secret }} + git_sync_credentials_secret = {{ .Values.dags.gitSync.credentials_secret }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml index 79f9b7329b8f2..53ea550ddbc52 100644 --- a/chart/templates/dags-persistent-volume-claim.yaml +++ b/chart/templates/dags-persistent-volume-claim.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -{{- if and .Values.gitSync.enabled .Values.gitSync.persistence.enabled }} +{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }} kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -27,15 +27,15 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} spec: - accessModes: ["ReadWriteMany"] + accessModes: [{{ .Values.dags.persistence.accessMode | quote }}] resources: requests: - storage: {{ .Values.gitSync.persistence.size | quote }} - {{- if .Values.gitSync.persistence.storageClass }} - {{- if (eq "-" .Values.gitSync.persistence.storageClass) }} + storage: {{ .Values.dags.persistence.size | quote }} + {{- if .Values.dags.persistence.storageClass }} + {{- if (eq "-" .Values.dags.persistence.storageClass) }} storageClassName: "" {{- else }} - storageClassName: "{{ .Values.gitSync.persistence.storageClass }}" + storageClassName: "{{ .Values.dags.persistence.storageClass }}" {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index c56342ca200e9..7f7a6d9571f0a 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -145,7 +145,7 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} -{{- if .Values.gitSync.enabled }} +{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} - name: dags mountPath: {{ template "airflow_dags_mount_path" . }} {{- include "git_sync_container" . | indent 8 }} @@ -182,19 +182,17 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} - {{- if and .Values.gitSync.enabled .Values.gitSync.ssh_key_secret }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.ssh_key_secret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} - {{- if .Values.gitSync.enabled }} - {{- if .Values.gitSync.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} - name: dags persistentVolumeClaim: - claimName: {{ .Release.Name }}-dags - {{- else }} + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {} {{- end }} - {{- end }} {{- if not $stateful }} - name: logs emptyDir: {} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index 53db3bab275a9..c6fba61723104 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -83,7 +83,7 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: -{{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }} +{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" . | indent 8 }} {{- end }} - name: webserver @@ -109,7 +109,7 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} -{{- if .Values.gitSync.enabled }} +{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} - name: dags mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} @@ -142,19 +142,17 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} - {{- if .Values.gitSync.enabled }} - {{- if .Values.gitSync.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} - name: dags persistentVolumeClaim: claimName: {{ .Release.Name }}-dags - {{- else }} + {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {} - {{- if .Values.gitSync.ssh_key_secret }} + {{- if .Values.dags.gitSync.ssh_key_secret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} - {{- end }} {{- if .Values.webserver.extraVolumes }} {{ toYaml .Values.webserver.extraVolumes | indent 8 }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 38edff7e54b80..dc48757d17918 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -105,7 +105,7 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} containers: - {{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }} + {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" . | indent 8 }} {{- end }} - name: worker @@ -130,7 +130,7 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} -{{- if .Values.gitSync.enabled }} +{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} - name: dags mountPath: {{ template "airflow_dags_mount_path" . }} {{- end }} @@ -149,17 +149,15 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} - {{- if .Values.gitSync.enabled }} - {{- if .Values.gitSync.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} - name: dags persistentVolumeClaim: - claimName: {{ .Release.Name }}-dags - {{- else }} - {{- if .Values.gitSync.ssh_key_secret }} - {{- include "git_sync_ssh_key_volume" . | indent 8 }} - {{- end }} + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {} + {{- if .Values.dags.gitSync.ssh_key_secret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} {{- if not $persistence }} diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml index f41e66dcdabf0..2e24615ec17c2 100644 --- a/chart/tests/dags-persistent-volume-claim_test.yaml +++ b/chart/tests/dags-persistent-volume-claim_test.yaml @@ -20,38 +20,39 @@ templates: tests: - it: should not generate a document if persistence is disabled set: - gitSync: - enabled: true + dags: persistence: enabled: false asserts: - hasDocuments: count: 0 - - it: should not generate a document if git sync is disabled + - it: should not generate a document when using an existingClaim set: - gitSync: - enabled: false + dags: persistence: enabled: true + existingClaim: test-claim asserts: - hasDocuments: count: 0 - - it: should generate a document if git sync & persistence is enabled + - it: should generate a document if persistence is enabled & not using an existingClaim set: - gitSync: - enabled: true + dags: persistence: enabled: true + existingClaim: ~ asserts: - hasDocuments: count: 1 - it: should set PVC details correctly set: - gitSync: + dags: persistence: enabled: true size: 1G + existingClaim: ~ storageClass: "MyStorageClass" + accessMode: ReadWriteMany asserts: - equal: path: spec diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index cdb0925f8890c..e3203561cd522 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -28,19 +28,20 @@ tests: value: dags - it: validate the git sync container spec set: - gitSync: - enabled: true - container_name: git-sync-test - container_tag: test-tag - container_repository: test-registry/test-repo - wait: 66 - subpath: "path1/path2" - dest: "test-dest" - root: "/git-root" - rev: HEAD - depth: 1 - repo: https://github.com/apache/airflow.git - branch: test-branch + dags: + gitSync: + enabled: true + container_name: git-sync-test + container_tag: test-tag + container_repository: test-registry/test-repo + wait: 66 + subpath: "path1/path2" + dest: "test-dest" + root: "/git-root" + rev: HEAD + depth: 1 + repo: https://github.com/apache/airflow.git + branch: test-branch persistence: enabled: true asserts: @@ -71,11 +72,12 @@ tests: name: dags - it: validate if ssh params are added set: - gitSync: - enabled: true - container_name: git-sync-test - ssh_key_secret: ssh-secret - branch: test-branch + dags: + gitSync: + enabled: true + container_name: git-sync-test + ssh_key_secret: ssh-secret + branch: test-branch asserts: - contains: path: spec.template.spec.containers[1].env @@ -101,9 +103,10 @@ tests: defaultMode: 288 - it: should set username and pass env variables set: - gitSync: - enabled: true - credentials_secret: user-pass-secret + dags: + gitSync: + enabled: true + credentials_secret: user-pass-secret asserts: - contains: path: spec.template.spec.containers[1].env diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml index b506e8700e815..e28ff2c57f92e 100644 --- a/chart/tests/git-sync-webserver_test.yaml +++ b/chart/tests/git-sync-webserver_test.yaml @@ -20,29 +20,32 @@ templates: tests: - it: should add dags volume to the webserver if git sync & peristence is enabled set: - gitSync: - enabled: true - persistence: - enabled: true + dags: + gitSync: + enabled: true + persistence: + enabled: true asserts: - equal: path: spec.template.spec.volumes[1].name value: dags - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled set: - gitSync: - enabled: true - persistence: - enabled: false + dags: + gitSync: + enabled: true + persistence: + enabled: false asserts: - equal: path: spec.template.spec.volumes[1].name value: dags - it: should add git sync container to webserver if persistence is not enabled, but git sync is set: - gitSync: - enabled: true - container_name: git-sync + dags: + gitSync: + enabled: true + container_name: git-sync persistence: enabled: false asserts: @@ -51,9 +54,10 @@ tests: value: git-sync - it: should not add sync container to webserver if git sync and persistence are enabled set: - gitSync: - enabled: true - container_name: git-sync + dags: + gitSync: + enabled: true + container_name: git-sync persistence: enabled: true asserts: diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml index 677d113379f3d..9b51fbe9ef974 100644 --- a/chart/tests/git-sync-worker_test.yaml +++ b/chart/tests/git-sync-worker_test.yaml @@ -21,43 +21,47 @@ tests: - it: should add dags volume to the worker if git sync & peristence is enabled set: executor: CeleryExecutor - gitSync: - enabled: true - persistence: - enabled: true + dags: + persistence: + enabled: true + gitSync: + enabled: true asserts: - equal: path: spec.template.spec.volumes[1].name value: dags - - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled + - it: should add dags volume to the worker if git sync is enabled & peristence is disabled set: executor: CeleryExecutor - gitSync: - enabled: true - persistence: - enabled: false + dags: + gitSync: + enabled: true + persistence: + enabled: false asserts: - equal: path: spec.template.spec.volumes[1].name value: dags - - it: should add git sync container to webserver if persistence is not enabled, but git sync is + - it: should add git sync container to worker if persistence is not enabled, but git sync is set: executor: CeleryExecutor - gitSync: - enabled: true - container_name: git-sync + dags: + gitSync: + enabled: true + container_name: git-sync persistence: enabled: false asserts: - equal: path: spec.template.spec.containers[0].name value: git-sync - - it: should not add sync container to webserver if git sync and persistence are enabled + - it: should not add sync container to worker if git sync and persistence are enabled set: executor: CeleryExecutor - gitSync: - enabled: true - container_name: git-sync + dags: + gitSync: + enabled: true + container_name: git-sync persistence: enabled: true asserts: diff --git a/chart/values.yaml b/chart/values.yaml index 7ab4d5ad0853f..7a43a657b7b84 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -440,57 +440,62 @@ api: authBackend: airflow.api.auth.backend.deny_all # Git sync -gitSync: - enabled: true +dags: persistence: - # Enable persistent volumes for storing synced dags + # Enable persistent volumes for storing dags enabled: true # Volume size for dags size: 1Gi # If using a custom storageClass, pass name here storageClassName: - # git repo clone url - # ssh examples ssh://git@github.com/apache/airflow.git - # git@github.com:apache/airflow.git - # https example: https://github.com/apache/airflow.git - repo: https://github.com/apache/airflow.git - branch: v1-10-stable - rev: HEAD - root: "/git" - dest: "repo" - depth: 1 - # subpath within the repo where dags are located - # should be "" if dags are at repo root - subpath: "tests/dags" - # if your repo needs a user name password - # you can load them to a k8s secret like the one below - # --- - # apiVersion: v1 - # kind: Secret - # metadata: - # name: git-credentials - # data: - # GIT_SYNC_USERNAME: - # GIT_SYNC_PASSWORD: - # and specify the name of the secret below - #credentials_secret: git-credentials - # - # - # If you are using an ssh clone url, you can load - # the ssh private key to a k8s secret like the one below - # --- - # apiVersion: v1 - # kind: Secret - # metadata: - # name: airflow-ssh-secret - # data: - # # key needs to be gitSshKey - # gitSshKey: - # and specify the name of the secret below - #ssh_key_secret: airflow-ssh-secret - # interval between git sync attempts in seconds - wait: 60 - # git sync image details - container_repository: k8s.gcr.io/git-sync - container_tag: v3.1.6 - container_name: git-sync + # access mode of the persistent volume + accessMode: ReadWriteMany + ## the name of an existing PVC to use + existingClaim: ~ + gitSync: + enabled: true + # git repo clone url + # ssh examples ssh://git@github.com/apache/airflow.git + # git@github.com:apache/airflow.git + # https example: https://github.com/apache/airflow.git + repo: https://github.com/apache/airflow.git + branch: v1-10-stable + rev: HEAD + root: "/git" + dest: "repo" + depth: 1 + # subpath within the repo where dags are located + # should be "" if dags are at repo root + subpath: "tests/dags" + # if your repo needs a user name password + # you can load them to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: git-credentials + # data: + # GIT_SYNC_USERNAME: + # GIT_SYNC_PASSWORD: + # and specify the name of the secret below + #credentials_secret: git-credentials + # + # + # If you are using an ssh clone url, you can load + # the ssh private key to a k8s secret like the one below + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: airflow-ssh-secret + # data: + # # key needs to be gitSshKey + # gitSshKey: + # and specify the name of the secret below + #ssh_key_secret: airflow-ssh-secret + # interval between git sync attempts in seconds + wait: 60 + # git sync image details + container_repository: k8s.gcr.io/git-sync + container_tag: v3.1.6 + container_name: git-sync From c4370ea96bb77e1b559aa6e83cb55724536f8b1d Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sat, 20 Jun 2020 14:35:22 +0530 Subject: [PATCH 12/19] rename valueFields --- chart/templates/_helpers.yaml | 18 +++++++++--------- chart/templates/configmap.yaml | 16 ++++++++-------- .../scheduler/scheduler-deployment.yaml | 2 +- .../webserver/webserver-deployment.yaml | 2 +- chart/templates/workers/worker-deployment.yaml | 2 +- chart/tests/git-sync-scheduler_test.yaml | 14 +++++++------- chart/values.yaml | 12 ++++++------ 7 files changed, 33 insertions(+), 33 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 01840e0cc88b9..53b2ef2c9bf73 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -97,32 +97,32 @@ {{- define "git_sync_ssh_key_volume"}} - name: git-sync-ssh-key secret: - secretName: {{ .Values.dags.gitSync.ssh_key_secret }} + secretName: {{ .Values.dags.gitSync.sshKeySecret }} defaultMode: 288 {{- end }} {{/* Git sync container */}} {{- define "git_sync_container"}} -- name: {{ .Values.dags.gitSync.container_name }} - image: "{{ .Values.dags.gitSync.container_repository }}:{{ .Values.dags.gitSync.container_tag }}" +- name: {{ .Values.dags.gitSync.containerName }} + image: "{{ .Values.dags.gitSync.containerRepository }}:{{ .Values.dags.gitSync.containerTag }}" env: - {{- if .Values.dags.gitSync.ssh_key_secret }} + {{- if .Values.dags.gitSync.sshKeySecret }} - name: GIT_SSH_KEY_FILE value: "/etc/git-secret/ssh" - name: GIT_SYNC_SSH value: "true" - name: GIT_KNOWN_HOSTS value: "false" - {{ else if .Values.dags.gitSync.credentials_secret }} + {{ else if .Values.dags.gitSync.credentialsSecret }} - name: GIT_SYNC_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.credentials_secret }} + name: {{ .Values.dags.gitSync.credentialsSecret }} key: GIT_SYNC_USERNAME - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.credentials_secret }} + name: {{ .Values.dags.gitSync.credentialsSecret }} key: GIT_SYNC_PASSWORD {{- end }} - name: GIT_SYNC_REV @@ -144,7 +144,7 @@ volumeMounts: - name: dags mountPath: {{ .Values.dags.gitSync.root }} - {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.ssh_key_secret }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} - name: git-sync-ssh-key mountPath: /etc/git-secret/ssh readOnly: true @@ -246,7 +246,7 @@ log_connections = {{ .Values.pgbouncer.logConnections }} {{ define "airflow_dags" -}} {{- if .Values.dags.gitSync.enabled -}} -{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subpath ) }} +{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subPath ) }} {{- else -}} {{ (printf "%s/dags" .Values.airflowHome) }} {{- end -}} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index acc82d29b84a6..c7badb3f1a9f2 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -102,7 +102,7 @@ data: dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }} {{- if .Values.dags.persistence.enabled }} dags_volume_claim = {{ .Release.Name }}-dags - dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subpath }} + dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }} {{- else }} git_repo = {{ .Values.dags.gitSync.repo }} git_branch = {{ .Values.dags.gitSync.branch }} @@ -110,14 +110,14 @@ data: git_sync_depth = {{ .Values.dags.gitSync.depth }} git_sync_root = {{ .Values.dags.gitSync.root }} git_sync_dest = {{ .Values.dags.gitSync.dest }} - git_sync_container_repository = {{ .Values.dags.gitSync.container_repository }} - git_sync_container_tag = {{ .Values.dags.gitSync.container_tag }} - git_sync_init_container_name = {{ .Values.dags.gitSync.container_name }} + git_sync_container_repository = {{ .Values.dags.gitSync.containerRepository }} + git_sync_container_tag = {{ .Values.dags.gitSync.containerTag }} + git_sync_init_container_name = {{ .Values.dags.gitSync.containerName }} git_sync_run_as_user = {{ .Values.uid }} - {{- if .Values.dags.gitSync.ssh_key_secret }} - git_ssh_key_secret_name = {{ .Values.dags.gitSync.ssh_key_secret }} - {{- else if .Values.dags.gitSync.credentials_secret }} - git_sync_credentials_secret = {{ .Values.dags.gitSync.credentials_secret }} + {{- if .Values.dags.gitSync.sshKeySecret }} + git_ssh_key_secret_name = {{ .Values.dags.gitSync.sshKeySecret }} + {{- else if .Values.dags.gitSync.credentialsSecret }} + git_sync_credentials_secret = {{ .Values.dags.gitSync.credentialsSecret }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 7f7a6d9571f0a..698220dc783aa 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -182,7 +182,7 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} - {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.ssh_key_secret }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- if .Values.dags.persistence.enabled }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index c6fba61723104..9ea2bc11826f2 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -149,7 +149,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {} - {{- if .Values.dags.gitSync.ssh_key_secret }} + {{- if .Values.dags.gitSync.sshKeySecret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index dc48757d17918..439843dac92e8 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -156,7 +156,7 @@ spec: {{- else if .Values.dags.gitSync.enabled }} - name: dags emptyDir: {} - {{- if .Values.dags.gitSync.ssh_key_secret }} + {{- if .Values.dags.gitSync.sshKeySecret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} {{- end }} diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index e3203561cd522..2d2eb811c18c4 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -31,11 +31,11 @@ tests: dags: gitSync: enabled: true - container_name: git-sync-test - container_tag: test-tag - container_repository: test-registry/test-repo + containerName: git-sync-test + containerTag: test-tag + containerRepository: test-registry/test-repo wait: 66 - subpath: "path1/path2" + subPath: "path1/path2" dest: "test-dest" root: "/git-root" rev: HEAD @@ -75,8 +75,8 @@ tests: dags: gitSync: enabled: true - container_name: git-sync-test - ssh_key_secret: ssh-secret + containerName: git-sync-test + sshKeySecret: ssh-secret branch: test-branch asserts: - contains: @@ -106,7 +106,7 @@ tests: dags: gitSync: enabled: true - credentials_secret: user-pass-secret + credentialsSecret: user-pass-secret asserts: - contains: path: spec.template.spec.containers[1].env diff --git a/chart/values.yaml b/chart/values.yaml index 7a43a657b7b84..6e0c60b1f1b79 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -466,7 +466,7 @@ dags: depth: 1 # subpath within the repo where dags are located # should be "" if dags are at repo root - subpath: "tests/dags" + subPath: "tests/dags" # if your repo needs a user name password # you can load them to a k8s secret like the one below # --- @@ -478,7 +478,7 @@ dags: # GIT_SYNC_USERNAME: # GIT_SYNC_PASSWORD: # and specify the name of the secret below - #credentials_secret: git-credentials + #credentialsSecret: git-credentials # # # If you are using an ssh clone url, you can load @@ -492,10 +492,10 @@ dags: # # key needs to be gitSshKey # gitSshKey: # and specify the name of the secret below - #ssh_key_secret: airflow-ssh-secret + #sshKeySecret: airflow-ssh-secret # interval between git sync attempts in seconds wait: 60 # git sync image details - container_repository: k8s.gcr.io/git-sync - container_tag: v3.1.6 - container_name: git-sync + containerRepository: k8s.gcr.io/git-sync + containerTag: v3.1.6 + containerName: git-sync From 362e1636cc14388aaad96b19f52b6c8a47fd90ae Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sat, 20 Jun 2020 15:15:20 +0530 Subject: [PATCH 13/19] turn off git sync and dag persistence by default --- chart/templates/scheduler/scheduler-deployment.yaml | 6 +++--- chart/tests/git-sync-scheduler_test.yaml | 5 +++-- chart/values.yaml | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 698220dc783aa..1616aec161a34 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -182,9 +182,6 @@ spec: - name: config configMap: name: {{ template "airflow_config" . }} - {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} - {{- include "git_sync_ssh_key_volume" . | indent 8 }} - {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags persistentVolumeClaim: @@ -193,6 +190,9 @@ spec: - name: dags emptyDir: {} {{- end }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} {{- if not $stateful }} - name: logs emptyDir: {} diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index 2d2eb811c18c4..9074fd4c78846 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -20,8 +20,9 @@ templates: tests: - it: should add dags volume set: - gitSync: - enabled: true + dags: + gitSync: + enabled: true asserts: - equal: path: spec.template.spec.volumes[1].name diff --git a/chart/values.yaml b/chart/values.yaml index 6e0c60b1f1b79..94cbb4affeb8a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -442,8 +442,8 @@ api: # Git sync dags: persistence: - # Enable persistent volumes for storing dags - enabled: true + # Enable persistent volume for storing dags + enabled: false # Volume size for dags size: 1Gi # If using a custom storageClass, pass name here @@ -453,7 +453,7 @@ dags: ## the name of an existing PVC to use existingClaim: ~ gitSync: - enabled: true + enabled: false # git repo clone url # ssh examples ssh://git@github.com/apache/airflow.git # git@github.com:apache/airflow.git From 4f2496c84c277438c84ae6eae7317548a0737309 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sat, 20 Jun 2020 18:07:16 +0530 Subject: [PATCH 14/19] provide option to specify known_hosts --- chart/templates/_helpers.yaml | 17 +++++++++++++++-- chart/templates/configmap.yaml | 7 +++++++ chart/tests/git-sync-scheduler_test.yaml | 5 +++++ chart/tests/git-sync-webserver_test.yaml | 2 +- chart/tests/git-sync-worker_test.yaml | 4 ++-- chart/values.yaml | 5 +++++ 6 files changed, 35 insertions(+), 5 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 53b2ef2c9bf73..84d8108bac2ce 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -111,18 +111,25 @@ value: "/etc/git-secret/ssh" - name: GIT_SYNC_SSH value: "true" + {{- if .Values.dags.gitSync.knownHosts }} + - name: GIT_KNOWN_HOSTS + value: "true" + - name: GIT_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" + {{- else }} - name: GIT_KNOWN_HOSTS value: "false" + {{- end }} {{ else if .Values.dags.gitSync.credentialsSecret }} - name: GIT_SYNC_USERNAME valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.credentialsSecret }} + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GIT_SYNC_USERNAME - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.dags.gitSync.credentialsSecret }} + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GIT_SYNC_PASSWORD {{- end }} - name: GIT_SYNC_REV @@ -149,6 +156,12 @@ mountPath: /etc/git-secret/ssh readOnly: true subPath: gitSshKey + {{- if .Values.dags.gitSync.knownHosts }} + - name: config + mountPath: /etc/git-secret/known_hosts + readOnly: true + subPath: known_hosts + {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index c7badb3f1a9f2..229399c4f59f5 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -114,6 +114,9 @@ data: git_sync_container_tag = {{ .Values.dags.gitSync.containerTag }} git_sync_init_container_name = {{ .Values.dags.gitSync.containerName }} git_sync_run_as_user = {{ .Values.uid }} + {{- if .Values.dags.gitSync.knownHosts }} + git_ssh_known_hosts_configmap_name = {{ include "airflow_config" . }} + {{- end }} {{- if .Values.dags.gitSync.sshKeySecret }} git_ssh_key_secret_name = {{ .Values.dags.gitSync.sshKeySecret }} {{- else if .Values.dags.gitSync.credentialsSecret }} @@ -147,3 +150,7 @@ data: airflow_local_settings.py: | {{ .Values.scheduler.airflowLocalSettings | nindent 4 }} {{- end }} +{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }} + known_hosts: | + {{ .Values.dags.gitSync.knownHosts | nindent 4 }} +{{- end }} diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index 9074fd4c78846..7da924d2a1ea1 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -43,6 +43,9 @@ tests: depth: 1 repo: https://github.com/apache/airflow.git branch: test-branch + sshKeySecret: ~ + credentialsSecret: ~ + knownHosts: ~ persistence: enabled: true asserts: @@ -78,6 +81,7 @@ tests: enabled: true containerName: git-sync-test sshKeySecret: ssh-secret + knownHosts: ~ branch: test-branch asserts: - contains: @@ -108,6 +112,7 @@ tests: gitSync: enabled: true credentialsSecret: user-pass-secret + sshKeySecret: ~ asserts: - contains: path: spec.template.spec.containers[1].env diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml index e28ff2c57f92e..bdbb7c66c2767 100644 --- a/chart/tests/git-sync-webserver_test.yaml +++ b/chart/tests/git-sync-webserver_test.yaml @@ -45,7 +45,7 @@ tests: dags: gitSync: enabled: true - container_name: git-sync + containerName: git-sync persistence: enabled: false asserts: diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml index 9b51fbe9ef974..847a4dc3baa00 100644 --- a/chart/tests/git-sync-worker_test.yaml +++ b/chart/tests/git-sync-worker_test.yaml @@ -48,7 +48,7 @@ tests: dags: gitSync: enabled: true - container_name: git-sync + containerName: git-sync persistence: enabled: false asserts: @@ -61,7 +61,7 @@ tests: dags: gitSync: enabled: true - container_name: git-sync + containerName: git-sync persistence: enabled: true asserts: diff --git a/chart/values.yaml b/chart/values.yaml index 94cbb4affeb8a..b103c54e9cc53 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -493,6 +493,11 @@ dags: # gitSshKey: # and specify the name of the secret below #sshKeySecret: airflow-ssh-secret + # If you are using an ssh private key, you can additionally + # specify the content of your known_hosts file, example: + #knownHosts: | + # , + # , # interval between git sync attempts in seconds wait: 60 # git sync image details From a4a0cf542e2f2267458ebeeff9ef532da1022986 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Mon, 22 Jun 2020 14:35:22 +0530 Subject: [PATCH 15/19] add git-sync details into the chart documentation --- chart/README.md | 40 ++++++++++++++++++- .../scheduler/scheduler-deployment.yaml | 2 +- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/chart/README.md b/chart/README.md index fb04517ee8630..4eb80eddb8f5e 100644 --- a/chart/README.md +++ b/chart/README.md @@ -66,7 +66,7 @@ The command removes all the Kubernetes components associated with the chart and ## Updating DAGs -The recommended way to update your DAGs with this chart is to build a new docker image with the latest code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image: +The recommended way to update your DAGs with this chart is to build a new docker image with the latest DAG code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image: ```bash helm upgrade airflow . \ @@ -76,6 +76,42 @@ helm upgrade airflow . \ For local development purpose you can also build the image locally and use it via deployment method described by Breeze. +## Mounting DAGS using Git-Sync side car with Persistence enabled + +This option will use a Persistent Volume Claim with an accessMode of `ReadWriteMany`. The scheduler pod will sync DAGs from a git repository onto the PVC every configured number of seconds. The other pods will read the synced DAGs. Not all volume plugins have support for `ReadWriteMany` accessMode. Refer [Persistent Volume Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) for details + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=true \ + --set dags.gitSync.enabled=true + # you can also override the other persistence or gitSync values + # by setting the dags.persistence.* and dags.gitSync.* values + # Refer values.yaml for details +``` + +## Mounting DAGS using Git-Sync side car without Persistence +This option will use an always running Git-Sync side car on every scheduler,webserver and worker pods. The Git-Sync side car containers will sync DAGs from a git repository every configured number of seconds. If you are using the KubernetesExecutor, Git-sync will run as an initContainer on your worker pods. + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=false \ + --set dags.gitSync.enabled=true + # you can also override the other gitSync values + # by setting the dags.gitSync.* values + # Refer values.yaml for details +``` + +## Mounting DAGS from an externally populated PVC +In this approach, Airflow will read the DAGs from a PVC which has `ReadOnlyMany` or `ReadWriteMany` accessMode. You will have to ensure that the PVC is populated/updated with the required DAGs(this won't be handled by the chart). You can pass in the name of the volume claim to the chart + +```bash +helm upgrade airflow . \ + --set dags.persistence.enabled=true \ + --set dags.persistence.existingClaim=my-volume-claim + --set dags.gitSync.enabled=false +``` + + ## Parameters The following tables lists the configurable parameters of the Airflow chart and their default values. @@ -159,6 +195,8 @@ The following tables lists the configurable parameters of the Airflow chart and | `webserver.resources.requests.cpu` | CPU Request of webserver | `~` | | `webserver.resources.requests.memory` | Memory Request of webserver | `~` | | `webserver.defaultUser` | Optional default airflow user information | `{}` | +| `dags.persistence.*` | Dag persistence configutation | Refer `values.yaml` | +| `dags.gitSync.*` | Git sync configuration | Refer `values.yaml` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 1616aec161a34..933155687bf56 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -145,7 +145,7 @@ spec: subPath: airflow_local_settings.py readOnly: true {{- end }} -{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }} +{{- if .Values.dags.gitSync.enabled }} - name: dags mountPath: {{ template "airflow_dags_mount_path" . }} {{- include "git_sync_container" . | indent 8 }} From 465a018f986f8e460f4498b260d953373e44e7c7 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Thu, 25 Jun 2020 19:21:58 +0530 Subject: [PATCH 16/19] Update .gitignore Co-authored-by: Ash Berlin-Taylor --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6699e29a3365a..70ac006d31a27 100644 --- a/.gitignore +++ b/.gitignore @@ -201,5 +201,5 @@ log.txt* Chart.lock # Chart dependencies -requirements.lock +chart/requirements.lock **/charts/*.tgz From 0dc34711ba436d30c0f1bda3573b6a32fc7d606e Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Thu, 2 Jul 2020 22:00:04 +0530 Subject: [PATCH 17/19] make git sync max failures configurable --- chart/templates/_helpers.yaml | 2 ++ chart/tests/git-sync-scheduler_test.yaml | 3 +++ chart/values.yaml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 84d8108bac2ce..898924f2ff930 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -148,6 +148,8 @@ value: "true" - name: GIT_SYNC_WAIT value: {{ .Values.dags.gitSync.wait | quote }} + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} volumeMounts: - name: dags mountPath: {{ .Values.dags.gitSync.root }} diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml index 7da924d2a1ea1..10d2288ed4876 100644 --- a/chart/tests/git-sync-scheduler_test.yaml +++ b/chart/tests/git-sync-scheduler_test.yaml @@ -36,6 +36,7 @@ tests: containerTag: test-tag containerRepository: test-registry/test-repo wait: 66 + maxFailures: 70 subPath: "path1/path2" dest: "test-dest" root: "/git-root" @@ -71,6 +72,8 @@ tests: value: "true" - name: GIT_SYNC_WAIT value: "66" + - name: GIT_SYNC_MAX_SYNC_FAILURES + value: "70" volumeMounts: - mountPath: /git-root name: dags diff --git a/chart/values.yaml b/chart/values.yaml index b103c54e9cc53..8fa7ab63292b5 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -464,6 +464,8 @@ dags: root: "/git" dest: "repo" depth: 1 + # the number of consecutive failures allowed before aborting + maxFailures: 0 # subpath within the repo where dags are located # should be "" if dags are at repo root subPath: "tests/dags" From b7a40e87a2e2d35ba93899de5b9665b3e60f61fb Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sun, 5 Jul 2020 14:06:20 +0530 Subject: [PATCH 18/19] Apply suggestions from code review Co-authored-by: Jarek Potiuk --- chart/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chart/README.md b/chart/README.md index 4eb80eddb8f5e..c0d7847624680 100644 --- a/chart/README.md +++ b/chart/README.md @@ -86,7 +86,7 @@ helm upgrade airflow . \ --set dags.gitSync.enabled=true # you can also override the other persistence or gitSync values # by setting the dags.persistence.* and dags.gitSync.* values - # Refer values.yaml for details + # Please refer to values.yaml for details ``` ## Mounting DAGS using Git-Sync side car without Persistence @@ -195,8 +195,8 @@ The following tables lists the configurable parameters of the Airflow chart and | `webserver.resources.requests.cpu` | CPU Request of webserver | `~` | | `webserver.resources.requests.memory` | Memory Request of webserver | `~` | | `webserver.defaultUser` | Optional default airflow user information | `{}` | -| `dags.persistence.*` | Dag persistence configutation | Refer `values.yaml` | -| `dags.gitSync.*` | Git sync configuration | Refer `values.yaml` | +| `dags.persistence.*` | Dag persistence configutation | Please refer to `values.yaml` | +| `dags.gitSync.*` | Git sync configuration | Please refer to `values.yaml` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, From eabf6c0779860fa9c390296d138ecc7ff1e27287 Mon Sep 17 00:00:00 2001 From: Aneesh Joseph Date: Sun, 5 Jul 2020 14:59:32 +0530 Subject: [PATCH 19/19] add back requirements.lock --- .gitignore | 1 - chart/requirements.lock | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 chart/requirements.lock diff --git a/.gitignore b/.gitignore index 70ac006d31a27..c7ae00a28ccfd 100644 --- a/.gitignore +++ b/.gitignore @@ -201,5 +201,4 @@ log.txt* Chart.lock # Chart dependencies -chart/requirements.lock **/charts/*.tgz diff --git a/chart/requirements.lock b/chart/requirements.lock new file mode 100644 index 0000000000000..f86e6968adf29 --- /dev/null +++ b/chart/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://kubernetes-charts.storage.googleapis.com + version: 6.3.12 +digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af +generated: "2020-06-21T19:11:53.498134738+02:00"