From f1ba8959bc16018d72f4163113eac084c38c1dcc Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Thu, 18 Jun 2020 14:10:37 +0530
Subject: [PATCH 01/19] add git sync sidecars

---
 chart/templates/_helpers.yaml                 | 61 +++++++++++++++++++
 chart/templates/configmap.yaml                | 26 +++++++-
 .../dags-persistent-volume-claim.yaml         | 41 +++++++++++++
 .../scheduler/scheduler-deployment.yaml       | 18 ++++++
 .../webserver/webserver-deployment.yaml       | 20 ++++++
 .../templates/workers/worker-deployment.yaml  | 20 ++++++
 chart/values.yaml                             | 32 ++++++++++
 7 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 chart/templates/dags-persistent-volume-claim.yaml

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index ac121a4260c40..ead6e526e2e3f 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -93,6 +93,54 @@
   {{ end }}
 {{- end }}
 
+{{/*  Git ssh key volume */}}
+{{- define "git_sync_ssh_key_volume"}}
+- name: git-sync-ssh-key
+  secret:
+    secretName: {{ .Values.gitSync.ssh_key_secret }}
+    defaultMode: 288
+{{- end }}
+
+{{/*  Git sync container */}}
+{{- define "git_sync_container"}}
+- name: {{ .Values.gitSync.container_name }}
+  image: "{{ .Values.gitSync.container_repository }}:{{ .Values.gitSync.container_tag }}"
+  env:
+    {{- if .Values.gitSync.ssh_key_secret }}
+    - name: GIT_SSH_KEY_FILE
+      value: "/etc/git-secret/ssh"
+    - name: GIT_SYNC_SSH
+      value: "true"
+    {{- end }}
+    - name: GIT_SYNC_REV
+      value: {{ .Values.gitSync.rev | quote }}
+    - name: GIT_SYNC_BRANCH
+      value: {{ .Values.gitSync.branch | quote }}
+    - name: GIT_SYNC_REPO
+      value: {{ .Values.gitSync.repo | quote }}
+    - name: GIT_SYNC_DEPTH
+      value: {{ .Values.gitSync.depth | quote }}
+    - name: GIT_SYNC_ROOT
+      value: {{ .Values.gitSync.root | quote }}
+    - name: GIT_SYNC_DEST
+      value: {{ .Values.gitSync.dest | quote }}
+    - name: GIT_SYNC_ADD_USER
+      value: "true"
+    - name: GIT_KNOWN_HOSTS
+      value: "false"
+    - name: GIT_SYNC_WAIT
+      value: {{ .Values.gitSync.wait | quote }}
+  volumeMounts:
+  - name: dags
+    mountPath: {{ .Values.gitSync.root }}
+  {{- if and  .Values.gitSync.enabled  .Values.gitSync.ssh_key_secret }}
+  - name: git-sync-ssh-key
+    mountPath: /etc/git-secret/ssh
+    readOnly: true
+    subPath: gitSshKey
+  {{- end }}
+{{- end }}
+
 # This helper will change when customers deploy a new image.
 {{ define "airflow_image" -}}
 {{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }}
@@ -185,9 +233,22 @@ log_connections = {{ .Values.pgbouncer.logConnections }}
 {{ (printf "%s/logs" .Values.airflowHome) | quote }}
 {{- end }}
 
+{{ define "airflow_dags" -}}
+{{- if .Values.gitSync.enabled -}}
+{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.gitSync.dest .Values.gitSync.subpath ) }}
+{{- else -}}
+{{ (printf "%s/dags" .Values.airflowHome) }}
+{{- end -}}
+{{- end -}}
+
+{{ define "airflow_dags_mount_path" -}}
+{{ (printf "%s/dags" .Values.airflowHome) }}
+{{- end }}
+
 {{ define "airflow_config_path" -}}
 {{ (printf "%s/airflow.cfg" .Values.airflowHome) | quote }}
 {{- end }}
+
 {{ define "airflow_webserver_config_path" -}}
 {{ (printf "%s/webserver_config.py" .Values.airflowHome) | quote }}
 {{- end }}
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index 86affe8fcbda8..c4a0e2d2e8a5d 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -35,6 +35,7 @@ data:
   # These are system-specified config overrides.
   airflow.cfg: |
     [core]
+    dags_folder = {{ include "airflow_dags" . }}
     load_examples = False
     colored_console_log = False
     executor = {{ .Values.executor }}
@@ -92,8 +93,31 @@ data:
     worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }}
     worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount
     image_pull_secrets = {{ template "registry_secret" . }}
-    dags_in_image = True
+    dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }}
     delete_worker_pods = True
+    run_as_user = {{ .Values.uid }}
+    {{- if .Values.gitSync.enabled }}
+    git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }}
+    dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }}
+
+    {{- if .Values.gitSync.persistence.enabled }}
+    dags_volume_claim = {{ .Release.Name }}-dags
+    dags_volume_subpath = {{.Values.gitSync.dest }}/{{ .Values.gitSync.subpath }}
+    {{- else }}
+    git_repo = {{ .Values.gitSync.repo }}
+    git_branch = {{ .Values.gitSync.branch }}
+    git_sync_rev = {{ .Values.gitSync.rev }}
+    git_sync_depth = {{ .Values.gitSync.depth }}
+    git_sync_root = {{ .Values.gitSync.root }}
+    git_sync_dest = {{ .Values.gitSync.dest }}
+    git_sync_container_repository = {{ .Values.gitSync.container_repository }}
+    git_sync_container_tag = {{ .Values.gitSync.container_tag }}
+    git_sync_init_container_name = {{ .Values.gitSync.container_name }}
+    {{- if .Values.gitSync.ssh_key_secret }}
+    git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }}
+    {{- end }}
+    {{- end }}
+    {{- end }}
 
     [kubernetes_secrets]
     AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }}
diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml
new file mode 100644
index 0000000000000..12e7d7d5b74a4
--- /dev/null
+++ b/chart/templates/dags-persistent-volume-claim.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.gitSync.persistence.enabled }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Release.Name }}-dags
+  labels:
+    tier: airflow
+    component: dags-pvc
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+spec:
+  accessModes: ["ReadWriteMany"]
+  resources:
+    requests:
+      storage: {{ .Values.gitSync.persistence.size | quote }}
+  {{- if .Values.gitSync.persistence.storageClass }}
+  {{- if (eq "-" .Values.gitSync.persistence.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.gitSync.persistence.storageClass }}"
+  {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index d5c3a0662bd50..c56342ca200e9 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -144,6 +144,11 @@ spec:
               mountPath: {{ template "airflow_local_setting_path" . }}
               subPath: airflow_local_settings.py
               readOnly: true
+{{- end }}
+{{- if .Values.gitSync.enabled }}
+            - name: dags
+              mountPath: {{ template "airflow_dags_mount_path" . }}
+        {{- include "git_sync_container" . | indent 8 }}
 {{- end }}
         # Always start the garbage collector sidecar.
         - name: scheduler-gc
@@ -177,6 +182,19 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
+        {{- if and  .Values.gitSync.enabled  .Values.gitSync.ssh_key_secret }}
+        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
+        {{- end }}
+        {{- if .Values.gitSync.enabled }}
+        {{- if .Values.gitSync.persistence.enabled }}
+        - name: dags
+          persistentVolumeClaim:
+            claimName: {{ .Release.Name }}-dags
+        {{- else }}
+        - name: dags
+          emptyDir: {}
+        {{- end }}
+        {{- end }}
 {{- if not $stateful }}
         - name: logs
           emptyDir: {}
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index b4c9714bec9ed..fd376be90891c 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -82,6 +82,9 @@ spec:
           {{- include "custom_airflow_environment" . | indent 10 }}
           {{- include "standard_airflow_environment" . | indent 10 }}
       containers:
+{{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }}
+{{- include "git_sync_container" . | indent 8 }}
+{{- end }}
         - name: webserver
           image: {{ template "airflow_image" . }}
           imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
@@ -105,6 +108,10 @@ spec:
               subPath: airflow_local_settings.py
               readOnly: true
 {{- end }}
+{{- if .Values.gitSync.enabled }}
+            - name: dags
+              mountPath: {{ template "airflow_dags_mount_path" . }}
+{{- end }}
 {{- if .Values.webserver.extraVolumeMounts }}
 {{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }}
 {{- end }}
@@ -134,6 +141,19 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
+        {{- if .Values.gitSync.enabled }}
+        {{- if .Values.gitSync.persistence.enabled }}
+        - name: dags
+          persistentVolumeClaim:
+            claimName: {{ .Release.Name }}-dags
+        {{- else }}
+        {{- if  .Values.gitSync.ssh_key_secret }}
+        {{- include "git_ssh_key_volume" . | indent 8 }}
+        {{- end }}
+        - name: dags
+          emptyDir: {}
+        {{- end }}
+        {{- end }}
 {{- if .Values.webserver.extraVolumes }}
 {{ toYaml .Values.webserver.extraVolumes | indent 8 }}
 {{- end }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index d5043a7735978..38edff7e54b80 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -105,6 +105,9 @@ spec:
           {{- include "custom_airflow_environment" . | indent 10 }}
           {{- include "standard_airflow_environment" . | indent 10 }}
       containers:
+        {{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }}
+        {{- include "git_sync_container" . | indent 8 }}
+        {{- end }}
         - name: worker
           image: {{ template "airflow_image" . }}
           imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
@@ -126,6 +129,10 @@ spec:
               mountPath: {{ template "airflow_local_setting_path" . }}
               subPath: airflow_local_settings.py
               readOnly: true
+{{- end }}
+{{- if .Values.gitSync.enabled }}
+            - name: dags
+              mountPath: {{ template "airflow_dags_mount_path" . }}
 {{- end }}
           env:
           {{- include "custom_airflow_environment" . | indent 10 }}
@@ -142,6 +149,19 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
+        {{- if .Values.gitSync.enabled }}
+        {{- if .Values.gitSync.persistence.enabled }}
+        - name: dags
+          persistentVolumeClaim:
+            claimName: {{ .Release.Name }}-dags
+        {{- else }}
+        {{- if  .Values.gitSync.ssh_key_secret }}
+        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
+        {{- end }}
+        - name: dags
+          emptyDir: {}
+        {{- end }}
+        {{- end }}
 {{- if not $persistence }}
         - name: logs
           emptyDir: {}
diff --git a/chart/values.yaml b/chart/values.yaml
index dc13064927bfd..acf4ff7631242 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -438,3 +438,35 @@ postgresql:
 # Authentication backend used for the experimental API
 api:
   authBackend: airflow.api.auth.backend.deny_all
+
+# Git sync
+gitSync:
+  enabled: true
+  persistence:
+    # Enable persistent volumes for storing synced dags
+    enabled: true
+    # Volume size for dags
+    size: 1Gi
+    # If using a custom storageClass, pass name here
+    storageClassName:
+  # git repo clone url
+  # ssh examples ssh://git@github.com/apache/airflow.git
+  # git@github.com:apache/airflow.git
+  # https example: https://github.com/apache/airflow.git
+  repo: https://github.com/apache/airflow.git
+  branch: v1-10-stable
+  rev: HEAD
+  root: "/git"
+  dest: "repo"
+  depth: 1
+  # subpath within the repo where dags are located
+  # should be "" if dags are at repo root
+  subpath: "tests/dags"
+  #TODO Handle username password secrets
+  #TODO Handle known_hosts
+  #credentials_secret:
+  #ssh_key_secret: git-ssh
+  wait: 60
+  container_repository: k8s.gcr.io/git-sync
+  container_tag: v3.1.6
+  container_name: git-sync-clone

From c8fb2ec850e8243b46c1ada44c54d8d7b5b6f316 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 10:41:13 +0530
Subject: [PATCH 02/19] add a helm test

---
 .github/workflows/ci.yml                      | 11 ++++
 .gitignore                                    |  1 +
 chart/requirements.lock                       |  6 --
 .../dags-persistent-volume-claim.yaml         |  2 +-
 .../dags-persistent-volume-claim_test.yaml    | 63 +++++++++++++++++++
 chart/tests/git-sync-scheduler_test.yaml      | 26 ++++++++
 scripts/ci/ci_run_helm_testing.sh             | 28 +++++++++
 7 files changed, 130 insertions(+), 7 deletions(-)
 delete mode 100644 chart/requirements.lock
 create mode 100644 chart/tests/dags-persistent-volume-claim_test.yaml
 create mode 100644 chart/tests/git-sync-scheduler_test.yaml
 create mode 100755 scripts/ci/ci_run_helm_testing.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f3b0e351b9a8b..2c7702e74e925 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -363,6 +363,17 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
       - name: "Tests"
         run: ./scripts/ci/ci_run_airflow_testing.sh
 
+  helm-tests:
+    timeout-minutes: 60
+    name: "Checks: Helm tests"
+    runs-on: ubuntu-latest
+    env:
+      CI_JOB_TYPE: "Tests"
+    steps:
+      - uses: actions/checkout@master
+      - name: "Helm Tests"
+        run: ./scripts/ci/ci_run_helm_testing.sh
+
   requirements:
     timeout-minutes: 80
     name: "Requirements"
diff --git a/.gitignore b/.gitignore
index c7ae00a28ccfd..6699e29a3365a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -201,4 +201,5 @@ log.txt*
 Chart.lock
 
 # Chart dependencies
+requirements.lock
 **/charts/*.tgz
diff --git a/chart/requirements.lock b/chart/requirements.lock
deleted file mode 100644
index f86e6968adf29..0000000000000
--- a/chart/requirements.lock
+++ /dev/null
@@ -1,6 +0,0 @@
-dependencies:
-- name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com
-  version: 6.3.12
-digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af
-generated: "2020-06-21T19:11:53.498134738+02:00"
diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml
index 12e7d7d5b74a4..79f9b7329b8f2 100644
--- a/chart/templates/dags-persistent-volume-claim.yaml
+++ b/chart/templates/dags-persistent-volume-claim.yaml
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-{{- if .Values.gitSync.persistence.enabled }}
+{{- if and .Values.gitSync.enabled .Values.gitSync.persistence.enabled }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml
new file mode 100644
index 0000000000000..ffc003934f2ff
--- /dev/null
+++ b/chart/tests/dags-persistent-volume-claim_test.yaml
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+templates:
+  - dags-persistent-volume-claim.yaml
+tests:
+  - it: should  not generate a document if persistence is disabled
+    set:
+      gitSync:
+        enabled: true
+        persistence:
+          enabled: false
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: should  not generate a document if git sync is disabled
+    set:
+      gitSync:
+        enabled: false
+        persistence:
+          enabled: true
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: should  generate a document if git sync & persistence is enabled
+    set:
+      gitSync:
+        enabled: true
+        persistence:
+          enabled: true
+    asserts:
+      - hasDocuments:
+          count: 1
+  - it: should set PVC details correctly
+    set:
+      gitSync:
+        persistence:
+          enabled: true
+          size: 1G
+          storageClass: "MyStorageClass"
+    asserts:
+      - equal:
+          path: spec
+          value:
+            accessModes: ["ReadWriteMany"]
+            resources:
+              requests:
+                storage: 1G
+            storageClassName: "MyStorageClass"
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
new file mode 100644
index 0000000000000..483caf3a5b8d2
--- /dev/null
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+tests:
+  - it: should add dags volume
+    set:
+      gitSync:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[1].name
+          value: dags
+        template: scheduler/scheduler-deployment.yaml
diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/ci_run_helm_testing.sh
new file mode 100755
index 0000000000000..43cc2d41961b8
--- /dev/null
+++ b/scripts/ci/ci_run_helm_testing.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Running helm tests"
+
+CHART_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../chart/"
+
+echo "Chart directory is $CHART_DIR"
+
+docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \
+  --entrypoint /bin/sh \
+  divya12/helm-unittest \
+  -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ."

From e17430c585b8572aed10627ca7f87d43a856ba95 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 17:04:58 +0530
Subject: [PATCH 03/19] add more tests

---
 .../webserver/webserver-deployment.yaml       |  6 +-
 .../dags-persistent-volume-claim_test.yaml    |  2 +-
 chart/tests/git-sync-scheduler_test.yaml      | 49 +++++++++++++-
 chart/tests/git-sync-webserver_test.yaml      | 62 +++++++++++++++++
 chart/tests/git-sync-worker_test.yaml         | 66 +++++++++++++++++++
 chart/values.yaml                             |  4 +-
 scripts/ci/ci_run_helm_testing.sh             |  2 +-
 7 files changed, 183 insertions(+), 8 deletions(-)
 create mode 100644 chart/tests/git-sync-webserver_test.yaml
 create mode 100644 chart/tests/git-sync-worker_test.yaml

diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index fd376be90891c..9aa6e078e8b18 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -147,11 +147,11 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Release.Name }}-dags
         {{- else }}
-        {{- if  .Values.gitSync.ssh_key_secret }}
-        {{- include "git_ssh_key_volume" . | indent 8 }}
-        {{- end }}
         - name: dags
           emptyDir: {}
+        {{- if  .Values.gitSync.ssh_key_secret }}
+        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
+        {{- end }}
         {{- end }}
         {{- end }}
 {{- if .Values.webserver.extraVolumes }}
diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml
index ffc003934f2ff..f41e66dcdabf0 100644
--- a/chart/tests/dags-persistent-volume-claim_test.yaml
+++ b/chart/tests/dags-persistent-volume-claim_test.yaml
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+---
 templates:
   - dags-persistent-volume-claim.yaml
 tests:
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index 483caf3a5b8d2..85569df1677ac 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -14,6 +14,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+---
+templates:
+  - scheduler/scheduler-deployment.yaml
 tests:
   - it: should add dags volume
     set:
@@ -23,4 +26,48 @@ tests:
       - equal:
           path: spec.template.spec.volumes[1].name
           value: dags
-        template: scheduler/scheduler-deployment.yaml
+  - it: validate the git sync container spec
+    set:
+      gitSync:
+        enabled: true
+        container_name: git-sync-test
+        container_tag: test-tag
+        container_repository: test-registry/test-repo
+        wait: 66
+        subpath: "path1/path2"
+        dest: "test-dest"
+        root: "/git-root"
+        rev: HEAD
+        depth: 1
+        repo: https://github.com/apache/airflow.git
+        branch: test-branch
+        persistence:
+          enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1]
+          value:
+            name: git-sync-test
+            image: test-registry/test-repo:test-tag
+            env:
+              - name: GIT_SYNC_REV
+                value: HEAD
+              - name: GIT_SYNC_BRANCH
+                value: test-branch
+              - name: GIT_SYNC_REPO
+                value: https://github.com/apache/airflow.git
+              - name: GIT_SYNC_DEPTH
+                value: "1"
+              - name: GIT_SYNC_ROOT
+                value: /git-root
+              - name: GIT_SYNC_DEST
+                value: test-dest
+              - name: GIT_SYNC_ADD_USER
+                value: "true"
+              - name: GIT_KNOWN_HOSTS
+                value: "false"
+              - name: GIT_SYNC_WAIT
+                value: "66"
+            volumeMounts:
+              - mountPath: /git-root
+                name: dags
diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml
new file mode 100644
index 0000000000000..b506e8700e815
--- /dev/null
+++ b/chart/tests/git-sync-webserver_test.yaml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+templates:
+  - webserver/webserver-deployment.yaml
+tests:
+  - it: should add dags volume to the webserver if git sync & peristence is enabled
+    set:
+      gitSync:
+        enabled: true
+      persistence:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[1].name
+          value: dags
+  - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled
+    set:
+      gitSync:
+        enabled: true
+      persistence:
+        enabled: false
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[1].name
+          value: dags
+  - it: should add git sync container to webserver if persistence is not enabled, but git sync is
+    set:
+      gitSync:
+        enabled: true
+        container_name: git-sync
+        persistence:
+          enabled: false
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].name
+          value: git-sync
+  - it: should not add sync container to webserver if git sync and persistence are enabled
+    set:
+      gitSync:
+        enabled: true
+        container_name: git-sync
+        persistence:
+          enabled: true
+    asserts:
+      - notEqual:
+          path: spec.template.spec.containers[0].name
+          value: git-sync
diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml
new file mode 100644
index 0000000000000..677d113379f3d
--- /dev/null
+++ b/chart/tests/git-sync-worker_test.yaml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+templates:
+  - workers/worker-deployment.yaml
+tests:
+  - it: should add dags volume to the worker if git sync & peristence is enabled
+    set:
+      executor: CeleryExecutor
+      gitSync:
+        enabled: true
+      persistence:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[1].name
+          value: dags
+  - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled
+    set:
+      executor: CeleryExecutor
+      gitSync:
+        enabled: true
+      persistence:
+        enabled: false
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[1].name
+          value: dags
+  - it: should add git sync container to webserver if persistence is not enabled, but git sync is
+    set:
+      executor: CeleryExecutor
+      gitSync:
+        enabled: true
+        container_name: git-sync
+        persistence:
+          enabled: false
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].name
+          value: git-sync
+  - it: should not add sync container to webserver if git sync and persistence are enabled
+    set:
+      executor: CeleryExecutor
+      gitSync:
+        enabled: true
+        container_name: git-sync
+        persistence:
+          enabled: true
+    asserts:
+      - notEqual:
+          path: spec.template.spec.containers[0].name
+          value: git-sync
diff --git a/chart/values.yaml b/chart/values.yaml
index acf4ff7631242..d6df79d13fe24 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+---
 # Default values for airflow.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
@@ -469,4 +469,4 @@ gitSync:
   wait: 60
   container_repository: k8s.gcr.io/git-sync
   container_tag: v3.1.6
-  container_name: git-sync-clone
+  container_name: git-sync
diff --git a/scripts/ci/ci_run_helm_testing.sh b/scripts/ci/ci_run_helm_testing.sh
index 43cc2d41961b8..0a267d49aac5a 100755
--- a/scripts/ci/ci_run_helm_testing.sh
+++ b/scripts/ci/ci_run_helm_testing.sh
@@ -24,5 +24,5 @@ echo "Chart directory is $CHART_DIR"
 
 docker run -w /airflow-chart -v "$CHART_DIR":/airflow-chart \
   --entrypoint /bin/sh \
-  divya12/helm-unittest \
+  aneeshkj/helm-unittest \
   -c "helm repo add stable https://kubernetes-charts.storage.googleapis.com; helm dependency update ; helm unittest ."

From 351d69ad42f609e8a74b2a7c2722347df218d1ad Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 20:26:26 +0530
Subject: [PATCH 04/19] allow users to provide git username and pass via  a k8s
 secrets

---
 chart/templates/_helpers.yaml            | 15 ++++++-
 chart/tests/git-sync-scheduler_test.yaml | 54 +++++++++++++++++++++++-
 chart/values.yaml                        | 32 ++++++++++++--
 3 files changed, 93 insertions(+), 8 deletions(-)

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index ead6e526e2e3f..015c53ead5223 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -111,6 +111,19 @@
       value: "/etc/git-secret/ssh"
     - name: GIT_SYNC_SSH
       value: "true"
+    - name: GIT_KNOWN_HOSTS
+      value: "false"
+    {{ else if .Values.gitSync.credentials_secret }}
+    - name: GIT_SYNC_USERNAME
+      valueFrom:
+        secretKeyRef:
+          name: {{ .Values.gitSync.credentials_secret }}
+          key: GIT_SYNC_USERNAME
+    - name: GIT_SYNC_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: {{ .Values.gitSync.credentials_secret }}
+          key: GIT_SYNC_PASSWORD
     {{- end }}
     - name: GIT_SYNC_REV
       value: {{ .Values.gitSync.rev | quote }}
@@ -126,8 +139,6 @@
       value: {{ .Values.gitSync.dest | quote }}
     - name: GIT_SYNC_ADD_USER
       value: "true"
-    - name: GIT_KNOWN_HOSTS
-      value: "false"
     - name: GIT_SYNC_WAIT
       value: {{ .Values.gitSync.wait | quote }}
   volumeMounts:
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index 85569df1677ac..cdb0925f8890c 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -64,10 +64,60 @@ tests:
                 value: test-dest
               - name: GIT_SYNC_ADD_USER
                 value: "true"
-              - name: GIT_KNOWN_HOSTS
-                value: "false"
               - name: GIT_SYNC_WAIT
                 value: "66"
             volumeMounts:
               - mountPath: /git-root
                 name: dags
+  - it: validate if ssh params are added
+    set:
+      gitSync:
+        enabled: true
+        container_name: git-sync-test
+        ssh_key_secret: ssh-secret
+        branch: test-branch
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: GIT_SSH_KEY_FILE
+            value: "/etc/git-secret/ssh"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: GIT_SYNC_SSH
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: GIT_KNOWN_HOSTS
+            value: "false"
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: git-sync-ssh-key
+            secret:
+              secretName: ssh-secret
+              defaultMode: 288
+  - it: should set username and pass env variables
+    set:
+      gitSync:
+        enabled: true
+        credentials_secret: user-pass-secret
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: GIT_SYNC_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: user-pass-secret
+                key: GIT_SYNC_USERNAME
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: GIT_SYNC_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: user-pass-secret
+                key: GIT_SYNC_PASSWORD
diff --git a/chart/values.yaml b/chart/values.yaml
index d6df79d13fe24..7ab4d5ad0853f 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -462,11 +462,35 @@ gitSync:
   # subpath within the repo where dags are located
   # should be "" if dags are at repo root
   subpath: "tests/dags"
-  #TODO Handle username password secrets
-  #TODO Handle known_hosts
-  #credentials_secret:
-  #ssh_key_secret: git-ssh
+  # if your repo needs a user name password
+  # you can load them to a k8s secret like the one below
+  #   ---
+  #   apiVersion: v1
+  #   kind: Secret
+  #   metadata:
+  #     name: git-credentials
+  #   data:
+  #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
+  #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+  # and specify the name of the secret below
+  #credentials_secret: git-credentials
+  #
+  #
+  # If you are using an ssh clone url, you can load
+  # the ssh private key to a k8s secret like the one below
+  #   ---
+  #   apiVersion: v1
+  #   kind: Secret
+  #   metadata:
+  #     name: airflow-ssh-secret
+  #   data:
+  #     # key needs to be gitSshKey
+  #     gitSshKey: <base64_encoded_data>
+  # and specify the name of the secret below
+  #ssh_key_secret: airflow-ssh-secret
+  # interval between git sync attempts in seconds
   wait: 60
+  # git sync image details
   container_repository: k8s.gcr.io/git-sync
   container_tag: v3.1.6
   container_name: git-sync

From dd08690da054ffe3addb2874e47fb52f6db13acf Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 20:54:59 +0530
Subject: [PATCH 05/19] set default values for airflow worker repository & tag

---
 chart/templates/configmap.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index c4a0e2d2e8a5d..c3ef69ce03996 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -88,8 +88,8 @@ data:
     namespace = {{ .Release.Namespace }}
     airflow_configmap = {{ include "airflow_config" . }}
     airflow_local_settings_configmap = {{ include "airflow_config" . }}
-    worker_container_repository = {{ .Values.images.airflow.repository }}
-    worker_container_tag = {{ .Values.images.airflow.tag }}
+    worker_container_repository = {{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}
+    worker_container_tag = {{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}
     worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }}
     worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount
     image_pull_secrets = {{ template "registry_secret" . }}

From 6786d5e471a9d266758142681bdfc93d57722dd1 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 21:05:55 +0530
Subject: [PATCH 06/19] change ci timeout

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2c7702e74e925..5714869f54d06 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -364,7 +364,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         run: ./scripts/ci/ci_run_airflow_testing.sh
 
   helm-tests:
-    timeout-minutes: 60
+    timeout-minutes: 5
     name: "Checks: Helm tests"
     runs-on: ubuntu-latest
     env:

From 784e2856dcc0be0dbe78d08c439dbbbcf5ffa1d0 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 21:35:50 +0530
Subject: [PATCH 07/19] fix link

---
 CI.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CI.rst b/CI.rst
index 11a2f578329d3..a1a38ba9bd27a 100644
--- a/CI.rst
+++ b/CI.rst
@@ -33,7 +33,7 @@ environments we use. Most of our CI jobs are written as bash scripts which are e
 the CI jobs and we are mapping all the CI-specific environment variables to generic "CI" variables.
 The only two places where CI-specific code might be are:
 
-- CI-specific declaration file (for example it is `<.github/workflow/ci.yml>`_ for GitHub Actions
+- CI-specific declaration file (for example it is `<.github/workflows/ci.yml>`_ for GitHub Actions
 - The ``get_environment_for_builds_on_ci`` function in `<scripts/ci/libraries/_build_images.sh>`_ where mapping is
   performed from the CI-environment specific to generic values. Example for that is CI_EVENT_TYPE variable
   which determines whether we are running a ``push``. ``schedule`` or ``pull_request`` kind of CI job. For

From 60f768f8b152ef0fce486060c0c0e938a217be91 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 21:47:24 +0530
Subject: [PATCH 08/19] add credentials_secret to airflow.cfg configmap

---
 chart/templates/configmap.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index c3ef69ce03996..c5cc86d4063c8 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -115,6 +115,8 @@ data:
     git_sync_init_container_name = {{ .Values.gitSync.container_name }}
     {{- if .Values.gitSync.ssh_key_secret }}
     git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }}
+    {{- else if .Values.gitSync.credentials_secret }}
+    git_sync_credentials_secret = {{ .Values.gitSync.credentials_secret }}
     {{- end }}
     {{- end }}
     {{- end }}

From 0e69d66e71739d27a7a9c36755c5418060e606be Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 22:48:07 +0530
Subject: [PATCH 09/19] set GIT_SYNC_ADD_USER on kubernetes worker pods, set
 uid

---
 airflow/kubernetes/worker_configuration.py | 4 ++++
 chart/templates/configmap.yaml             | 1 +
 2 files changed, 5 insertions(+)

diff --git a/airflow/kubernetes/worker_configuration.py b/airflow/kubernetes/worker_configuration.py
index 732d5accbc56e..383a705bf4b4e 100644
--- a/airflow/kubernetes/worker_configuration.py
+++ b/airflow/kubernetes/worker_configuration.py
@@ -127,6 +127,10 @@ def _get_init_containers(self) -> List[k8s.V1Container]:
                     name='GIT_SSH_KEY_FILE',
                     value='/etc/git-secret/ssh'
                 ),
+                k8s.V1EnvVar(
+                    name='GIT_SYNC_ADD_USER',
+                    value='true'
+                ),
                 k8s.V1EnvVar(
                     name='GIT_SYNC_SSH',
                     value='true'
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index c5cc86d4063c8..7a824729dea59 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -113,6 +113,7 @@ data:
     git_sync_container_repository = {{ .Values.gitSync.container_repository }}
     git_sync_container_tag = {{ .Values.gitSync.container_tag }}
     git_sync_init_container_name = {{ .Values.gitSync.container_name }}
+    git_sync_run_as_user = {{ .Values.uid }}
     {{- if .Values.gitSync.ssh_key_secret }}
     git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }}
     {{- else if .Values.gitSync.credentials_secret }}

From 22b1b1584aa4f03aa1e90b3667d691ce6923a841 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Fri, 19 Jun 2020 23:12:17 +0530
Subject: [PATCH 10/19] add fsGroup to webserver and kubernete workers

---
 chart/templates/configmap.yaml                      | 1 +
 chart/templates/webserver/webserver-deployment.yaml | 1 +
 tests/kubernetes/test_worker_configuration.py       | 4 ++++
 3 files changed, 6 insertions(+)

diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index 7a824729dea59..6ede82c24b1a4 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -96,6 +96,7 @@ data:
     dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }}
     delete_worker_pods = True
     run_as_user = {{ .Values.uid }}
+    fs_group = {{ .Values.gid }}
     {{- if .Values.gitSync.enabled }}
     git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }}
     dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }}
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index 9aa6e078e8b18..53db3bab275a9 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -68,6 +68,7 @@ spec:
       restartPolicy: Always
       securityContext:
         runAsUser: {{ .Values.uid }}
+        fsGroup: {{ .Values.gid }}
       {{- if or .Values.registry.secretName .Values.registry.connection }}
       imagePullSecrets:
         - name: {{ template "registry_secret" . }}
diff --git a/tests/kubernetes/test_worker_configuration.py b/tests/kubernetes/test_worker_configuration.py
index b223eefdf0d09..98dd0c706e952 100644
--- a/tests/kubernetes/test_worker_configuration.py
+++ b/tests/kubernetes/test_worker_configuration.py
@@ -244,6 +244,7 @@ def test_init_environment_using_git_sync_ssh_without_known_hosts(self):
         env = init_containers[0].env
 
         self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env)
+        self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_SSH', value='true'), env)
 
@@ -264,6 +265,7 @@ def test_init_environment_using_git_sync_ssh_with_known_hosts(self):
         env = init_containers[0].env
 
         self.assertIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env)
+        self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env)
         self.assertIn(k8s.V1EnvVar(
             name='GIT_SSH_KNOWN_HOSTS_FILE',
@@ -290,6 +292,7 @@ def test_init_environment_using_git_sync_user_without_known_hosts(self):
         env = init_containers[0].env
 
         self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env)
+        self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='false'), env)
@@ -318,6 +321,7 @@ def test_init_environment_using_git_sync_user_with_known_hosts(self):
         env = init_containers[0].env
 
         self.assertNotIn(k8s.V1EnvVar(name='GIT_SSH_KEY_FILE', value='/etc/git-secret/ssh'), env)
+        self.assertNotIn(k8s.V1EnvVar(name='GIT_SYNC_ADD_USER', value='true'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_USERNAME', value='git_user'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_SYNC_PASSWORD', value='git_password'), env)
         self.assertIn(k8s.V1EnvVar(name='GIT_KNOWN_HOSTS', value='true'), env)

From 8fdc1bd0c0a2a156c85aab7c9a60992c45343b4c Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sat, 20 Jun 2020 10:27:26 +0530
Subject: [PATCH 11/19] move gitSync to dags.gitSync

---
 chart/templates/_helpers.yaml                 |  44 ++++----
 chart/templates/configmap.yaml                |  35 +++---
 .../dags-persistent-volume-claim.yaml         |  12 +--
 .../scheduler/scheduler-deployment.yaml       |  12 +--
 .../webserver/webserver-deployment.yaml       |  12 +--
 .../templates/workers/worker-deployment.yaml  |  16 ++-
 .../dags-persistent-volume-claim_test.yaml    |  19 ++--
 chart/tests/git-sync-scheduler_test.yaml      |  45 ++++----
 chart/tests/git-sync-webserver_test.yaml      |  32 +++---
 chart/tests/git-sync-worker_test.yaml         |  38 ++++---
 chart/values.yaml                             | 101 +++++++++---------
 11 files changed, 192 insertions(+), 174 deletions(-)

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 015c53ead5223..01840e0cc88b9 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -97,54 +97,54 @@
 {{- define "git_sync_ssh_key_volume"}}
 - name: git-sync-ssh-key
   secret:
-    secretName: {{ .Values.gitSync.ssh_key_secret }}
+    secretName: {{ .Values.dags.gitSync.ssh_key_secret }}
     defaultMode: 288
 {{- end }}
 
 {{/*  Git sync container */}}
 {{- define "git_sync_container"}}
-- name: {{ .Values.gitSync.container_name }}
-  image: "{{ .Values.gitSync.container_repository }}:{{ .Values.gitSync.container_tag }}"
+- name: {{ .Values.dags.gitSync.container_name }}
+  image: "{{ .Values.dags.gitSync.container_repository }}:{{ .Values.dags.gitSync.container_tag }}"
   env:
-    {{- if .Values.gitSync.ssh_key_secret }}
+    {{- if .Values.dags.gitSync.ssh_key_secret }}
     - name: GIT_SSH_KEY_FILE
       value: "/etc/git-secret/ssh"
     - name: GIT_SYNC_SSH
       value: "true"
     - name: GIT_KNOWN_HOSTS
       value: "false"
-    {{ else if .Values.gitSync.credentials_secret }}
+    {{ else if .Values.dags.gitSync.credentials_secret }}
     - name: GIT_SYNC_USERNAME
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.gitSync.credentials_secret }}
+          name: {{ .Values.dags.gitSync.credentials_secret }}
           key: GIT_SYNC_USERNAME
     - name: GIT_SYNC_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.gitSync.credentials_secret }}
+          name: {{ .Values.dags.gitSync.credentials_secret }}
           key: GIT_SYNC_PASSWORD
     {{- end }}
     - name: GIT_SYNC_REV
-      value: {{ .Values.gitSync.rev | quote }}
+      value: {{ .Values.dags.gitSync.rev | quote }}
     - name: GIT_SYNC_BRANCH
-      value: {{ .Values.gitSync.branch | quote }}
+      value: {{ .Values.dags.gitSync.branch | quote }}
     - name: GIT_SYNC_REPO
-      value: {{ .Values.gitSync.repo | quote }}
+      value: {{ .Values.dags.gitSync.repo | quote }}
     - name: GIT_SYNC_DEPTH
-      value: {{ .Values.gitSync.depth | quote }}
+      value: {{ .Values.dags.gitSync.depth | quote }}
     - name: GIT_SYNC_ROOT
-      value: {{ .Values.gitSync.root | quote }}
+      value: {{ .Values.dags.gitSync.root | quote }}
     - name: GIT_SYNC_DEST
-      value: {{ .Values.gitSync.dest | quote }}
+      value: {{ .Values.dags.gitSync.dest | quote }}
     - name: GIT_SYNC_ADD_USER
       value: "true"
     - name: GIT_SYNC_WAIT
-      value: {{ .Values.gitSync.wait | quote }}
+      value: {{ .Values.dags.gitSync.wait | quote }}
   volumeMounts:
   - name: dags
-    mountPath: {{ .Values.gitSync.root }}
-  {{- if and  .Values.gitSync.enabled  .Values.gitSync.ssh_key_secret }}
+    mountPath: {{ .Values.dags.gitSync.root }}
+  {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.ssh_key_secret }}
   - name: git-sync-ssh-key
     mountPath: /etc/git-secret/ssh
     readOnly: true
@@ -245,13 +245,21 @@ log_connections = {{ .Values.pgbouncer.logConnections }}
 {{- end }}
 
 {{ define "airflow_dags" -}}
-{{- if .Values.gitSync.enabled -}}
-{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.gitSync.dest .Values.gitSync.subpath ) }}
+{{- if .Values.dags.gitSync.enabled -}}
+{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subpath ) }}
 {{- else -}}
 {{ (printf "%s/dags" .Values.airflowHome) }}
 {{- end -}}
 {{- end -}}
 
+{{ define "airflow_dags_volume_claim" -}}
+{{- if and .Values.dags.persistence.enabled .Values.dags.persistence.existingClaim -}}
+{{ .Values.dags.persistence.existingClaim }}
+{{- else -}}
+{{ .Release.Name }}-dags
+{{- end -}}
+{{- end -}}
+
 {{ define "airflow_dags_mount_path" -}}
 {{ (printf "%s/dags" .Values.airflowHome) }}
 {{- end }}
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index 6ede82c24b1a4..acc82d29b84a6 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -93,32 +93,31 @@ data:
     worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }}
     worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount
     image_pull_secrets = {{ template "registry_secret" . }}
-    dags_in_image = {{ if .Values.gitSync.enabled }}False{{ else }}True{{ end }}
+    dags_in_image = {{ if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}False{{ else }}True{{ end }}
     delete_worker_pods = True
     run_as_user = {{ .Values.uid }}
     fs_group = {{ .Values.gid }}
-    {{- if .Values.gitSync.enabled }}
+    {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}
     git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }}
     dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }}
-
-    {{- if .Values.gitSync.persistence.enabled }}
+    {{- if .Values.dags.persistence.enabled }}
     dags_volume_claim = {{ .Release.Name }}-dags
-    dags_volume_subpath = {{.Values.gitSync.dest }}/{{ .Values.gitSync.subpath }}
+    dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subpath }}
     {{- else }}
-    git_repo = {{ .Values.gitSync.repo }}
-    git_branch = {{ .Values.gitSync.branch }}
-    git_sync_rev = {{ .Values.gitSync.rev }}
-    git_sync_depth = {{ .Values.gitSync.depth }}
-    git_sync_root = {{ .Values.gitSync.root }}
-    git_sync_dest = {{ .Values.gitSync.dest }}
-    git_sync_container_repository = {{ .Values.gitSync.container_repository }}
-    git_sync_container_tag = {{ .Values.gitSync.container_tag }}
-    git_sync_init_container_name = {{ .Values.gitSync.container_name }}
+    git_repo = {{ .Values.dags.gitSync.repo }}
+    git_branch = {{ .Values.dags.gitSync.branch }}
+    git_sync_rev = {{ .Values.dags.gitSync.rev }}
+    git_sync_depth = {{ .Values.dags.gitSync.depth }}
+    git_sync_root = {{ .Values.dags.gitSync.root }}
+    git_sync_dest = {{ .Values.dags.gitSync.dest }}
+    git_sync_container_repository = {{ .Values.dags.gitSync.container_repository }}
+    git_sync_container_tag = {{ .Values.dags.gitSync.container_tag }}
+    git_sync_init_container_name = {{ .Values.dags.gitSync.container_name }}
     git_sync_run_as_user = {{ .Values.uid }}
-    {{- if .Values.gitSync.ssh_key_secret }}
-    git_ssh_key_secret_name = {{ .Values.gitSync.ssh_key_secret }}
-    {{- else if .Values.gitSync.credentials_secret }}
-    git_sync_credentials_secret = {{ .Values.gitSync.credentials_secret }}
+    {{- if .Values.dags.gitSync.ssh_key_secret }}
+    git_ssh_key_secret_name = {{ .Values.dags.gitSync.ssh_key_secret }}
+    {{- else if .Values.dags.gitSync.credentials_secret }}
+    git_sync_credentials_secret = {{ .Values.dags.gitSync.credentials_secret }}
     {{- end }}
     {{- end }}
     {{- end }}
diff --git a/chart/templates/dags-persistent-volume-claim.yaml b/chart/templates/dags-persistent-volume-claim.yaml
index 79f9b7329b8f2..53ea550ddbc52 100644
--- a/chart/templates/dags-persistent-volume-claim.yaml
+++ b/chart/templates/dags-persistent-volume-claim.yaml
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-{{- if and .Values.gitSync.enabled .Values.gitSync.persistence.enabled }}
+{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
@@ -27,15 +27,15 @@ metadata:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     heritage: {{ .Release.Service }}
 spec:
-  accessModes: ["ReadWriteMany"]
+  accessModes: [{{ .Values.dags.persistence.accessMode | quote }}]
   resources:
     requests:
-      storage: {{ .Values.gitSync.persistence.size | quote }}
-  {{- if .Values.gitSync.persistence.storageClass }}
-  {{- if (eq "-" .Values.gitSync.persistence.storageClass) }}
+      storage: {{ .Values.dags.persistence.size | quote }}
+  {{- if .Values.dags.persistence.storageClass }}
+  {{- if (eq "-" .Values.dags.persistence.storageClass) }}
   storageClassName: ""
   {{- else }}
-  storageClassName: "{{ .Values.gitSync.persistence.storageClass }}"
+  storageClassName: "{{ .Values.dags.persistence.storageClass }}"
   {{- end }}
   {{- end }}
 {{- end }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index c56342ca200e9..7f7a6d9571f0a 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -145,7 +145,7 @@ spec:
               subPath: airflow_local_settings.py
               readOnly: true
 {{- end }}
-{{- if .Values.gitSync.enabled }}
+{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
             - name: dags
               mountPath: {{ template "airflow_dags_mount_path" . }}
         {{- include "git_sync_container" . | indent 8 }}
@@ -182,19 +182,17 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
-        {{- if and  .Values.gitSync.enabled  .Values.gitSync.ssh_key_secret }}
+        {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.ssh_key_secret }}
         {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
-        {{- if .Values.gitSync.enabled }}
-        {{- if .Values.gitSync.persistence.enabled }}
+        {{- if .Values.dags.persistence.enabled }}
         - name: dags
           persistentVolumeClaim:
-            claimName: {{ .Release.Name }}-dags
-        {{- else }}
+            claimName: {{ template "airflow_dags_volume_claim" . }}
+        {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
           emptyDir: {}
         {{- end }}
-        {{- end }}
 {{- if not $stateful }}
         - name: logs
           emptyDir: {}
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index 53db3bab275a9..c6fba61723104 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -83,7 +83,7 @@ spec:
           {{- include "custom_airflow_environment" . | indent 10 }}
           {{- include "standard_airflow_environment" . | indent 10 }}
       containers:
-{{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }}
+{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
 {{- include "git_sync_container" . | indent 8 }}
 {{- end }}
         - name: webserver
@@ -109,7 +109,7 @@ spec:
               subPath: airflow_local_settings.py
               readOnly: true
 {{- end }}
-{{- if .Values.gitSync.enabled }}
+{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}
             - name: dags
               mountPath: {{ template "airflow_dags_mount_path" . }}
 {{- end }}
@@ -142,19 +142,17 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
-        {{- if .Values.gitSync.enabled }}
-        {{- if .Values.gitSync.persistence.enabled }}
+        {{- if .Values.dags.persistence.enabled }}
         - name: dags
           persistentVolumeClaim:
             claimName: {{ .Release.Name }}-dags
-        {{- else }}
+        {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
           emptyDir: {}
-        {{- if  .Values.gitSync.ssh_key_secret }}
+        {{- if  .Values.dags.gitSync.ssh_key_secret }}
         {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- end }}
-        {{- end }}
 {{- if .Values.webserver.extraVolumes }}
 {{ toYaml .Values.webserver.extraVolumes | indent 8 }}
 {{- end }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index 38edff7e54b80..dc48757d17918 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -105,7 +105,7 @@ spec:
           {{- include "custom_airflow_environment" . | indent 10 }}
           {{- include "standard_airflow_environment" . | indent 10 }}
       containers:
-        {{- if and (.Values.gitSync.enabled) (not .Values.gitSync.persistence.enabled) }}
+        {{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
         {{- include "git_sync_container" . | indent 8 }}
         {{- end }}
         - name: worker
@@ -130,7 +130,7 @@ spec:
               subPath: airflow_local_settings.py
               readOnly: true
 {{- end }}
-{{- if .Values.gitSync.enabled }}
+{{- if  or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
             - name: dags
               mountPath: {{ template "airflow_dags_mount_path" . }}
 {{- end }}
@@ -149,17 +149,15 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
-        {{- if .Values.gitSync.enabled }}
-        {{- if .Values.gitSync.persistence.enabled }}
+        {{- if .Values.dags.persistence.enabled }}
         - name: dags
           persistentVolumeClaim:
-            claimName: {{ .Release.Name }}-dags
-        {{- else }}
-        {{- if  .Values.gitSync.ssh_key_secret }}
-        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
-        {{- end }}
+            claimName: {{ template "airflow_dags_volume_claim" . }}
+        {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
           emptyDir: {}
+        {{- if  .Values.dags.gitSync.ssh_key_secret }}
+        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- end }}
 {{- if not $persistence }}
diff --git a/chart/tests/dags-persistent-volume-claim_test.yaml b/chart/tests/dags-persistent-volume-claim_test.yaml
index f41e66dcdabf0..2e24615ec17c2 100644
--- a/chart/tests/dags-persistent-volume-claim_test.yaml
+++ b/chart/tests/dags-persistent-volume-claim_test.yaml
@@ -20,38 +20,39 @@ templates:
 tests:
   - it: should  not generate a document if persistence is disabled
     set:
-      gitSync:
-        enabled: true
+      dags:
         persistence:
           enabled: false
     asserts:
       - hasDocuments:
           count: 0
-  - it: should  not generate a document if git sync is disabled
+  - it: should  not generate a document when using an existingClaim
     set:
-      gitSync:
-        enabled: false
+      dags:
         persistence:
           enabled: true
+          existingClaim: test-claim
     asserts:
       - hasDocuments:
           count: 0
-  - it: should  generate a document if git sync & persistence is enabled
+  - it: should  generate a document if persistence is enabled & not using an existingClaim
     set:
-      gitSync:
-        enabled: true
+      dags:
         persistence:
           enabled: true
+          existingClaim: ~
     asserts:
       - hasDocuments:
           count: 1
   - it: should set PVC details correctly
     set:
-      gitSync:
+      dags:
         persistence:
           enabled: true
           size: 1G
+          existingClaim: ~
           storageClass: "MyStorageClass"
+          accessMode: ReadWriteMany
     asserts:
       - equal:
           path: spec
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index cdb0925f8890c..e3203561cd522 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -28,19 +28,20 @@ tests:
           value: dags
   - it: validate the git sync container spec
     set:
-      gitSync:
-        enabled: true
-        container_name: git-sync-test
-        container_tag: test-tag
-        container_repository: test-registry/test-repo
-        wait: 66
-        subpath: "path1/path2"
-        dest: "test-dest"
-        root: "/git-root"
-        rev: HEAD
-        depth: 1
-        repo: https://github.com/apache/airflow.git
-        branch: test-branch
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync-test
+          container_tag: test-tag
+          container_repository: test-registry/test-repo
+          wait: 66
+          subpath: "path1/path2"
+          dest: "test-dest"
+          root: "/git-root"
+          rev: HEAD
+          depth: 1
+          repo: https://github.com/apache/airflow.git
+          branch: test-branch
         persistence:
           enabled: true
     asserts:
@@ -71,11 +72,12 @@ tests:
                 name: dags
   - it: validate if ssh params are added
     set:
-      gitSync:
-        enabled: true
-        container_name: git-sync-test
-        ssh_key_secret: ssh-secret
-        branch: test-branch
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync-test
+          ssh_key_secret: ssh-secret
+          branch: test-branch
     asserts:
       - contains:
           path: spec.template.spec.containers[1].env
@@ -101,9 +103,10 @@ tests:
               defaultMode: 288
   - it: should set username and pass env variables
     set:
-      gitSync:
-        enabled: true
-        credentials_secret: user-pass-secret
+      dags:
+        gitSync:
+          enabled: true
+          credentials_secret: user-pass-secret
     asserts:
       - contains:
           path: spec.template.spec.containers[1].env
diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml
index b506e8700e815..e28ff2c57f92e 100644
--- a/chart/tests/git-sync-webserver_test.yaml
+++ b/chart/tests/git-sync-webserver_test.yaml
@@ -20,29 +20,32 @@ templates:
 tests:
   - it: should add dags volume to the webserver if git sync & peristence is enabled
     set:
-      gitSync:
-        enabled: true
-      persistence:
-        enabled: true
+      dags:
+        gitSync:
+          enabled: true
+        persistence:
+          enabled: true
     asserts:
       - equal:
           path: spec.template.spec.volumes[1].name
           value: dags
   - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled
     set:
-      gitSync:
-        enabled: true
-      persistence:
-        enabled: false
+      dags:
+        gitSync:
+          enabled: true
+        persistence:
+          enabled: false
     asserts:
       - equal:
           path: spec.template.spec.volumes[1].name
           value: dags
   - it: should add git sync container to webserver if persistence is not enabled, but git sync is
     set:
-      gitSync:
-        enabled: true
-        container_name: git-sync
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync
         persistence:
           enabled: false
     asserts:
@@ -51,9 +54,10 @@ tests:
           value: git-sync
   - it: should not add sync container to webserver if git sync and persistence are enabled
     set:
-      gitSync:
-        enabled: true
-        container_name: git-sync
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync
         persistence:
           enabled: true
     asserts:
diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml
index 677d113379f3d..9b51fbe9ef974 100644
--- a/chart/tests/git-sync-worker_test.yaml
+++ b/chart/tests/git-sync-worker_test.yaml
@@ -21,43 +21,47 @@ tests:
   - it: should add dags volume to the worker if git sync & peristence is enabled
     set:
       executor: CeleryExecutor
-      gitSync:
-        enabled: true
-      persistence:
-        enabled: true
+      dags:
+        persistence:
+          enabled: true
+        gitSync:
+          enabled: true
     asserts:
       - equal:
           path: spec.template.spec.volumes[1].name
           value: dags
-  - it: should add dags volume to the webserver if git sync is enabled & peristence is disabled
+  - it: should add dags volume to the worker if git sync is enabled & peristence is disabled
     set:
       executor: CeleryExecutor
-      gitSync:
-        enabled: true
-      persistence:
-        enabled: false
+      dags:
+        gitSync:
+          enabled: true
+        persistence:
+          enabled: false
     asserts:
       - equal:
           path: spec.template.spec.volumes[1].name
           value: dags
-  - it: should add git sync container to webserver if persistence is not enabled, but git sync is
+  - it: should add git sync container to worker if persistence is not enabled, but git sync is
     set:
       executor: CeleryExecutor
-      gitSync:
-        enabled: true
-        container_name: git-sync
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync
         persistence:
           enabled: false
     asserts:
       - equal:
           path: spec.template.spec.containers[0].name
           value: git-sync
-  - it: should not add sync container to webserver if git sync and persistence are enabled
+  - it: should not add sync container to worker if git sync and persistence are enabled
     set:
       executor: CeleryExecutor
-      gitSync:
-        enabled: true
-        container_name: git-sync
+      dags:
+        gitSync:
+          enabled: true
+          container_name: git-sync
         persistence:
           enabled: true
     asserts:
diff --git a/chart/values.yaml b/chart/values.yaml
index 7ab4d5ad0853f..7a43a657b7b84 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -440,57 +440,62 @@ api:
   authBackend: airflow.api.auth.backend.deny_all
 
 # Git sync
-gitSync:
-  enabled: true
+dags:
   persistence:
-    # Enable persistent volumes for storing synced dags
+    # Enable persistent volumes for storing dags
     enabled: true
     # Volume size for dags
     size: 1Gi
     # If using a custom storageClass, pass name here
     storageClassName:
-  # git repo clone url
-  # ssh examples ssh://git@github.com/apache/airflow.git
-  # git@github.com:apache/airflow.git
-  # https example: https://github.com/apache/airflow.git
-  repo: https://github.com/apache/airflow.git
-  branch: v1-10-stable
-  rev: HEAD
-  root: "/git"
-  dest: "repo"
-  depth: 1
-  # subpath within the repo where dags are located
-  # should be "" if dags are at repo root
-  subpath: "tests/dags"
-  # if your repo needs a user name password
-  # you can load them to a k8s secret like the one below
-  #   ---
-  #   apiVersion: v1
-  #   kind: Secret
-  #   metadata:
-  #     name: git-credentials
-  #   data:
-  #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
-  #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
-  # and specify the name of the secret below
-  #credentials_secret: git-credentials
-  #
-  #
-  # If you are using an ssh clone url, you can load
-  # the ssh private key to a k8s secret like the one below
-  #   ---
-  #   apiVersion: v1
-  #   kind: Secret
-  #   metadata:
-  #     name: airflow-ssh-secret
-  #   data:
-  #     # key needs to be gitSshKey
-  #     gitSshKey: <base64_encoded_data>
-  # and specify the name of the secret below
-  #ssh_key_secret: airflow-ssh-secret
-  # interval between git sync attempts in seconds
-  wait: 60
-  # git sync image details
-  container_repository: k8s.gcr.io/git-sync
-  container_tag: v3.1.6
-  container_name: git-sync
+    # access mode of the persistent volume
+    accessMode: ReadWriteMany
+    ## the name of an existing PVC to use
+    existingClaim: ~
+  gitSync:
+    enabled: true
+    # git repo clone url
+    # ssh examples ssh://git@github.com/apache/airflow.git
+    # git@github.com:apache/airflow.git
+    # https example: https://github.com/apache/airflow.git
+    repo: https://github.com/apache/airflow.git
+    branch: v1-10-stable
+    rev: HEAD
+    root: "/git"
+    dest: "repo"
+    depth: 1
+    # subpath within the repo where dags are located
+    # should be "" if dags are at repo root
+    subpath: "tests/dags"
+    # if your repo needs a user name password
+    # you can load them to a k8s secret like the one below
+    #   ---
+    #   apiVersion: v1
+    #   kind: Secret
+    #   metadata:
+    #     name: git-credentials
+    #   data:
+    #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
+    #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+    # and specify the name of the secret below
+    #credentials_secret: git-credentials
+    #
+    #
+    # If you are using an ssh clone url, you can load
+    # the ssh private key to a k8s secret like the one below
+    #   ---
+    #   apiVersion: v1
+    #   kind: Secret
+    #   metadata:
+    #     name: airflow-ssh-secret
+    #   data:
+    #     # key needs to be gitSshKey
+    #     gitSshKey: <base64_encoded_data>
+    # and specify the name of the secret below
+    #ssh_key_secret: airflow-ssh-secret
+    # interval between git sync attempts in seconds
+    wait: 60
+    # git sync image details
+    container_repository: k8s.gcr.io/git-sync
+    container_tag: v3.1.6
+    container_name: git-sync

From c4370ea96bb77e1b559aa6e83cb55724536f8b1d Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sat, 20 Jun 2020 14:35:22 +0530
Subject: [PATCH 12/19] rename valueFields

---
 chart/templates/_helpers.yaml                  | 18 +++++++++---------
 chart/templates/configmap.yaml                 | 16 ++++++++--------
 .../scheduler/scheduler-deployment.yaml        |  2 +-
 .../webserver/webserver-deployment.yaml        |  2 +-
 chart/templates/workers/worker-deployment.yaml |  2 +-
 chart/tests/git-sync-scheduler_test.yaml       | 14 +++++++-------
 chart/values.yaml                              | 12 ++++++------
 7 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 01840e0cc88b9..53b2ef2c9bf73 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -97,32 +97,32 @@
 {{- define "git_sync_ssh_key_volume"}}
 - name: git-sync-ssh-key
   secret:
-    secretName: {{ .Values.dags.gitSync.ssh_key_secret }}
+    secretName: {{ .Values.dags.gitSync.sshKeySecret }}
     defaultMode: 288
 {{- end }}
 
 {{/*  Git sync container */}}
 {{- define "git_sync_container"}}
-- name: {{ .Values.dags.gitSync.container_name }}
-  image: "{{ .Values.dags.gitSync.container_repository }}:{{ .Values.dags.gitSync.container_tag }}"
+- name: {{ .Values.dags.gitSync.containerName }}
+  image: "{{ .Values.dags.gitSync.containerRepository }}:{{ .Values.dags.gitSync.containerTag }}"
   env:
-    {{- if .Values.dags.gitSync.ssh_key_secret }}
+    {{- if .Values.dags.gitSync.sshKeySecret }}
     - name: GIT_SSH_KEY_FILE
       value: "/etc/git-secret/ssh"
     - name: GIT_SYNC_SSH
       value: "true"
     - name: GIT_KNOWN_HOSTS
       value: "false"
-    {{ else if .Values.dags.gitSync.credentials_secret }}
+    {{ else if .Values.dags.gitSync.credentialsSecret }}
     - name: GIT_SYNC_USERNAME
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.credentials_secret }}
+          name: {{ .Values.dags.gitSync.credentialsSecret }}
           key: GIT_SYNC_USERNAME
     - name: GIT_SYNC_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.credentials_secret }}
+          name: {{ .Values.dags.gitSync.credentialsSecret }}
           key: GIT_SYNC_PASSWORD
     {{- end }}
     - name: GIT_SYNC_REV
@@ -144,7 +144,7 @@
   volumeMounts:
   - name: dags
     mountPath: {{ .Values.dags.gitSync.root }}
-  {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.ssh_key_secret }}
+  {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret }}
   - name: git-sync-ssh-key
     mountPath: /etc/git-secret/ssh
     readOnly: true
@@ -246,7 +246,7 @@ log_connections = {{ .Values.pgbouncer.logConnections }}
 
 {{ define "airflow_dags" -}}
 {{- if .Values.dags.gitSync.enabled -}}
-{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subpath ) }}
+{{ (printf "%s/dags/%s/%s" .Values.airflowHome .Values.dags.gitSync.dest .Values.dags.gitSync.subPath ) }}
 {{- else -}}
 {{ (printf "%s/dags" .Values.airflowHome) }}
 {{- end -}}
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index acc82d29b84a6..c7badb3f1a9f2 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -102,7 +102,7 @@ data:
     dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }}
     {{- if .Values.dags.persistence.enabled }}
     dags_volume_claim = {{ .Release.Name }}-dags
-    dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subpath }}
+    dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }}
     {{- else }}
     git_repo = {{ .Values.dags.gitSync.repo }}
     git_branch = {{ .Values.dags.gitSync.branch }}
@@ -110,14 +110,14 @@ data:
     git_sync_depth = {{ .Values.dags.gitSync.depth }}
     git_sync_root = {{ .Values.dags.gitSync.root }}
     git_sync_dest = {{ .Values.dags.gitSync.dest }}
-    git_sync_container_repository = {{ .Values.dags.gitSync.container_repository }}
-    git_sync_container_tag = {{ .Values.dags.gitSync.container_tag }}
-    git_sync_init_container_name = {{ .Values.dags.gitSync.container_name }}
+    git_sync_container_repository = {{ .Values.dags.gitSync.containerRepository }}
+    git_sync_container_tag = {{ .Values.dags.gitSync.containerTag }}
+    git_sync_init_container_name = {{ .Values.dags.gitSync.containerName }}
     git_sync_run_as_user = {{ .Values.uid }}
-    {{- if .Values.dags.gitSync.ssh_key_secret }}
-    git_ssh_key_secret_name = {{ .Values.dags.gitSync.ssh_key_secret }}
-    {{- else if .Values.dags.gitSync.credentials_secret }}
-    git_sync_credentials_secret = {{ .Values.dags.gitSync.credentials_secret }}
+    {{- if .Values.dags.gitSync.sshKeySecret }}
+    git_ssh_key_secret_name = {{ .Values.dags.gitSync.sshKeySecret }}
+    {{- else if .Values.dags.gitSync.credentialsSecret }}
+    git_sync_credentials_secret = {{ .Values.dags.gitSync.credentialsSecret }}
     {{- end }}
     {{- end }}
     {{- end }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index 7f7a6d9571f0a..698220dc783aa 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -182,7 +182,7 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
-        {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.ssh_key_secret }}
+        {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret }}
         {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- if .Values.dags.persistence.enabled }}
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index c6fba61723104..9ea2bc11826f2 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -149,7 +149,7 @@ spec:
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
           emptyDir: {}
-        {{- if  .Values.dags.gitSync.ssh_key_secret }}
+        {{- if  .Values.dags.gitSync.sshKeySecret }}
         {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- end }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index dc48757d17918..439843dac92e8 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -156,7 +156,7 @@ spec:
         {{- else if .Values.dags.gitSync.enabled }}
         - name: dags
           emptyDir: {}
-        {{- if  .Values.dags.gitSync.ssh_key_secret }}
+        {{- if  .Values.dags.gitSync.sshKeySecret }}
         {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- end }}
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index e3203561cd522..2d2eb811c18c4 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -31,11 +31,11 @@ tests:
       dags:
         gitSync:
           enabled: true
-          container_name: git-sync-test
-          container_tag: test-tag
-          container_repository: test-registry/test-repo
+          containerName: git-sync-test
+          containerTag: test-tag
+          containerRepository: test-registry/test-repo
           wait: 66
-          subpath: "path1/path2"
+          subPath: "path1/path2"
           dest: "test-dest"
           root: "/git-root"
           rev: HEAD
@@ -75,8 +75,8 @@ tests:
       dags:
         gitSync:
           enabled: true
-          container_name: git-sync-test
-          ssh_key_secret: ssh-secret
+          containerName: git-sync-test
+          sshKeySecret: ssh-secret
           branch: test-branch
     asserts:
       - contains:
@@ -106,7 +106,7 @@ tests:
       dags:
         gitSync:
           enabled: true
-          credentials_secret: user-pass-secret
+          credentialsSecret: user-pass-secret
     asserts:
       - contains:
           path: spec.template.spec.containers[1].env
diff --git a/chart/values.yaml b/chart/values.yaml
index 7a43a657b7b84..6e0c60b1f1b79 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -466,7 +466,7 @@ dags:
     depth: 1
     # subpath within the repo where dags are located
     # should be "" if dags are at repo root
-    subpath: "tests/dags"
+    subPath: "tests/dags"
     # if your repo needs a user name password
     # you can load them to a k8s secret like the one below
     #   ---
@@ -478,7 +478,7 @@ dags:
     #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
     #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
     # and specify the name of the secret below
-    #credentials_secret: git-credentials
+    #credentialsSecret: git-credentials
     #
     #
     # If you are using an ssh clone url, you can load
@@ -492,10 +492,10 @@ dags:
     #     # key needs to be gitSshKey
     #     gitSshKey: <base64_encoded_data>
     # and specify the name of the secret below
-    #ssh_key_secret: airflow-ssh-secret
+    #sshKeySecret: airflow-ssh-secret
     # interval between git sync attempts in seconds
     wait: 60
     # git sync image details
-    container_repository: k8s.gcr.io/git-sync
-    container_tag: v3.1.6
-    container_name: git-sync
+    containerRepository: k8s.gcr.io/git-sync
+    containerTag: v3.1.6
+    containerName: git-sync

From 362e1636cc14388aaad96b19f52b6c8a47fd90ae Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sat, 20 Jun 2020 15:15:20 +0530
Subject: [PATCH 13/19] turn off git sync and dag persistence by default

---
 chart/templates/scheduler/scheduler-deployment.yaml | 6 +++---
 chart/tests/git-sync-scheduler_test.yaml            | 5 +++--
 chart/values.yaml                                   | 6 +++---
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index 698220dc783aa..1616aec161a34 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -182,9 +182,6 @@ spec:
         - name: config
           configMap:
             name: {{ template "airflow_config" . }}
-        {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret }}
-        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
-        {{- end }}
         {{- if .Values.dags.persistence.enabled }}
         - name: dags
           persistentVolumeClaim:
@@ -193,6 +190,9 @@ spec:
         - name: dags
           emptyDir: {}
         {{- end }}
+        {{- if and  .Values.dags.gitSync.enabled  .Values.dags.gitSync.sshKeySecret }}
+        {{- include "git_sync_ssh_key_volume" . | indent 8 }}
+        {{- end }}
 {{- if not $stateful }}
         - name: logs
           emptyDir: {}
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index 2d2eb811c18c4..9074fd4c78846 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -20,8 +20,9 @@ templates:
 tests:
   - it: should add dags volume
     set:
-      gitSync:
-        enabled: true
+      dags:
+        gitSync:
+          enabled: true
     asserts:
       - equal:
           path: spec.template.spec.volumes[1].name
diff --git a/chart/values.yaml b/chart/values.yaml
index 6e0c60b1f1b79..94cbb4affeb8a 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -442,8 +442,8 @@ api:
 # Git sync
 dags:
   persistence:
-    # Enable persistent volumes for storing dags
-    enabled: true
+    # Enable persistent volume for storing dags
+    enabled: false
     # Volume size for dags
     size: 1Gi
     # If using a custom storageClass, pass name here
@@ -453,7 +453,7 @@ dags:
     ## the name of an existing PVC to use
     existingClaim: ~
   gitSync:
-    enabled: true
+    enabled: false
     # git repo clone url
     # ssh examples ssh://git@github.com/apache/airflow.git
     # git@github.com:apache/airflow.git

From 4f2496c84c277438c84ae6eae7317548a0737309 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sat, 20 Jun 2020 18:07:16 +0530
Subject: [PATCH 14/19] provide option to specify known_hosts

---
 chart/templates/_helpers.yaml            | 17 +++++++++++++++--
 chart/templates/configmap.yaml           |  7 +++++++
 chart/tests/git-sync-scheduler_test.yaml |  5 +++++
 chart/tests/git-sync-webserver_test.yaml |  2 +-
 chart/tests/git-sync-worker_test.yaml    |  4 ++--
 chart/values.yaml                        |  5 +++++
 6 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 53b2ef2c9bf73..84d8108bac2ce 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -111,18 +111,25 @@
       value: "/etc/git-secret/ssh"
     - name: GIT_SYNC_SSH
       value: "true"
+    {{- if .Values.dags.gitSync.knownHosts }}
+    - name: GIT_KNOWN_HOSTS
+      value: "true"
+    - name: GIT_SSH_KNOWN_HOSTS_FILE
+      value: "/etc/git-secret/known_hosts"
+    {{- else }}
     - name: GIT_KNOWN_HOSTS
       value: "false"
+    {{- end }}
     {{ else if .Values.dags.gitSync.credentialsSecret }}
     - name: GIT_SYNC_USERNAME
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.credentialsSecret }}
+          name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
           key: GIT_SYNC_USERNAME
     - name: GIT_SYNC_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: {{ .Values.dags.gitSync.credentialsSecret }}
+          name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
           key: GIT_SYNC_PASSWORD
     {{- end }}
     - name: GIT_SYNC_REV
@@ -149,6 +156,12 @@
     mountPath: /etc/git-secret/ssh
     readOnly: true
     subPath: gitSshKey
+  {{- if .Values.dags.gitSync.knownHosts }}
+  - name: config
+    mountPath: /etc/git-secret/known_hosts
+    readOnly: true
+    subPath: known_hosts
+  {{- end }}
   {{- end }}
 {{- end }}
 
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index c7badb3f1a9f2..229399c4f59f5 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -114,6 +114,9 @@ data:
     git_sync_container_tag = {{ .Values.dags.gitSync.containerTag }}
     git_sync_init_container_name = {{ .Values.dags.gitSync.containerName }}
     git_sync_run_as_user = {{ .Values.uid }}
+    {{- if  .Values.dags.gitSync.knownHosts }}
+    git_ssh_known_hosts_configmap_name = {{ include "airflow_config" . }}
+    {{- end }}
     {{- if .Values.dags.gitSync.sshKeySecret }}
     git_ssh_key_secret_name = {{ .Values.dags.gitSync.sshKeySecret }}
     {{- else if .Values.dags.gitSync.credentialsSecret }}
@@ -147,3 +150,7 @@ data:
   airflow_local_settings.py: |
     {{ .Values.scheduler.airflowLocalSettings | nindent 4 }}
 {{- end }}
+{{- if and .Values.dags.gitSync.enabled  .Values.dags.gitSync.knownHosts }}
+  known_hosts: |
+    {{ .Values.dags.gitSync.knownHosts | nindent 4 }}
+{{- end }}
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index 9074fd4c78846..7da924d2a1ea1 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -43,6 +43,9 @@ tests:
           depth: 1
           repo: https://github.com/apache/airflow.git
           branch: test-branch
+          sshKeySecret: ~
+          credentialsSecret: ~
+          knownHosts: ~
         persistence:
           enabled: true
     asserts:
@@ -78,6 +81,7 @@ tests:
           enabled: true
           containerName: git-sync-test
           sshKeySecret: ssh-secret
+          knownHosts: ~
           branch: test-branch
     asserts:
       - contains:
@@ -108,6 +112,7 @@ tests:
         gitSync:
           enabled: true
           credentialsSecret: user-pass-secret
+          sshKeySecret: ~
     asserts:
       - contains:
           path: spec.template.spec.containers[1].env
diff --git a/chart/tests/git-sync-webserver_test.yaml b/chart/tests/git-sync-webserver_test.yaml
index e28ff2c57f92e..bdbb7c66c2767 100644
--- a/chart/tests/git-sync-webserver_test.yaml
+++ b/chart/tests/git-sync-webserver_test.yaml
@@ -45,7 +45,7 @@ tests:
       dags:
         gitSync:
           enabled: true
-          container_name: git-sync
+          containerName: git-sync
         persistence:
           enabled: false
     asserts:
diff --git a/chart/tests/git-sync-worker_test.yaml b/chart/tests/git-sync-worker_test.yaml
index 9b51fbe9ef974..847a4dc3baa00 100644
--- a/chart/tests/git-sync-worker_test.yaml
+++ b/chart/tests/git-sync-worker_test.yaml
@@ -48,7 +48,7 @@ tests:
       dags:
         gitSync:
           enabled: true
-          container_name: git-sync
+          containerName: git-sync
         persistence:
           enabled: false
     asserts:
@@ -61,7 +61,7 @@ tests:
       dags:
         gitSync:
           enabled: true
-          container_name: git-sync
+          containerName: git-sync
         persistence:
           enabled: true
     asserts:
diff --git a/chart/values.yaml b/chart/values.yaml
index 94cbb4affeb8a..b103c54e9cc53 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -493,6 +493,11 @@ dags:
     #     gitSshKey: <base64_encoded_data>
     # and specify the name of the secret below
     #sshKeySecret: airflow-ssh-secret
+    # If you are using an ssh private key, you can additionally
+    # specify the content of your known_hosts file, example:
+    #knownHosts: |
+    #    <host1>,<ip1> <key1>
+    #    <host2>,<ip2> <key2>
     # interval between git sync attempts in seconds
     wait: 60
     # git sync image details

From a4a0cf542e2f2267458ebeeff9ef532da1022986 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Mon, 22 Jun 2020 14:35:22 +0530
Subject: [PATCH 15/19] add git-sync details into the chart documentation

---
 chart/README.md                               | 40 ++++++++++++++++++-
 .../scheduler/scheduler-deployment.yaml       |  2 +-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/chart/README.md b/chart/README.md
index fb04517ee8630..4eb80eddb8f5e 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -66,7 +66,7 @@ The command removes all the Kubernetes components associated with the chart and
 
 ## Updating DAGs
 
-The recommended way to update your DAGs with this chart is to build a new docker image with the latest code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image:
+The recommended way to update your DAGs with this chart is to build a new docker image with the latest DAG code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image:
 
 ```bash
 helm upgrade airflow . \
@@ -76,6 +76,42 @@ helm upgrade airflow . \
 
 For local development purpose you can also build the image locally and use it via deployment method described by Breeze.
 
+## Mounting DAGS using Git-Sync side car with Persistence enabled
+
+This option will use a Persistent Volume Claim with an accessMode of `ReadWriteMany`. The scheduler pod will sync DAGs from a git repository onto the PVC every configured number of seconds. The other pods will read the synced DAGs. Not all volume  plugins have support for `ReadWriteMany` accessMode. Refer [Persistent Volume Access Modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) for details
+
+```bash
+helm upgrade airflow . \
+  --set dags.persistence.enabled=true \
+  --set dags.gitSync.enabled=true
+  # you can also override the other persistence or gitSync values
+  # by setting the  dags.persistence.* and dags.gitSync.* values
+  # Refer values.yaml for details
+```
+
+## Mounting DAGS using Git-Sync side car without Persistence
+This option will use an always running Git-Sync side car on every scheduler,webserver and worker pods. The Git-Sync side car containers will sync DAGs from a git repository every configured number of seconds. If you are using the KubernetesExecutor, Git-sync will run as an initContainer on your worker pods.
+
+```bash
+helm upgrade airflow . \
+  --set dags.persistence.enabled=false \
+  --set dags.gitSync.enabled=true
+  # you can also override the other gitSync values
+  # by setting the  dags.gitSync.* values
+  # Refer values.yaml for details
+```
+
+## Mounting DAGS from an externally populated PVC
+In this approach, Airflow will read the DAGs from a PVC which has `ReadOnlyMany` or `ReadWriteMany` accessMode. You will have to ensure that the PVC is populated/updated with the required DAGs(this won't be handled by the chart). You can pass in the name of the  volume claim to the chart
+
+```bash
+helm upgrade airflow . \
+  --set dags.persistence.enabled=true \
+  --set dags.persistence.existingClaim=my-volume-claim
+  --set dags.gitSync.enabled=false
+```
+
+
 ## Parameters
 
 The following tables lists the configurable parameters of the Airflow chart and their default values.
@@ -159,6 +195,8 @@ The following tables lists the configurable parameters of the Airflow chart and
 | `webserver.resources.requests.cpu`                    | CPU Request of webserver                                                                                     | `~`                                               |
 | `webserver.resources.requests.memory`                 | Memory Request of webserver                                                                                  | `~`                                               |
 | `webserver.defaultUser`                               | Optional default airflow user information                                                                    | `{}`                                              |
+| `dags.persistence.*`                               | Dag persistence configutation                                                                    | Refer `values.yaml`                                              |
+| `dags.gitSync.*`                               | Git sync configuration                                                                   | Refer `values.yaml`                                              |
 
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index 1616aec161a34..933155687bf56 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -145,7 +145,7 @@ spec:
               subPath: airflow_local_settings.py
               readOnly: true
 {{- end }}
-{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
+{{- if .Values.dags.gitSync.enabled }}
             - name: dags
               mountPath: {{ template "airflow_dags_mount_path" . }}
         {{- include "git_sync_container" . | indent 8 }}

From 465a018f986f8e460f4498b260d953373e44e7c7 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Thu, 25 Jun 2020 19:21:58 +0530
Subject: [PATCH 16/19] Update .gitignore

Co-authored-by: Ash Berlin-Taylor <ash_github@firemirror.com>
---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 6699e29a3365a..70ac006d31a27 100644
--- a/.gitignore
+++ b/.gitignore
@@ -201,5 +201,5 @@ log.txt*
 Chart.lock
 
 # Chart dependencies
-requirements.lock
+chart/requirements.lock
 **/charts/*.tgz

From 0dc34711ba436d30c0f1bda3573b6a32fc7d606e Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Thu, 2 Jul 2020 22:00:04 +0530
Subject: [PATCH 17/19] make git sync max failures configurable

---
 chart/templates/_helpers.yaml            | 2 ++
 chart/tests/git-sync-scheduler_test.yaml | 3 +++
 chart/values.yaml                        | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 84d8108bac2ce..898924f2ff930 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -148,6 +148,8 @@
       value: "true"
     - name: GIT_SYNC_WAIT
       value: {{ .Values.dags.gitSync.wait | quote }}
+    - name: GIT_SYNC_MAX_SYNC_FAILURES
+      value: {{ .Values.dags.gitSync.maxFailures | quote }}
   volumeMounts:
   - name: dags
     mountPath: {{ .Values.dags.gitSync.root }}
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
index 7da924d2a1ea1..10d2288ed4876 100644
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ b/chart/tests/git-sync-scheduler_test.yaml
@@ -36,6 +36,7 @@ tests:
           containerTag: test-tag
           containerRepository: test-registry/test-repo
           wait: 66
+          maxFailures: 70
           subPath: "path1/path2"
           dest: "test-dest"
           root: "/git-root"
@@ -71,6 +72,8 @@ tests:
                 value: "true"
               - name: GIT_SYNC_WAIT
                 value: "66"
+              - name: GIT_SYNC_MAX_SYNC_FAILURES
+                value: "70"
             volumeMounts:
               - mountPath: /git-root
                 name: dags
diff --git a/chart/values.yaml b/chart/values.yaml
index b103c54e9cc53..8fa7ab63292b5 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -464,6 +464,8 @@ dags:
     root: "/git"
     dest: "repo"
     depth: 1
+    # the number of consecutive failures allowed before aborting
+    maxFailures: 0
     # subpath within the repo where dags are located
     # should be "" if dags are at repo root
     subPath: "tests/dags"

From b7a40e87a2e2d35ba93899de5b9665b3e60f61fb Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sun, 5 Jul 2020 14:06:20 +0530
Subject: [PATCH 18/19] Apply suggestions from code review

Co-authored-by: Jarek Potiuk <jarek@potiuk.com>
---
 chart/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/chart/README.md b/chart/README.md
index 4eb80eddb8f5e..c0d7847624680 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -86,7 +86,7 @@ helm upgrade airflow . \
   --set dags.gitSync.enabled=true
   # you can also override the other persistence or gitSync values
   # by setting the  dags.persistence.* and dags.gitSync.* values
-  # Refer values.yaml for details
+  # Please refer to values.yaml for details
 ```
 
 ## Mounting DAGS using Git-Sync side car without Persistence
@@ -195,8 +195,8 @@ The following tables lists the configurable parameters of the Airflow chart and
 | `webserver.resources.requests.cpu`                    | CPU Request of webserver                                                                                     | `~`                                               |
 | `webserver.resources.requests.memory`                 | Memory Request of webserver                                                                                  | `~`                                               |
 | `webserver.defaultUser`                               | Optional default airflow user information                                                                    | `{}`                                              |
-| `dags.persistence.*`                               | Dag persistence configutation                                                                    | Refer `values.yaml`                                              |
-| `dags.gitSync.*`                               | Git sync configuration                                                                   | Refer `values.yaml`                                              |
+| `dags.persistence.*`                               | Dag persistence configutation                                                                    | Please refer to `values.yaml`                                    |
+| `dags.gitSync.*`                               | Git sync configuration                                                                   | Please refer to `values.yaml`                                    |
 
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,

From eabf6c0779860fa9c390296d138ecc7ff1e27287 Mon Sep 17 00:00:00 2001
From: Aneesh Joseph <admin@coderplus.com>
Date: Sun, 5 Jul 2020 14:59:32 +0530
Subject: [PATCH 19/19] add back requirements.lock

---
 .gitignore              | 1 -
 chart/requirements.lock | 6 ++++++
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 chart/requirements.lock

diff --git a/.gitignore b/.gitignore
index 70ac006d31a27..c7ae00a28ccfd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -201,5 +201,4 @@ log.txt*
 Chart.lock
 
 # Chart dependencies
-chart/requirements.lock
 **/charts/*.tgz
diff --git a/chart/requirements.lock b/chart/requirements.lock
new file mode 100644
index 0000000000000..f86e6968adf29
--- /dev/null
+++ b/chart/requirements.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: postgresql
+  repository: https://kubernetes-charts.storage.googleapis.com
+  version: 6.3.12
+digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af
+generated: "2020-06-21T19:11:53.498134738+02:00"
