From a1c532eacfeddcbefaa3e565a0522e25315286c4 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Mon, 22 Nov 2021 13:36:39 -0500 Subject: [PATCH 01/12] [16185] Added LocalKubernetesExecutor to breeze supported executors --- BREEZE.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/BREEZE.rst b/BREEZE.rst index a62a9734cd3c1..b6a14240ffd33 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2422,6 +2422,7 @@ This is the current syntax for `./breeze <./breeze>`_: One of: KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor + LocalKubernetesExecutor Default: KubernetesExecutor From d16a343c1478eb068e01d7ef363d91bb704918f8 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Mon, 22 Nov 2021 13:38:51 -0500 Subject: [PATCH 02/12] Revert "[16185] Added LocalKubernetesExecutor to breeze supported executors" This reverts commit a1c532eacfeddcbefaa3e565a0522e25315286c4. --- BREEZE.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/BREEZE.rst b/BREEZE.rst index b6a14240ffd33..a62a9734cd3c1 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2422,7 +2422,6 @@ This is the current syntax for `./breeze <./breeze>`_: One of: KubernetesExecutor CeleryExecutor LocalExecutor CeleryKubernetesExecutor - LocalKubernetesExecutor Default: KubernetesExecutor From 085e4abd750295e67a4994c5ea8f45429dfec118 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Sun, 9 Jan 2022 20:47:47 -0500 Subject: [PATCH 03/12] Added priorityClassName to webserver values --- chart/templates/webserver/webserver-deployment.yaml | 1 + chart/values.schema.json | 5 +++++ chart/values.yaml | 1 + 3 files changed, 7 insertions(+) diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index d9087fc4a207d..eb21a081dc274 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -130,6 +130,7 @@ spec: {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }} {{- end }} containers: + priorityClassName: {{ .Values.webserver.priorityClassName }} - name: webserver image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 4e8a1b545e3bc..4c4b751adf6d2 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2632,6 +2632,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for webserver pods.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index e3719398d79ee..38ffb6e02fb72 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -866,6 +866,7 @@ webserver: # Select certain nodes for airflow webserver pods. nodeSelector: {} + priorityClassName: "" affinity: {} # default webserver affinity is: # podAntiAffinity: From 43129d3ec47aed5bfcbe8f5053aeb5ad8fd0d23d Mon Sep 17 00:00:00 2001 From: subkanthi Date: Mon, 10 Jan 2022 16:13:53 -0500 Subject: [PATCH 04/12] Added support for priorityClassName to flower, pgbouncer, scheduler, statsd, triggerer and webserver --- chart/templates/flower/flower-deployment.yaml | 1 + .../pgbouncer/pgbouncer-deployment.yaml | 1 + .../scheduler/scheduler-deployment.yaml | 1 + chart/templates/statsd/statsd-deployment.yaml | 1 + .../triggerer/triggerer-deployment.yaml | 1 + .../webserver/webserver-deployment.yaml | 2 +- chart/values.schema.json | 27 ++++++++++++++++++- chart/values.yaml | 22 +++++++++++++++ 8 files changed, 54 insertions(+), 2 deletions(-) diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml index 9e9244d64c670..32f1c369a0ce3 100644 --- a/chart/templates/flower/flower-deployment.yaml +++ b/chart/templates/flower/flower-deployment.yaml @@ -67,6 +67,7 @@ spec: tolerations: {{ toYaml $tolerations | indent 8 }} serviceAccountName: {{ include "flower.serviceAccountName" . }} + priorityClassName: {{ .Values.flower.priorityClassName }} restartPolicy: Always securityContext: {{ $securityContext | nindent 8 }} {{- if or .Values.registry.secretName .Values.registry.connection }} diff --git a/chart/templates/pgbouncer/pgbouncer-deployment.yaml b/chart/templates/pgbouncer/pgbouncer-deployment.yaml index a7527a869127e..9e48449be7544 100644 --- a/chart/templates/pgbouncer/pgbouncer-deployment.yaml +++ b/chart/templates/pgbouncer/pgbouncer-deployment.yaml @@ -59,6 +59,7 @@ spec: checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }} spec: + priorityClassName: {{ .Values.pgbouncer.priorityClassName }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 036b0a59d4c61..acf569966025f 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -90,6 +90,7 @@ spec: {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} {{- end }} spec: + priorityClassName: {{ .Values.scheduler.priorityClassName }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/statsd/statsd-deployment.yaml b/chart/templates/statsd/statsd-deployment.yaml index 670dc53c456a6..421ddc4682276 100644 --- a/chart/templates/statsd/statsd-deployment.yaml +++ b/chart/templates/statsd/statsd-deployment.yaml @@ -57,6 +57,7 @@ spec: checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }} {{- end }} spec: + priorityClassName: {{ .Values.statsd.priorityClassName }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index c602b42371dca..cd57e128ed7c7 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -73,6 +73,7 @@ spec: {{- toYaml .Values.triggerer.podAnnotations | nindent 8 }} {{- end }} spec: + priorityClassName: {{ .Values.triggerer.priorityClassName }} nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} affinity: diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index eb21a081dc274..27481130d716f 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -87,6 +87,7 @@ spec: {{- end }} spec: serviceAccountName: {{ include "webserver.serviceAccountName" . }} + priorityClassName: {{ .Values.webserver.priorityClassName }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: @@ -130,7 +131,6 @@ spec: {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }} {{- end }} containers: - priorityClassName: {{ .Values.webserver.priorityClassName }} - name: webserver image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 4c4b751adf6d2..156236d008b0a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1660,6 +1660,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for scheduler pods.", "type": "object", @@ -1930,6 +1935,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for triggerer pods.", "type": "object", @@ -2633,7 +2643,7 @@ } }, "priorityClassName": { - "description": "Set priority class name", + "description": "Set priority value from the priority class name", "type": "string", "default": "" }, @@ -2920,6 +2930,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for Flower pods.", "type": "object", @@ -3051,6 +3066,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for StatsD pods.", "type": "object", @@ -3352,6 +3372,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for PgBouncer pods.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 38ffb6e02fb72..9bf48be2af782 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -638,6 +638,10 @@ scheduler: # weight: 100 tolerations: [] + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: "" + podAnnotations: {} logGroomerSidecar: @@ -866,6 +870,8 @@ webserver: # Select certain nodes for airflow webserver pods. nodeSelector: {} + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: "" affinity: {} # default webserver affinity is: @@ -961,6 +967,10 @@ triggerer: # weight: 100 tolerations: [] + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: "" + podAnnotations: {} # Flower settings @@ -1050,6 +1060,10 @@ flower: affinity: {} tolerations: [] + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: "" + podAnnotations: {} # Statsd settings @@ -1091,6 +1105,10 @@ statsd: affinity: {} tolerations: [] + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: "" + # Additional mappings for statsd exporter. extraMappings: [] @@ -1197,6 +1215,10 @@ pgbouncer: affinity: {} tolerations: [] + # Set the priority class name + # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority + priorityClassName: "" + uid: 65534 metricsExporterSidecar: From 0a976e0fd21802e1414ee36a10e1db5022085810 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Mon, 10 Jan 2022 16:31:46 -0500 Subject: [PATCH 05/12] Added priorityClassName to workers --- chart/templates/workers/worker-deployment.yaml | 1 + chart/values.schema.json | 5 +++++ chart/values.yaml | 1 + 3 files changed, 7 insertions(+) diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index d72235ffacac5..b7bc046f05fa1 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -83,6 +83,7 @@ spec: {{- toYaml .Values.workers.podAnnotations | nindent 8 }} {{- end }} spec: + priorityClassName: {{ .Values.workers.priorityClassName }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/values.schema.json b/chart/values.schema.json index 156236d008b0a..69020dca3b808 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1351,6 +1351,11 @@ "type": "string" } }, + "priorityClassName": { + "description": "Set priority value from the priority class name", + "type": "string", + "default": "" + }, "affinity": { "description": "Specify scheduling constraints for worker pods.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 9bf48be2af782..62a9d3669ea50 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -514,6 +514,7 @@ workers: # Select certain nodes for airflow worker pods. nodeSelector: {} + priorityClassName: "" affinity: {} # default worker affinity is: # podAntiAffinity: From f1f4970a5de02cb43f575b4cdf08199680d8caea Mon Sep 17 00:00:00 2001 From: subkanthi Date: Wed, 26 Jan 2022 23:04:08 -0500 Subject: [PATCH 06/12] Updates based on PR review comments --- chart/templates/flower/flower-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml index 32f1c369a0ce3..b27c91956a002 100644 --- a/chart/templates/flower/flower-deployment.yaml +++ b/chart/templates/flower/flower-deployment.yaml @@ -67,7 +67,9 @@ spec: tolerations: {{ toYaml $tolerations | indent 8 }} serviceAccountName: {{ include "flower.serviceAccountName" . }} + {{- if .Values.flower.priorityClassName }} priorityClassName: {{ .Values.flower.priorityClassName }} + {{- end }} restartPolicy: Always securityContext: {{ $securityContext | nindent 8 }} {{- if or .Values.registry.secretName .Values.registry.connection }} From d0d4315d87fe66728834270ce733e0735f37a9ea Mon Sep 17 00:00:00 2001 From: subkanthi Date: Thu, 27 Jan 2022 14:15:02 -0500 Subject: [PATCH 07/12] Changed priorityClassName to tilde, update values.schema.json to include null, if/else in the templates --- .../pgbouncer/pgbouncer-deployment.yaml | 2 + .../scheduler/scheduler-deployment.yaml | 2 + chart/templates/statsd/statsd-deployment.yaml | 2 + .../triggerer/triggerer-deployment.yaml | 2 + .../webserver/webserver-deployment.yaml | 2 + .../templates/workers/worker-deployment.yaml | 2 + chart/values.schema.json | 49 +++++++++++++------ chart/values.yaml | 14 +++--- 8 files changed, 54 insertions(+), 21 deletions(-) diff --git a/chart/templates/pgbouncer/pgbouncer-deployment.yaml b/chart/templates/pgbouncer/pgbouncer-deployment.yaml index 9e48449be7544..b254f3c6fb710 100644 --- a/chart/templates/pgbouncer/pgbouncer-deployment.yaml +++ b/chart/templates/pgbouncer/pgbouncer-deployment.yaml @@ -59,7 +59,9 @@ spec: checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }} spec: + {{- if .Values.pgbouncer.priorityClassName }} priorityClassName: {{ .Values.pgbouncer.priorityClassName }} + {{- end }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index acf569966025f..926b0bdeb1f41 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -90,7 +90,9 @@ spec: {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} {{- end }} spec: + {{- if .Values.scheduler.priorityClassName }} priorityClassName: {{ .Values.scheduler.priorityClassName }} + {{- end }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/statsd/statsd-deployment.yaml b/chart/templates/statsd/statsd-deployment.yaml index 421ddc4682276..6fe66c74f1c86 100644 --- a/chart/templates/statsd/statsd-deployment.yaml +++ b/chart/templates/statsd/statsd-deployment.yaml @@ -57,7 +57,9 @@ spec: checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }} {{- end }} spec: + {{- if .Values.statsd.priorityClassName }} priorityClassName: {{ .Values.statsd.priorityClassName }} + {{- end }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index cd57e128ed7c7..895a5e8f3853c 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -73,7 +73,9 @@ spec: {{- toYaml .Values.triggerer.podAnnotations | nindent 8 }} {{- end }} spec: + {{- if .Values.triggerer.priorityClassName }} priorityClassName: {{ .Values.triggerer.priorityClassName }} + {{- end }} nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} affinity: diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index 27481130d716f..2bd93abd09399 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -87,7 +87,9 @@ spec: {{- end }} spec: serviceAccountName: {{ include "webserver.serviceAccountName" . }} + {{- if .Values.webserver.priorityClassName }} priorityClassName: {{ .Values.webserver.priorityClassName }} + {{- end }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index b7bc046f05fa1..cecc26ffc1716 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -83,7 +83,9 @@ spec: {{- toYaml .Values.workers.podAnnotations | nindent 8 }} {{- end }} spec: + {{- if .Values.workers.priorityClassName }} priorityClassName: {{ .Values.workers.priorityClassName }} + {{- end }} nodeSelector: {{ toYaml $nodeSelector | indent 8 }} affinity: diff --git a/chart/values.schema.json b/chart/values.schema.json index 69020dca3b808..f543e73a00111 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1353,8 +1353,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for worker pods.", @@ -1667,8 +1670,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for scheduler pods.", @@ -1942,8 +1948,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for triggerer pods.", @@ -2649,8 +2658,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for webserver pods.", @@ -2937,8 +2949,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for Flower pods.", @@ -3073,8 +3088,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for StatsD pods.", @@ -3379,8 +3397,11 @@ }, "priorityClassName": { "description": "Set priority value from the priority class name", - "type": "string", - "default": "" + "type": [ + "string", + "null" + ], + "default": null }, "affinity": { "description": "Specify scheduling constraints for PgBouncer pods.", diff --git a/chart/values.yaml b/chart/values.yaml index 62a9d3669ea50..d5946d359c40f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -514,7 +514,7 @@ workers: # Select certain nodes for airflow worker pods. nodeSelector: {} - priorityClassName: "" + priorityClassName: ~ affinity: {} # default worker affinity is: # podAntiAffinity: @@ -641,7 +641,7 @@ scheduler: # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ podAnnotations: {} @@ -873,7 +873,7 @@ webserver: nodeSelector: {} # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ affinity: {} # default webserver affinity is: # podAntiAffinity: @@ -970,7 +970,7 @@ triggerer: # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ podAnnotations: {} @@ -1063,7 +1063,7 @@ flower: # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ podAnnotations: {} @@ -1108,7 +1108,7 @@ statsd: # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ # Additional mappings for statsd exporter. extraMappings: [] @@ -1218,7 +1218,7 @@ pgbouncer: # Set the priority class name # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority - priorityClassName: "" + priorityClassName: ~ uid: 65534 From 5c3c3e2ca21ba9a6bc5d9686c8ac9366dd373605 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Thu, 27 Jan 2022 17:42:59 -0500 Subject: [PATCH 08/12] Added test case for priorityClassName in helm charts --- chart/tests/test_airflow_common.py | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index cec42fd43c05c..3514260ee3e7d 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -248,3 +248,46 @@ def test_have_all_variables(self): assert variables == jmespath.search( "spec.template.spec.containers[0].env[*].name", doc ), f"Wrong vars in {component}" + + def test_priority_class_name(self): + docs = render_chart( + values={ + "flower": {"priorityClassName": "low-priority-flower"}, + "pgbouncer": {"priorityClassName": "low-priority-pgbouncer"}, + "scheduler": {"priorityClassName": "low-priority-scheduler"}, + "statsd": {"priorityClassName": "low-priority-statsd"}, + "triggerer": {"priorityClassName": "low-priority-triggerer"}, + "webserver": {"priorityClassName": "low-priority-webserver"}, + "workers": {"priorityClassName": "low-priority-worker"}, + }, + show_only=[ + "templates/flower/flower-deployment.yaml", + "templates/pgbouncer/pgbouncer-deployment.yaml", + "templates/scheduler/scheduler-deployment.yaml", + "templates/statsd/statsd-deployment.yaml", + "templates/triggerer/triggerer-deployment.yaml", + "templates/webserver/webserver-deployment.yaml", + "templates/workers/worker-deployment.yaml", + ], + ) + + for doc in docs: + component = doc['metadata']['labels']['component'] + priority = doc['spec']['template']['spec']['priorityClassName'] + + if component == "flower": + assert priority == "low-priority-flower" + elif component == "pgbouncer": + assert priority == "low-priority-pgbouncer" + elif component == "scheduler": + assert priority == "low-priority-scheduler" + elif component == "statsd": + assert priority == "low-priority-statsd" + elif component == "triggerer": + assert priority == "low-priority-triggerer" + elif component == "webserver": + assert priority == "low-priority-webserver" + elif component == "worker": + assert priority == "low-priority-worker" + else: + assert False From 02775d1b0a6404dbf6be4b0192f5c3098c3760e8 Mon Sep 17 00:00:00 2001 From: Kanthi Date: Fri, 28 Jan 2022 17:37:42 -0500 Subject: [PATCH 09/12] Update chart/values.yaml Removed priority class name Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- chart/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index a048c85a0dd52..4cfbb376ead1d 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -639,8 +639,6 @@ scheduler: # weight: 100 tolerations: [] - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ podAnnotations: {} From b5335539318d2a7678e7e462cdb82a7757f2d6e6 Mon Sep 17 00:00:00 2001 From: Kanthi Date: Fri, 28 Jan 2022 17:41:35 -0500 Subject: [PATCH 10/12] Update chart/values.schema.json Schema description update for priorityClassName Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- chart/values.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 1780dad283bc4..4cd057b29b9c7 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1352,7 +1352,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for worker pods.", "type": [ "string", "null" From 1487425fb863d836f3ad3f7438303bd8addc5e5d Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Fri, 28 Jan 2022 17:55:36 -0500 Subject: [PATCH 11/12] Changed priorityClass helm test based on PR review comments --- chart/tests/test_airflow_common.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/chart/tests/test_airflow_common.py b/chart/tests/test_airflow_common.py index f3801ccef57bc..53e6960c97860 100644 --- a/chart/tests/test_airflow_common.py +++ b/chart/tests/test_airflow_common.py @@ -295,19 +295,4 @@ def test_priority_class_name(self): component = doc['metadata']['labels']['component'] priority = doc['spec']['template']['spec']['priorityClassName'] - if component == "flower": - assert priority == "low-priority-flower" - elif component == "pgbouncer": - assert priority == "low-priority-pgbouncer" - elif component == "scheduler": - assert priority == "low-priority-scheduler" - elif component == "statsd": - assert priority == "low-priority-statsd" - elif component == "triggerer": - assert priority == "low-priority-triggerer" - elif component == "webserver": - assert priority == "low-priority-webserver" - elif component == "worker": - assert priority == "low-priority-worker" - else: - assert False + assert priority == f"low-priority-{component}" From a5089f2108eeb27aa300dda79abf7e78168a8b63 Mon Sep 17 00:00:00 2001 From: subkanthi Date: Fri, 4 Feb 2022 10:41:20 -0500 Subject: [PATCH 12/12] Changed priorityClassName comments --- chart/values.schema.json | 12 ++++++------ chart/values.yaml | 10 ---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index f5af4ab4ec493..47ca1dea471ea 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1669,7 +1669,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for scheduler pods.", "type": [ "string", "null" @@ -1947,7 +1947,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for triggerer pods.", "type": [ "string", "null" @@ -2657,7 +2657,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for webserver pods.", "type": [ "string", "null" @@ -2948,7 +2948,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for Flower pods.", "type": [ "string", "null" @@ -3087,7 +3087,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for StatsD pods.", "type": [ "string", "null" @@ -3396,7 +3396,7 @@ } }, "priorityClassName": { - "description": "Set priority value from the priority class name", + "description": "Specify priority for PgBouncer pods.", "type": [ "string", "null" diff --git a/chart/values.yaml b/chart/values.yaml index 80c6b97d974a4..d5b0632725f30 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -869,8 +869,6 @@ webserver: # Select certain nodes for airflow webserver pods. nodeSelector: {} - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ affinity: {} # default webserver affinity is: @@ -966,8 +964,6 @@ triggerer: # weight: 100 tolerations: [] - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ podAnnotations: {} @@ -1059,8 +1055,6 @@ flower: affinity: {} tolerations: [] - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ podAnnotations: {} @@ -1104,8 +1098,6 @@ statsd: affinity: {} tolerations: [] - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ # Additional mappings for statsd exporter. @@ -1214,8 +1206,6 @@ pgbouncer: affinity: {} tolerations: [] - # Set the priority class name - # https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority priorityClassName: ~ uid: 65534