From 601e56f4635c493344c406c8251c6e2526ba00cd Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 30 Dec 2021 11:21:46 -0800 Subject: [PATCH 1/2] Move pod_mutation_hook call from PodManager to KubernetesPodOperator Previously, in KubernetesPodOperator, the invocation of the pod mutation hook occurred within the call to PodManager.run_pod_async. So, `run_pod_async` would not quite run the pod you asked it to run, but would mutate it first. With this change, `run_pod_async` runs exactly the pod you request, and the pod returned by `build_pod_request_obj` is actually the pod you request. --- airflow/providers/cncf/kubernetes/CHANGELOG.rst | 2 ++ airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py | 2 ++ airflow/providers/cncf/kubernetes/utils/pod_manager.py | 3 --- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index f794acf814d36..748ec93a1b8d6 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -27,6 +27,8 @@ Breaking changes * ``Simplify KubernetesPodOperator (#19572)`` * Class ``pod_launcher.PodLauncher`` renamed to ``pod_manager.PodManager`` +* :func:`airflow.settings.pod_mutation_hook` is no longer called in :meth:`~cncf.kubernetes.utils.pod_manager.PodManager.run_pod_async``. For ``KubernetesPodOperator``, mutation now occurs in ``build_pod_request_obj``. + .. warning:: Many methods in :class:`~.KubernetesPodOperator` and class:`~.PodManager` (formerly named ``PodLauncher``) have been renamed. If you have subclassed :class:`~.KubernetesPodOperator` you will need to update your subclass to diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py index e56860be757a6..c3dc5379de170 100644 --- a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -25,6 +25,7 @@ from kubernetes.client import CoreV1Api, models as k8s from airflow.providers.cncf.kubernetes.utils.pod_manager import PodLaunchFailedException, PodManager, PodPhase +from airflow.settings import pod_mutation_hook try: import airflow.utils.yaml as yaml @@ -574,6 +575,7 @@ def build_pod_request_obj(self, context=None): 'kubernetes_pod_operator': 'True', } ) + pod = pod_mutation_hook(pod) return pod diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 931fe60828118..70b5b4c2ebaed 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -36,7 +36,6 @@ from airflow.exceptions import AirflowException from airflow.kubernetes.kube_client import get_kube_client from airflow.kubernetes.pod_generator import PodDefaults -from airflow.settings import pod_mutation_hook from airflow.utils.log.logging_mixin import LoggingMixin @@ -104,8 +103,6 @@ def __init__( def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod: """Runs POD asynchronously""" - pod_mutation_hook(pod) - sanitized_pod = self._client.api_client.sanitize_for_serialization(pod) json_pod = json.dumps(sanitized_pod, indent=2) From eea190c9e1adcea847a1e25af5e2a93e0e537675 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Thu, 30 Dec 2021 13:26:16 -0800 Subject: [PATCH 2/2] fixup! Move pod_mutation_hook call from PodManager to KubernetesPodOperator --- airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py index c3dc5379de170..8cc1bc499f63c 100644 --- a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -575,7 +575,7 @@ def build_pod_request_obj(self, context=None): 'kubernetes_pod_operator': 'True', } ) - pod = pod_mutation_hook(pod) + pod_mutation_hook(pod) return pod