From 8837f03b5075419a2af4646c6d4fdcc371fbf789 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Tue, 21 Jul 2026 15:40:09 +0530 Subject: [PATCH] Better log when spark k8s driver remains in Unknown Phase --- .../src/airflow/providers/apache/spark/hooks/spark_submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py index 662966e4e1459..6d90f7b003bd2 100644 --- a/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py +++ b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py @@ -1237,7 +1237,9 @@ def _poll_k8s_driver_via_api(self) -> str | None: if consecutive_unknown >= max_consecutive_unknown: raise RuntimeError( f"Spark application {app_id} reported Unknown phase " - f"{consecutive_unknown} times consecutively; giving up." + f"{consecutive_unknown} times consecutively (the pod's state could not " + f"be obtained, typically due to an error communicating with the node the " + f"pod should be running on); giving up." ) else: consecutive_unknown = 0