From 00905090b24439971013d6b2c94189278f18f28f Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Tue, 29 Mar 2022 10:36:12 -0700 Subject: [PATCH 1/2] Log traceback only on DEBUG for KPO logs read interruption Logging the traceback after every disconnect is a little overly scary when this is an expected occurrence for long-running pods, and this can create false alarm for users. Here we reduce noise a bit while allowing users to troubleshoot if desired by changing the log level to debug. --- airflow/providers/cncf/kubernetes/utils/pod_manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 3b3a7b7a9eb9b..6cdfde0c57482 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -214,9 +214,13 @@ def consume_logs(*, since_time: Optional[DateTime] = None, follow: bool = True) for line in logs: timestamp, message = self.parse_log_line(line.decode('utf-8')) self.log.info(message) - except BaseHTTPError: # Catches errors like ProtocolError(TimeoutError). + except BaseHTTPError: self.log.warning( - 'Failed to read logs for pod %s', + "Reading of logs was interrupted, likely due to a connection issue. " + "Will retry momentarily.", + ) + self.log.debug( + "Traceback for interrupted logs read for pod %r", pod.metadata.name, exc_info=True, ) From 76317187d8ed21b318e1da19c72e59da40c9e83c Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Tue, 29 Mar 2022 12:40:09 -0700 Subject: [PATCH 2/2] log the error --- airflow/providers/cncf/kubernetes/utils/pod_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 6cdfde0c57482..f84a044d68c98 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -214,10 +214,11 @@ def consume_logs(*, since_time: Optional[DateTime] = None, follow: bool = True) for line in logs: timestamp, message = self.parse_log_line(line.decode('utf-8')) self.log.info(message) - except BaseHTTPError: + except BaseHTTPError as e: self.log.warning( - "Reading of logs was interrupted, likely due to a connection issue. " - "Will retry momentarily.", + "Reading of logs interrupted with error %r; will retry. " + "Set log level to DEBUG for traceback.", + e, ) self.log.debug( "Traceback for interrupted logs read for pod %r",