From ddab310823b56908002ab992d28714c4d0bc7a47 Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sun, 25 Jul 2021 16:01:38 +0200 Subject: [PATCH 01/14] ECSOperator AirflowException returns last cloudwatch logs when ECS task is stopped --- airflow/providers/amazon/aws/operators/ecs.py | 19 +++++++++++---- .../amazon/aws/operators/test_ecs.py | 23 ++++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 07f5702815ae9..954f62adc305c 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -136,13 +136,16 @@ class ECSOperator(BaseOperator): Only required if you want logs to be shown in the Airflow UI after your job has finished. :type awslogs_stream_prefix: str + :param quota_retry: Config if and how to retry _start_task() for transient errors. + :type quota_retry: dict :param reattach: If set to True, will check if the task previously launched by the task_instance is already running. If so, the operator will attach to it instead of starting a new task. This is to avoid relaunching a new task when the connection drops between Airflow and ECS while the task is running (when the Airflow worker is restarted for example). :type reattach: bool - :param quota_retry: Config if and how to retry _start_task() for transient errors. - :type quota_retry: dict + :param number_logs_exception: number of lines from the last Cloudwatch logs to return in the AirflowException if + an ECS task is stopped (to receive Airflow alerts with the logs of what failed in the code running in ECS) + :type number_logs_exception: int """ ui_color = '#f0ede4' @@ -178,6 +181,7 @@ def __init__( propagate_tags: Optional[str] = None, quota_retry: Optional[dict] = None, reattach: bool = False, + number_logs_exception=10, **kwargs, ): super().__init__(**kwargs) @@ -201,6 +205,7 @@ def __init__( self.awslogs_region = awslogs_region self.propagate_tags = propagate_tags self.reattach = reattach + self.number_logs_exception = number_logs_exception if self.awslogs_region is None: self.awslogs_region = region_name @@ -343,9 +348,12 @@ def _cloudwatch_log_events(self) -> Generator: def _aws_logs_enabled(self): return self.awslogs_group and self.awslogs_stream_prefix + def _last_log_messages(self, number_messages): + return [log["message"] for log in deque(self._cloudwatch_log_events(), maxlen=number_messages)] + def _last_log_message(self): try: - return deque(self._cloudwatch_log_events(), maxlen=1).pop()["message"] + return self._last_log_messages(1)[0] except IndexError: return None @@ -378,7 +386,10 @@ def _check_success_task(self) -> None: containers = task['containers'] for container in containers: if container.get('lastStatus') == 'STOPPED' and container['exitCode'] != 0: - raise AirflowException(f'This task is not in success state {task}') + last_log_messages = "\n".join(self._last_log_messages(self.number_logs_exception)) + raise AirflowException( + f"This task is not in success state - last logs from Cloudwatch:\n{last_log_messages}" + ) elif container.get('lastStatus') == 'PENDING': raise AirflowException(f'This task is still pending {task}') elif 'error' in container.get('reason', '').lower(): diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 4013450ec9c55..5d276d2a8b92b 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -217,7 +217,8 @@ def test_wait_end_tasks(self): client_mock.get_waiter.return_value.wait.assert_called_once_with(cluster='c', tasks=['arn']) assert sys.maxsize == client_mock.get_waiter.return_value.config.max_attempts - def test_check_success_tasks_raises(self): + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["1", "2", "3", "4", "5"]) + def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages): client_mock = mock.Mock() self.ecs.arn = 'arn' self.ecs.client = client_mock @@ -228,11 +229,11 @@ def test_check_success_tasks_raises(self): with pytest.raises(Exception) as ctx: self.ecs._check_success_task() - # Ordering of str(dict) is not guaranteed. - assert "This task is not in success state " in str(ctx.value) - assert "'name': 'foo'" in str(ctx.value) - assert "'lastStatus': 'STOPPED'" in str(ctx.value) - assert "'exitCode': 1" in str(ctx.value) + expected_last_logs = "\n".join(mock_last_log_messages.return_value) + assert ( + str(ctx.value) + == f"This task is not in success state - last logs from Cloudwatch:\n{expected_last_logs}" + ) client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) def test_check_success_tasks_raises_pending(self): @@ -415,17 +416,17 @@ def test_reattach_save_task_arn_xcom( xcom_del_mock.assert_called_once() assert self.ecs.arn == 'arn:aws:ecs:us-east-1:012345678910:task/d8c67b3c-ac87-4ffe-a847-4785bc3a8b55' - @mock.patch.object(ECSOperator, '_last_log_message', return_value="Log output") + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["Log output"]) def test_execute_xcom_with_log(self, mock_cloudwatch_log_message): self.ecs.do_xcom_push = True - assert self.ecs.execute(None) == mock_cloudwatch_log_message.return_value + assert self.ecs.execute(None) == mock_cloudwatch_log_message.return_value[-1] - @mock.patch.object(ECSOperator, '_last_log_message', return_value=None) + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=[]) def test_execute_xcom_with_no_log(self, mock_cloudwatch_log_message): self.ecs.do_xcom_push = True - assert self.ecs.execute(None) == mock_cloudwatch_log_message.return_value + assert self.ecs.execute(None) is None - @mock.patch.object(ECSOperator, '_last_log_message', return_value="Log output") + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["Log output"]) def test_execute_xcom_disabled(self, mock_cloudwatch_log_message): self.ecs.do_xcom_push = False assert self.ecs.execute(None) is None From 9f4e9eed56569b49c213645084f02e77b0056d1d Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sun, 25 Jul 2021 17:13:06 +0200 Subject: [PATCH 02/14] reduce length lines code <= 110 --- airflow/providers/amazon/aws/operators/ecs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 954f62adc305c..2f5c268c4d67c 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -143,8 +143,9 @@ class ECSOperator(BaseOperator): This is to avoid relaunching a new task when the connection drops between Airflow and ECS while the task is running (when the Airflow worker is restarted for example). :type reattach: bool - :param number_logs_exception: number of lines from the last Cloudwatch logs to return in the AirflowException if - an ECS task is stopped (to receive Airflow alerts with the logs of what failed in the code running in ECS) + :param number_logs_exception: number of lines from the last Cloudwatch logs to return in the + AirflowException if an ECS task is stopped (to receive Airflow alerts with the logs of what + failed in the code running in ECS) :type number_logs_exception: int """ @@ -386,9 +387,9 @@ def _check_success_task(self) -> None: containers = task['containers'] for container in containers: if container.get('lastStatus') == 'STOPPED' and container['exitCode'] != 0: - last_log_messages = "\n".join(self._last_log_messages(self.number_logs_exception)) + last_logs = "\n".join(self._last_log_messages(self.number_logs_exception)) raise AirflowException( - f"This task is not in success state - last logs from Cloudwatch:\n{last_log_messages}" + f"This task is not in success state - last logs from Cloudwatch:\n{last_logs}" ) elif container.get('lastStatus') == 'PENDING': raise AirflowException(f'This task is still pending {task}') From 2d8bb767b9daf8df4012a4861d3e25829ed5c135 Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Sat, 31 Jul 2021 17:10:50 +0200 Subject: [PATCH 03/14] Update airflow/providers/amazon/aws/operators/ecs.py Co-authored-by: Tzu-ping Chung --- airflow/providers/amazon/aws/operators/ecs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 2f5c268c4d67c..3ae62599a5c88 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -143,9 +143,9 @@ class ECSOperator(BaseOperator): This is to avoid relaunching a new task when the connection drops between Airflow and ECS while the task is running (when the Airflow worker is restarted for example). :type reattach: bool - :param number_logs_exception: number of lines from the last Cloudwatch logs to return in the + :param number_logs_exception: Number of lines from the last Cloudwatch logs to return in the AirflowException if an ECS task is stopped (to receive Airflow alerts with the logs of what - failed in the code running in ECS) + failed in the code running in ECS). :type number_logs_exception: int """ From 3b51a47bfa2e63be61b5cae3088508736e102016 Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Sat, 31 Jul 2021 17:11:45 +0200 Subject: [PATCH 04/14] Update airflow/providers/amazon/aws/operators/ecs.py Co-authored-by: Tzu-ping Chung --- airflow/providers/amazon/aws/operators/ecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 3ae62599a5c88..5971abf8ba19e 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -389,7 +389,8 @@ def _check_success_task(self) -> None: if container.get('lastStatus') == 'STOPPED' and container['exitCode'] != 0: last_logs = "\n".join(self._last_log_messages(self.number_logs_exception)) raise AirflowException( - f"This task is not in success state - last logs from Cloudwatch:\n{last_logs}" + f"This task is not in success state - last {self.number_logs_exception} " + f"logs from Cloudwatch:\n{last_logs}" ) elif container.get('lastStatus') == 'PENDING': raise AirflowException(f'This task is still pending {task}') From 2c3549f54c6eb5e6254411c869fd6bbe5912ef0f Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Sat, 31 Jul 2021 17:13:04 +0200 Subject: [PATCH 05/14] Update tests/providers/amazon/aws/operators/test_ecs.py Co-authored-by: Tzu-ping Chung --- tests/providers/amazon/aws/operators/test_ecs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 5d276d2a8b92b..b8423ce4f1f5a 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -229,10 +229,8 @@ def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages with pytest.raises(Exception) as ctx: self.ecs._check_success_task() - expected_last_logs = "\n".join(mock_last_log_messages.return_value) - assert ( - str(ctx.value) - == f"This task is not in success state - last logs from Cloudwatch:\n{expected_last_logs}" + assert str(ctx.value) == ( + f"This task is not in success state - last logs from Cloudwatch:\n1\n2\n3\n4\n5" ) client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) From 344ae4ea4f030ac1bfeaa9628d3e21ae6e15b973 Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Sat, 31 Jul 2021 17:13:38 +0200 Subject: [PATCH 06/14] Update tests/providers/amazon/aws/operators/test_ecs.py Co-authored-by: Tzu-ping Chung --- tests/providers/amazon/aws/operators/test_ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index b8423ce4f1f5a..2bf30689b42f3 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -417,7 +417,7 @@ def test_reattach_save_task_arn_xcom( @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["Log output"]) def test_execute_xcom_with_log(self, mock_cloudwatch_log_message): self.ecs.do_xcom_push = True - assert self.ecs.execute(None) == mock_cloudwatch_log_message.return_value[-1] + assert self.ecs.execute(None) == "Log output" @mock.patch.object(ECSOperator, '_last_log_messages', return_value=[]) def test_execute_xcom_with_no_log(self, mock_cloudwatch_log_message): From 8dbb97982f4cdb16238954be5c0c424e24e940e2 Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Sat, 31 Jul 2021 17:13:57 +0200 Subject: [PATCH 07/14] Update airflow/providers/amazon/aws/operators/ecs.py Co-authored-by: Tzu-ping Chung --- airflow/providers/amazon/aws/operators/ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 5971abf8ba19e..207e0170bdabe 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -182,7 +182,7 @@ def __init__( propagate_tags: Optional[str] = None, quota_retry: Optional[dict] = None, reattach: bool = False, - number_logs_exception=10, + number_logs_exception: int = 10, **kwargs, ): super().__init__(**kwargs) From f706eee066a90cb4545156272936a8e7c52afd65 Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sat, 31 Jul 2021 17:19:28 +0200 Subject: [PATCH 08/14] made description quota_retry more explicit --- airflow/providers/amazon/aws/operators/ecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py index 207e0170bdabe..b1d6d5324b98c 100644 --- a/airflow/providers/amazon/aws/operators/ecs.py +++ b/airflow/providers/amazon/aws/operators/ecs.py @@ -136,7 +136,8 @@ class ECSOperator(BaseOperator): Only required if you want logs to be shown in the Airflow UI after your job has finished. :type awslogs_stream_prefix: str - :param quota_retry: Config if and how to retry _start_task() for transient errors. + :param quota_retry: Config if and how to retry the launch of a new ECS task, to handle + transient errors. :type quota_retry: dict :param reattach: If set to True, will check if the task previously launched by the task_instance is already running. If so, the operator will attach to it instead of starting a new task. From 65449cb5dce4234467bdc995f62248f1faf2705a Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sat, 31 Jul 2021 17:22:27 +0200 Subject: [PATCH 09/14] fixed test ecs --- tests/providers/amazon/aws/operators/test_ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 2bf30689b42f3..28114e66acd81 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -230,7 +230,7 @@ def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages self.ecs._check_success_task() assert str(ctx.value) == ( - f"This task is not in success state - last logs from Cloudwatch:\n1\n2\n3\n4\n5" + f"This task is not in success state - last 10 logs from Cloudwatch:\n1\n2\n3\n4\n5" ) client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) From c60efb481c49f3984136a7755b6b6769ee2c73f0 Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sat, 31 Jul 2021 17:29:31 +0200 Subject: [PATCH 10/14] added test last_log_messages --- tests/providers/amazon/aws/operators/test_ecs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 28114e66acd81..5a6a397a255c2 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -217,6 +217,14 @@ def test_wait_end_tasks(self): client_mock.get_waiter.return_value.wait.assert_called_once_with(cluster='c', tasks=['arn']) assert sys.maxsize == client_mock.get_waiter.return_value.config.max_attempts + @mock.patch.object(ECSOperator, '_cloudwatch_log_events', return_value=({"message": str(i)} for i in range(10))) + def test_last_log_messages(self, mock_cloudwatch_log_events): + client_mock = mock.Mock() + self.ecs.arn = 'arn' + self.ecs.client = client_mock + + assert self.ecs._last_log_messages(5) == ["5", "6", "7", "8", "9"] + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["1", "2", "3", "4", "5"]) def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages): client_mock = mock.Mock() From cb504e154e8a80c36f7e6bd7b9e6da375af46a48 Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sat, 31 Jul 2021 18:56:12 +0200 Subject: [PATCH 11/14] linting test_ecs --- tests/providers/amazon/aws/operators/test_ecs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 5a6a397a255c2..5cb7b01aa2abd 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -217,7 +217,9 @@ def test_wait_end_tasks(self): client_mock.get_waiter.return_value.wait.assert_called_once_with(cluster='c', tasks=['arn']) assert sys.maxsize == client_mock.get_waiter.return_value.config.max_attempts - @mock.patch.object(ECSOperator, '_cloudwatch_log_events', return_value=({"message": str(i)} for i in range(10))) + @mock.patch.object( + ECSOperator, '_cloudwatch_log_events', return_value=({"message": str(i)} for i in range(10)) + ) def test_last_log_messages(self, mock_cloudwatch_log_events): client_mock = mock.Mock() self.ecs.arn = 'arn' From bda600e60d251cd67b46ad3b760f09803ef67e20 Mon Sep 17 00:00:00 2001 From: Pierre Malafosse Date: Sat, 31 Jul 2021 19:08:04 +0200 Subject: [PATCH 12/14] added tests for ECSOperator last logs --- .../amazon/aws/operators/test_ecs.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index 5cb7b01aa2abd..a0ca83e0d9e53 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -227,6 +227,14 @@ def test_last_log_messages(self, mock_cloudwatch_log_events): assert self.ecs._last_log_messages(5) == ["5", "6", "7", "8", "9"] + @mock.patch.object(ECSOperator, '_cloudwatch_log_events', return_value=()) + def test_last_log_messages_empty(self, mock_cloudwatch_log_events): + client_mock = mock.Mock() + self.ecs.arn = 'arn' + self.ecs.client = client_mock + + assert self.ecs._last_log_messages(10) == [] + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=["1", "2", "3", "4", "5"]) def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages): client_mock = mock.Mock() @@ -244,6 +252,21 @@ def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages ) client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) + @mock.patch.object(ECSOperator, '_last_log_messages', return_value=[]) + def test_check_success_tasks_raises_cloudwatch_logs_empty(self, mock_last_log_messages): + client_mock = mock.Mock() + self.ecs.arn = 'arn' + self.ecs.client = client_mock + + client_mock.describe_tasks.return_value = { + 'tasks': [{'containers': [{'name': 'foo', 'lastStatus': 'STOPPED', 'exitCode': 1}]}] + } + with pytest.raises(Exception) as ctx: + self.ecs._check_success_task() + + assert str(ctx.value) == (f"This task is not in success state - last 10 logs from Cloudwatch:\n") + client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) + def test_check_success_tasks_raises_pending(self): client_mock = mock.Mock() self.ecs.client = client_mock From 9f5f6e2b90cc6794751520cbc38f57e20e2f3197 Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Mon, 2 Aug 2021 11:09:30 +0200 Subject: [PATCH 13/14] Update tests/providers/amazon/aws/operators/test_ecs.py Co-authored-by: Tzu-ping Chung --- tests/providers/amazon/aws/operators/test_ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index a0ca83e0d9e53..aa59c98e04fa0 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -248,7 +248,7 @@ def test_check_success_tasks_raises_cloudwatch_logs(self, mock_last_log_messages self.ecs._check_success_task() assert str(ctx.value) == ( - f"This task is not in success state - last 10 logs from Cloudwatch:\n1\n2\n3\n4\n5" + "This task is not in success state - last 10 logs from Cloudwatch:\n1\n2\n3\n4\n5" ) client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) From 6deaa29c8229d1608141dce36c371ec26ab8602d Mon Sep 17 00:00:00 2001 From: pmalafosse Date: Mon, 2 Aug 2021 11:09:43 +0200 Subject: [PATCH 14/14] Update tests/providers/amazon/aws/operators/test_ecs.py Co-authored-by: Tzu-ping Chung --- tests/providers/amazon/aws/operators/test_ecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/operators/test_ecs.py b/tests/providers/amazon/aws/operators/test_ecs.py index aa59c98e04fa0..12a34f45142f6 100644 --- a/tests/providers/amazon/aws/operators/test_ecs.py +++ b/tests/providers/amazon/aws/operators/test_ecs.py @@ -264,7 +264,7 @@ def test_check_success_tasks_raises_cloudwatch_logs_empty(self, mock_last_log_me with pytest.raises(Exception) as ctx: self.ecs._check_success_task() - assert str(ctx.value) == (f"This task is not in success state - last 10 logs from Cloudwatch:\n") + assert str(ctx.value) == "This task is not in success state - last 10 logs from Cloudwatch:\n" client_mock.describe_tasks.assert_called_once_with(cluster='c', tasks=['arn']) def test_check_success_tasks_raises_pending(self):