From 9634e8722b2964be84e8314457b710d9b2004b7f Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 18 Aug 2022 13:37:35 -0700 Subject: [PATCH] Improve error handling/messaging around bucket exist check S3Hook.check_for_bucket() method uses the boto3 s3 client method `head_bucket` to check for bucket existence. This client method does not work like most boto3 APIs, it only returns a small subset of error codes. --- airflow/providers/amazon/aws/hooks/s3.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index 149887f6c97f6..22398fc878566 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -203,7 +203,19 @@ def check_for_bucket(self, bucket_name: Optional[str] = None) -> bool: self.get_conn().head_bucket(Bucket=bucket_name) return True except ClientError as e: - self.log.error(e.response["Error"]["Message"]) + # The head_bucket api is odd in that it cannot return proper + # exception objects, so error codes must be used. Only 200, 404 and 403 + # are ever returned. See the following links for more details: + # https://github.com/boto/boto3/issues/2499 + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_bucket + return_code = int(e.response['Error']['Code']) + if return_code == 404: + self.log.error('Bucket "%s" does not exist', bucket_name) + elif return_code == 403: + self.log.error( + 'Access to bucket "%s" is forbidden or there was an error with the request', bucket_name + ) + self.log.error(e) return False @provide_bucket_name