From 98749b6a56eeca0f60bc3d5c1337164bc57b860d Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 28 Aug 2023 21:03:41 +0100 Subject: [PATCH 1/3] Truncate Wasb storage account name if it's more than 24 characters Storage account names must be between 3 and 24 characters in length but for some reasons that I can't explain, we saw a situation where the storage name is more than 24 characters and had to be truncated before it could work. Maybe it was possible in the past to have more than 24 characters or it could come from cluster but whichever way, the solution that worked was truncating the account name to 24 characters. --- airflow/providers/microsoft/azure/hooks/wasb.py | 16 ++++++++++++---- .../providers/microsoft/azure/hooks/test_wasb.py | 8 ++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 59f4b26de5dda..70ba910a7cc23 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -163,10 +163,18 @@ def get_conn(self) -> BlobServiceClient: account_url = conn.host if conn.host else f"https://{conn.login}.blob.core.windows.net/" parsed_url = urlparse(account_url) - if not parsed_url.netloc and "." not in parsed_url.path: - # if there's no netloc and no dots in the path, then user only - # provided the Active Directory ID, not the full URL or DNS name - account_url = f"https://{conn.login}.blob.core.windows.net/" + if not parsed_url.netloc: + if "." not in parsed_url.path: + # if there's no netloc and no dots in the path, then user only + # provided the Active Directory ID, not the full URL or DNS name + account_url = f"https://{conn.login}.blob.core.windows.net/" + else: + # if there's no netloc but there are dots in the path, then user + # provided the DNS name without the https:// prefix. + # Azure storage account name can only be 3 to 24 characters in length + # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-name + acc_name = parsed_url.path.split(".")[0][:24] + account_url = f"https://{acc_name}." + ".".join(parsed_url.path.split(".")[1:]) tenant = self._get_field(extra, "tenant_id") if tenant: diff --git a/tests/providers/microsoft/azure/hooks/test_wasb.py b/tests/providers/microsoft/azure/hooks/test_wasb.py index 06ef4eedfbf32..80cd627fa0cd8 100644 --- a/tests/providers/microsoft/azure/hooks/test_wasb.py +++ b/tests/providers/microsoft/azure/hooks/test_wasb.py @@ -346,8 +346,12 @@ def test_extra_client_secret_auth_config_ad_connection(self): "https://testaccountname.blob.core.windows.net", ), ("testhost", "https://accountlogin.blob.core.windows.net/"), - ("testhost.dns", "testhost.dns"), - ("testhost.blob.net", "testhost.blob.net"), + ("testhost.dns", "https://testhost.dns"), + ("testhost.blob.net", "https://testhost.blob.net"), + ( + "testhostakjhdisdfbearioyo.blob.core.windows.net", + "https://testhostakjhdisdfbearioy.blob.core.windows.net", + ), # more than 24 characters ], ) def test_proper_account_url_update( From b7cd8d447137f57f3ebce79822ef682db8a322c9 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 28 Aug 2023 22:44:32 +0100 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- airflow/providers/microsoft/azure/hooks/wasb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 70ba910a7cc23..e8b84846a07ad 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -173,8 +173,8 @@ def get_conn(self) -> BlobServiceClient: # provided the DNS name without the https:// prefix. # Azure storage account name can only be 3 to 24 characters in length # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-name - acc_name = parsed_url.path.split(".")[0][:24] - account_url = f"https://{acc_name}." + ".".join(parsed_url.path.split(".")[1:]) + acc_name = account_url.split(".")[0][:24] + account_url = f"https://{acc_name}." + ".".join(account_url.split(".")[1:]) tenant = self._get_field(extra, "tenant_id") if tenant: From 73f7b5bba745cd30892bd4a039cf55050af2bd44 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 28 Aug 2023 22:46:49 +0100 Subject: [PATCH 3/3] Also add the change to the async part --- airflow/providers/microsoft/azure/hooks/wasb.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index e8b84846a07ad..72579b8bf94d8 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -576,10 +576,18 @@ async def get_async_conn(self) -> AsyncBlobServiceClient: account_url = conn.host if conn.host else f"https://{conn.login}.blob.core.windows.net/" parsed_url = urlparse(account_url) - if not parsed_url.netloc and "." not in parsed_url.path: - # if there's no netloc and no dots in the path, then user only - # provided the Active Directory ID, not the full URL or DNS name - account_url = f"https://{conn.login}.blob.core.windows.net/" + if not parsed_url.netloc: + if "." not in parsed_url.path: + # if there's no netloc and no dots in the path, then user only + # provided the Active Directory ID, not the full URL or DNS name + account_url = f"https://{conn.login}.blob.core.windows.net/" + else: + # if there's no netloc but there are dots in the path, then user + # provided the DNS name without the https:// prefix. + # Azure storage account name can only be 3 to 24 characters in length + # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-name + acc_name = account_url.split(".")[0][:24] + account_url = f"https://{acc_name}." + ".".join(account_url.split(".")[1:]) tenant = self._get_field(extra, "tenant_id") if tenant: