diff --git a/providers/apache/hive/provider.yaml b/providers/apache/hive/provider.yaml index db2e4006942ff..5af0916987e1b 100644 --- a/providers/apache/hive/provider.yaml +++ b/providers/apache/hive/provider.yaml @@ -195,6 +195,20 @@ connection-types: - boolean - 'null' default: false + ssl: + label: Ssl + schema: + type: + - boolean + - 'null' + default: true + zoo_keeper_namespace: + label: Zoo Keeper Namespace + schema: + type: + - string + - 'null' + default: 'hiveserver2' - hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveServer2Hook hook-name: "Hive Server 2 Thrift" connection-type: hiveserver2 diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py index f9d7cd5b0d7ce..654afa27467e3 100644 --- a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py +++ b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py @@ -131,6 +131,11 @@ def get_provider_info(): "label": "High Availability mode", "schema": {"type": ["boolean", "null"], "default": False}, }, + "ssl": {"label": "Ssl", "schema": {"type": ["boolean", "null"], "default": True}}, + "zoo_keeper_namespace": { + "label": "Zoo Keeper Namespace", + "schema": {"type": ["string", "null"], "default": "hiveserver2"}, + }, }, }, { diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml index 3dbed0b0485bc..3a9d2b7ad1a2d 100644 --- a/providers/google/provider.yaml +++ b/providers/google/provider.yaml @@ -1202,6 +1202,10 @@ connection-types: schema: type: ["boolean", "null"] default: false + quota_project_id: + label: "Quota Project ID" + schema: + type: ["string", "null"] - hook-class-name: airflow.providers.google.cloud.hooks.spanner.SpannerHook connection-type: gcpspanner - hook-class-name: airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook @@ -1281,6 +1285,10 @@ connection-types: schema: type: ["boolean", "null"] default: false + quota_project_id: + label: "Quota Project ID" + schema: + type: ["string", "null"] use_legacy_sql: label: "Use Legacy SQL" schema: diff --git a/providers/google/src/airflow/providers/google/get_provider_info.py b/providers/google/src/airflow/providers/google/get_provider_info.py index 92966a4b3679b..13feaa8d1e761 100644 --- a/providers/google/src/airflow/providers/google/get_provider_info.py +++ b/providers/google/src/airflow/providers/google/get_provider_info.py @@ -1434,6 +1434,7 @@ def get_provider_info(): "label": "Anonymous credentials (ignores all other settings)", "schema": {"type": ["boolean", "null"], "default": False}, }, + "quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}}, }, }, { @@ -1512,6 +1513,7 @@ def get_provider_info(): "label": "Anonymous credentials (ignores all other settings)", "schema": {"type": ["boolean", "null"], "default": False}, }, + "quota_project_id": {"label": "Quota Project ID", "schema": {"type": ["string", "null"]}}, "use_legacy_sql": {"label": "Use Legacy SQL", "schema": {"type": ["boolean", "null"]}}, "location": {"label": "Location", "schema": {"type": ["string", "null"]}}, "priority": { diff --git a/providers/microsoft/azure/provider.yaml b/providers/microsoft/azure/provider.yaml index fe6366023e724..cbcc049aa0930 100644 --- a/providers/microsoft/azure/provider.yaml +++ b/providers/microsoft/azure/provider.yaml @@ -384,7 +384,6 @@ transfers: how-to-guide: /docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.gcs_to_wasb - connection-types: - hook-class-name: airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook hook-name: "Azure" @@ -417,6 +416,10 @@ connection-types: label: Workload Identity Tenant ID schema: type: ["string", "null"] + cloud_environment: + label: Azure Cloud Environment + schema: + type: ["string", "null"] - hook-class-name: airflow.providers.microsoft.azure.hooks.ai_agents.AzureAIAgentsHook hook-name: "Azure AI Foundry Hosted Agents" connection-type: azure_ai_agents diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py index e6a6be7c87b1d..bf7ab5e96a808 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py @@ -408,6 +408,10 @@ def get_provider_info(): "label": "Workload Identity Tenant ID", "schema": {"type": ["string", "null"]}, }, + "cloud_environment": { + "label": "Azure Cloud Environment", + "schema": {"type": ["string", "null"]}, + }, }, }, { diff --git a/providers/snowflake/provider.yaml b/providers/snowflake/provider.yaml index ecb3451a27453..2189d5a1ef7f8 100644 --- a/providers/snowflake/provider.yaml +++ b/providers/snowflake/provider.yaml @@ -252,6 +252,12 @@ connection-types: - string - 'null' format: password + workload_identity_provider: + label: Workload Identity Provider + schema: + type: + - string + - 'null' insecure_mode: label: Insecure Mode schema: diff --git a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py index e5b5260387646..7e20af8004ee2 100644 --- a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py +++ b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py @@ -142,6 +142,10 @@ def get_provider_info(): "label": "Private key (Text)", "schema": {"type": ["string", "null"], "format": "password"}, }, + "workload_identity_provider": { + "label": "Workload Identity Provider", + "schema": {"type": ["string", "null"]}, + }, "insecure_mode": { "label": "Insecure Mode", "schema": {"type": ["boolean", "null"]}, diff --git a/scripts/ci/prek/check_provider_conn_fields.py b/scripts/ci/prek/check_provider_conn_fields.py index ed4faf157c553..8d50c16335b01 100644 --- a/scripts/ci/prek/check_provider_conn_fields.py +++ b/scripts/ci/prek/check_provider_conn_fields.py @@ -81,22 +81,23 @@ def build_mismatch_error( hook_class_name: str, ) -> str | None: """ - Check that every key declared in ``conn-fields`` exists in - ``get_connection_form_widgets()``. - - ``conn-fields`` is the new React UI's view of a connection type and is - intentionally a *subset* of the Flask form widgets — fields can be omitted - from ``conn-fields`` on purpose. We therefore only flag keys that appear in - ``conn-fields`` but are absent from the hook's form (invalid / stale - declarations). The reverse direction (hook fields not listed in - ``conn-fields``) is not an error. - - Return an error string when stale keys are found, or ``None`` when the - declared keys are all valid. + Check that ``conn-fields`` and ``get_connection_form_widgets()`` are in sync. + + Once a provider declares ``conn-fields``, those declarations must exactly + match the hook's form widgets — we flag both directions: + + - Keys present in ``conn-fields`` but absent from the hook → stale / invalid + declarations that should be removed. + - Keys present in the hook but absent from ``conn-fields`` → fields that will + be invisible in the new React connection UI, almost certainly unintentional. + + Return an error string when any mismatch is found, or ``None`` when the sets + are identical. """ only_in_yaml = yaml_keys - hook_keys + only_in_hook = hook_keys - yaml_keys - if not only_in_yaml: + if not only_in_yaml and not only_in_hook: return None lines = [ @@ -104,9 +105,16 @@ def build_mismatch_error( f"`{hook_class_name}.get_connection_form_widgets()` " f"for connection-type '{connection_type}':" ] - lines.append( - " Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): " - + ", ".join(sorted(only_in_yaml)) - ) - lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.") + if only_in_yaml: + lines.append( + " Fields in provider.yaml conn-fields but NOT in get_connection_form_widgets(): " + + ", ".join(sorted(only_in_yaml)) + ) + lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from conn-fields in provider.yaml.") + if only_in_hook: + lines.append( + " Fields in get_connection_form_widgets() but NOT in provider.yaml conn-fields: " + + ", ".join(sorted(only_in_hook)) + ) + lines.append("[yellow]How to fix it[/]: Add the missing key(s) to conn-fields in provider.yaml.") return "\n".join(lines) diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 922e84b710bc4..9518e75ad6591 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -481,14 +481,11 @@ def check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict @run_check("Checking that conn-fields in provider.yaml match get_connection_form_widgets() of the hook class") def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) -> tuple[int, int]: """ - For every connection-type entry whose hook declares ``conn-fields``, - verify that every key in ``conn-fields`` also exists in the hook's - ``get_connection_form_widgets()``. - - ``conn-fields`` is optional and is intentionally allowed to be a *subset* - of the hook's form widgets (the new React UI may expose fewer fields than - the legacy Flask form), so extra hook widgets are not flagged — only - ``conn-fields`` keys absent from the hook are reported as stale. + For every connection-type entry that both declares ``conn-fields`` and whose + hook overrides ``get_connection_form_widgets()``, verify the two sets are + identical — stale YAML keys (in ``conn-fields`` but not in the hook) and + missing YAML keys (in the hook but absent from ``conn-fields``) are both + reported as errors. """ num_checks = 0 num_errors = 0 diff --git a/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py b/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py index caa2b2fc35264..db63f6783aaf5 100644 --- a/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py +++ b/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py @@ -54,8 +54,6 @@ class TestBuildMismatchError: [ pytest.param({"a", "b"}, {"a", "b"}, id="matching-keys"), pytest.param(set(), set(), id="empty-sets"), - # Hook may have more keys than conn-fields — that is intentional (subset allowed). - pytest.param({"a"}, {"a", "extra_hook"}, id="hook-has-extra-keys-no-error"), ], ) def test_no_mismatch_returns_none(self, yaml_keys, hook_keys): @@ -69,18 +67,26 @@ def test_no_mismatch_returns_none(self, yaml_keys, hook_keys): {"a"}, ["extra_yaml", "NOT in get_connection_form_widgets"], ["NOT in provider.yaml conn-fields"], - id="extra-in-yaml", + id="extra-in-yaml-only", + ), + pytest.param( + {"a"}, + {"a", "extra_hook"}, + ["extra_hook", "NOT in provider.yaml conn-fields"], + ["NOT in get_connection_form_widgets"], + id="extra-in-hook-only", ), - # only_in_hook no longer triggers an error — only only_in_yaml does pytest.param( {"a", "only_yaml"}, {"a", "only_hook"}, [ "only_yaml", "NOT in get_connection_form_widgets", + "only_hook", + "NOT in provider.yaml conn-fields", ], - ["only_hook", "NOT in provider.yaml conn-fields"], - id="both-sides-only-yaml-reported", + [], + id="both-sides-both-reported", ), ], ) @@ -103,8 +109,6 @@ class TestCheckConnFieldsForEntry: pytest.param(None, _skip, id="skip-missing-conn-fields-when-hook-has-no-widgets"), # Hook with widgets but no conn-fields is allowed: new UI intentionally omits custom fields. pytest.param(None, _get_keys("field_a"), id="no-conn-fields-with-hook-widgets-is-ok"), - # Hook has extra keys not in conn-fields — allowed (conn-fields is a valid subset). - pytest.param(["a"], _get_keys("a", "extra_hook"), id="hook-extra-keys-no-error"), ], ) def test_no_errors(self, conn_fields, get_keys): @@ -114,6 +118,9 @@ def test_no_errors(self, conn_fields, get_keys): "conn_fields, get_keys, expected_in_error", [ pytest.param(["a", "extra"], _get_keys("a"), "extra", id="extra-key-in-yaml"), + pytest.param( + ["a"], _get_keys("a", "extra_hook"), "extra_hook", id="hook-has-extra-key-not-in-yaml" + ), pytest.param(["a"], _raise, "boom", id="unexpected-exception-message"), pytest.param(["a"], _raise, HOOK_CLASS, id="unexpected-exception-mentions-hook-class"), ],