Add possibility to collect all provider tests for compatibility testing

contributing-docs/testing/unit_tests.rst

@@ -1167,11 +1167,26 @@ are not part of the public API. We deal with it in one of the following ways:
   def test_plugin():
      pass
 
+5) Sometimes Pytest collection fails to work, when certain imports used by the tests either do not exist
+   or fail with RuntimeError about compatibility ("minimum Airflow version is required") or because they
+   raise AirflowOptionalProviderFeatureException. In such case you should wrap the imports in
+   ``ignore_provider_compatibility_error`` context manager adding the ``__file__``
+   module name as parameter.  This will stop failing pytest collection and automatically skip the whole
+   module from tests.
+
+   For example:
+
+.. code-block::python
+
+   with ignore_provider_compatibility_error("2.8.0", __file__):
+       from airflow.providers.common.io.xcom.backend import XComObjectStorageBackend
+
+6) In some cases in order to enable collection of pytest on older airflow version you might need to convert
+   top-level import into a local import, so that Pytest parser does not fail on collection.
 
 Running provider compatibility tests in CI
 ------------------------------------------
 
-
 In CI those tests are run in a slightly more complex way because we want to run them against the build
 provider packages, rather than mounted from sources.
 

tests/providers/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py

@@ -21,9 +21,12 @@
 import pytest
 from flask import Flask
 
-from airflow.providers.amazon.aws.auth_manager.security_manager.aws_security_manager_override import (
-    AwsSecurityManagerOverride,
-)
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.8.0", __file__):
+    from airflow.providers.amazon.aws.auth_manager.security_manager.aws_security_manager_override import (
+        AwsSecurityManagerOverride,
+    )
 from airflow.www.extensions.init_appbuilder import init_appbuilder
 
 

tests/providers/amazon/aws/auth_manager/test_aws_auth_manager.py

@@ -23,16 +23,27 @@
 from flask import Flask, session
 from flask_appbuilder.menu import MenuItem
 
-from airflow.auth.managers.models.resource_details import (
-    AccessView,
-    ConfigurationDetails,
-    ConnectionDetails,
-    DagAccessEntity,
-    DagDetails,
-    DatasetDetails,
-    PoolDetails,
-    VariableDetails,
-)
+from tests.test_utils.compat import AIRFLOW_V_2_8_PLUS
+
+try:
+    from airflow.auth.managers.models.resource_details import (
+        AccessView,
+        ConfigurationDetails,
+        ConnectionDetails,
+        DagAccessEntity,
+        DagDetails,
+        DatasetDetails,
+        PoolDetails,
+        VariableDetails,
+    )
+except ImportError:
+    if not AIRFLOW_V_2_8_PLUS:
+        pytest.skip(
+            "Skipping tests that require AwsSecurityManagerOverride for Airflow < 2.8.0",
+            allow_module_level=True,
+        )
+    else:
+        raise
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities
 from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade
 from airflow.providers.amazon.aws.auth_manager.aws_auth_manager import AwsAuthManager

tests/providers/common/io/operators/test_file_transfer.py

@@ -21,7 +21,10 @@
 
 from openlineage.client.run import Dataset
 
-from airflow.providers.common.io.operators.file_transfer import FileTransferOperator
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.8.0", __file__):
+    from airflow.providers.common.io.operators.file_transfer import FileTransferOperator
 
 
 def test_file_transfer_copy():

tests/providers/common/io/xcom/test_backend.py

@@ -19,8 +19,7 @@
 
 import pytest
 
-from airflow.exceptions import AirflowOptionalProviderFeatureException
-from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS
+from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS, ignore_provider_compatibility_error
 
 pytestmark = [
     pytest.mark.db_test,
@@ -32,10 +31,9 @@
 from airflow.models.xcom import BaseXCom, resolve_xcom_backend
 from airflow.operators.empty import EmptyOperator
 
-try:
+with ignore_provider_compatibility_error("2.8.0", __file__):
     from airflow.providers.common.io.xcom.backend import XComObjectStorageBackend
-except AirflowOptionalProviderFeatureException:
-    pass
+
 from airflow.utils import timezone
 from airflow.utils.xcom import XCOM_RETURN_KEY
 from tests.test_utils import db

tests/providers/fab/auth_manager/api/auth/backend/test_basic_auth.py

@@ -24,6 +24,11 @@
 
 from airflow.api.auth.backend.basic_auth import requires_authentication
 from airflow.www import app as application
+from tests.test_utils.compat import AIRFLOW_V_2_9_PLUS
+
+pytestmark = [
+    pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Tests for Airflow 2.9.0+ only"),
+]
 
 
 @pytest.fixture

tests/providers/fab/auth_manager/api/auth/backend/test_kerberos_auth.py

@@ -19,7 +19,10 @@
 from airflow.api.auth.backend.kerberos_auth import (
     init_app as base_init_app,
 )
-from airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth import init_app
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth import init_app
 
 
 class TestKerberosAuth:

tests/providers/fab/auth_manager/api_endpoints/test_role_and_permission_endpoint.py

@@ -19,8 +19,13 @@
 import pytest
 
 from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP
-from airflow.providers.fab.auth_manager.models import Role
-from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.models import Role
+    from airflow.providers.fab.auth_manager.security_manager.override import EXISTING_ROLES
+
+
 from airflow.security import permissions
 from tests.test_utils.api_connexion_utils import (
     assert_401,

tests/providers/fab/auth_manager/api_endpoints/test_user_endpoint.py

@@ -22,15 +22,20 @@
 from sqlalchemy.sql.functions import count
 
 from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP
-from airflow.providers.fab.auth_manager.models import User
 from airflow.security import permissions
 from airflow.utils import timezone
 from airflow.utils.session import create_session
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.models import User
+
 from tests.test_utils.api_connexion_utils import assert_401, create_user, delete_role, delete_user
 from tests.test_utils.config import conf_vars
 
 pytestmark = pytest.mark.db_test
 
+
 DEFAULT_TIME = "2020-06-11T18:00:00+00:00"
 
 

tests/providers/fab/auth_manager/api_endpoints/test_user_schema.py

@@ -18,15 +18,20 @@
 
 import pytest
 
-from airflow.api_connexion.schemas.user_schema import user_collection_item_schema, user_schema
-from airflow.providers.fab.auth_manager.models import User
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.api_connexion.schemas.user_schema import user_collection_item_schema, user_schema
+    from airflow.providers.fab.auth_manager.models import User
+
 from airflow.utils import timezone
-from tests.test_utils.api_connexion_utils import create_role, delete_role
 
 TEST_EMAIL = "test@example.org"
 
 DEFAULT_TIME = "2021-01-09T13:59:56.336000+00:00"
 
+from tests.test_utils.api_connexion_utils import create_role, delete_role  # noqa: E402
+
 pytestmark = pytest.mark.db_test
 
 

tests/providers/fab/auth_manager/cli_commands/test_definition.py

@@ -16,11 +16,14 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.providers.fab.auth_manager.cli_commands.definition import (
-    ROLES_COMMANDS,
-    SYNC_PERM_COMMAND,
-    USERS_COMMANDS,
-)
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.cli_commands.definition import (
+        ROLES_COMMANDS,
+        SYNC_PERM_COMMAND,
+        USERS_COMMANDS,
+    )
 
 
 class TestCliDefinition:

tests/providers/fab/auth_manager/cli_commands/test_role_command.py

@@ -25,12 +25,17 @@
 import pytest
 
 from airflow.cli import cli_parser
-from airflow.providers.fab.auth_manager.cli_commands import role_command
-from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.cli_commands import role_command
+    from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
+
 from airflow.security import permissions
 
 pytestmark = pytest.mark.db_test
 
+
 if TYPE_CHECKING:
     from airflow.providers.fab.auth_manager.models import Role
 

tests/providers/fab/auth_manager/cli_commands/test_sync_perm_command.py

@@ -22,7 +22,10 @@
 import pytest
 
 from airflow.cli import cli_parser
-from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.cli_commands import sync_perm_command
 
 pytestmark = pytest.mark.db_test
 

tests/providers/fab/auth_manager/cli_commands/test_user_command.py

@@ -26,8 +26,11 @@
 import pytest
 
 from airflow.cli import cli_parser
-from airflow.providers.fab.auth_manager.cli_commands import user_command
-from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.cli_commands import user_command
+    from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
 
 pytestmark = pytest.mark.db_test
 

tests/providers/fab/auth_manager/cli_commands/test_utils.py

@@ -18,11 +18,16 @@
 
 import pytest
 
-from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder
+
 from airflow.www.extensions.init_appbuilder import AirflowAppBuilder
 
+pytestmark = pytest.mark.db_test
+
 
-@pytest.mark.db_test
 class TestCliUtils:
     def test_get_application_builder(self):
         with get_application_builder() as appbuilder:

tests/providers/fab/auth_manager/decorators/test_auth.py

@@ -20,10 +20,16 @@
 
 import pytest
 
-from airflow.api_connexion.exceptions import PermissionDenied
-from airflow.providers.fab.auth_manager.decorators.auth import _has_access_fab, _requires_access_fab
 from airflow.security.permissions import ACTION_CAN_READ, RESOURCE_DAG
-from airflow.www import app as application
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+permissions = [(ACTION_CAN_READ, RESOURCE_DAG)]
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.api_connexion.exceptions import PermissionDenied
+    from airflow.providers.fab.auth_manager.decorators.auth import _has_access_fab, _requires_access_fab
+
+from airflow.www import app as application  # noqa: E402
 
 
 @pytest.fixture(scope="module")

tests/providers/fab/auth_manager/models/test_anonymous_user.py

@@ -17,7 +17,10 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.models.anonymous_user import AnonymousUser
 
 
 class TestAnonymousUser:

tests/providers/fab/auth_manager/security_manager/test_constants.py

@@ -16,7 +16,10 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.providers.fab.auth_manager.security_manager.constants import EXISTING_ROLES
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.security_manager.constants import EXISTING_ROLES
 
 
 class TestFbSecurityManagerConstants:

tests/providers/fab/auth_manager/security_manager/test_override.py

@@ -19,7 +19,10 @@
 from unittest import mock
 from unittest.mock import Mock
 
-from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
 
 
 class EmptySecurityManager(FabAirflowSecurityManagerOverride):

tests/providers/fab/auth_manager/test_fab_auth_manager.py

@@ -24,11 +24,20 @@
 import pytest
 from flask import Flask
 
-from airflow.auth.managers.models.resource_details import AccessView, DagAccessEntity, DagDetails
 from airflow.exceptions import AirflowConfigException, AirflowException
-from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager
-from airflow.providers.fab.auth_manager.models import User
-from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
+
+try:
+    from airflow.auth.managers.models.resource_details import AccessView, DagAccessEntity, DagDetails
+except ImportError:
+    pass
+
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager
+    from airflow.providers.fab.auth_manager.models import User
+    from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
+
 from airflow.security.permissions import (
     ACTION_CAN_ACCESS_MENU,
     ACTION_CAN_CREATE,

tests/providers/fab/auth_manager/test_models.py

@@ -20,10 +20,13 @@
 
 from sqlalchemy import Column, MetaData, String, Table
 
-from airflow.providers.fab.auth_manager.models import (
-    add_index_on_ab_register_user_username_postgres,
-    add_index_on_ab_user_username_postgres,
-)
+from tests.test_utils.compat import ignore_provider_compatibility_error
+
+with ignore_provider_compatibility_error("2.9.0+", __file__):
+    from airflow.providers.fab.auth_manager.models import (
+        add_index_on_ab_register_user_username_postgres,
+        add_index_on_ab_user_username_postgres,
+    )
 
 _mock_conn = mock.MagicMock()
 _mock_conn.dialect = mock.MagicMock()