Limit the full tests to representative items in Python/Backend matrix

dev/breeze/src/airflow_breeze/utils/exclude_from_matrix.py

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+
+def representative_combos(list_1: list[str], list_2: list[str]) -> list[tuple[str, str]]:
+    """
+    Include only representative combos from the matrix of the two lists - making sure that each of the
+    elements contributing is present at least once.
+    :param list_1: first list
+    :param list_2: second list
+    :return: list of combinations with guaranteed at least one element from each of the list
+    """
+    all_selected_combinations: list[tuple[str, str]] = []
+    for i in range(max(len(list_1), len(list_2))):
+        all_selected_combinations.append((list_1[i % len(list_1)], list_2[i % len(list_2)]))
+    return all_selected_combinations
+
+
+def excluded_combos(list_1: list[str], list_2: list[str]) -> list[tuple[str, str]]:
+    """
+    Return exclusion lists of elements that should be excluded from the matrix of the two list of items
+    if what's left should be representative list of combos (i.e. each item from both lists,
+    has to be present at least once in the combos).
+    :param list_1: first list
+    :param list_2: second list
+    :return: list of exclusions = list 1 x list 2 - representative_combos
+    """
+    all_combos: list[tuple[str, str]] = []
+    for item_1 in list_1:
+        for item_2 in list_2:
+            all_combos.append((item_1, item_2))
+    return [item for item in all_combos if item not in set(representative_combos(list_1, list_2))]

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -21,6 +21,7 @@
 import sys
 from enum import Enum
 
+from airflow_breeze.utils.exclude_from_matrix import excluded_combos
 from airflow_breeze.utils.github_actions import get_ga_output
 from airflow_breeze.utils.kubernetes_utils import get_kubernetes_python_combos
 from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
@@ -353,19 +354,46 @@ def providers_package_format_exclude(self) -> list[dict[str, str]]:
 
     @cached_property
     def postgres_exclude(self) -> list[dict[str, str]]:
-        return [{"python-version": "3.7"}] if self.full_tests_needed else []
+        if not self.full_tests_needed:
+            # Only basic combination so we do not need to exclude anything
+            return []
+        return [
+            # Exclude all combinations that are repeating python/postgres versions
+            {"python-version": python_version, "postgres-version": postgres_version}
+            for python_version, postgres_version in excluded_combos(
+                CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, CURRENT_POSTGRES_VERSIONS
+            )
+        ]
 
     @cached_property
     def mssql_exclude(self) -> list[dict[str, str]]:
-        return [{"python-version": "3.8"}] if self.full_tests_needed else []
+        if not self.full_tests_needed:
+            # Only basic combination so we do not need to exclude anything
+            return []
+        return [
+            # Exclude all combinations that are repeating python/mssql versions
+            {"python-version": python_version, "mssql-version": mssql_version}
+            for python_version, mssql_version in excluded_combos(
+                CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, CURRENT_MSSQL_VERSIONS
+            )
+        ]
 
     @cached_property
     def mysql_exclude(self) -> list[dict[str, str]]:
-        return [{"python-version": "3.10"}] if self.full_tests_needed else []
+        if not self.full_tests_needed:
+            # Only basic combination so we do not need to exclude anything
+            return []
+        return [
+            # Exclude all combinations that are repeating python/mysql versions
+            {"python-version": python_version, "mysql-version": mysql_version}
+            for python_version, mysql_version in excluded_combos(
+                CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, CURRENT_MYSQL_VERSIONS
+            )
+        ]
 
     @cached_property
     def sqlite_exclude(self) -> list[dict[str, str]]:
-        return [{"python-version": "3.9"}] if self.full_tests_needed else []
+        return []
 
     @cached_property
     def kubernetes_versions(self) -> list[str]:

dev/breeze/tests/test_exclude_from_matrix.py

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+
+from airflow_breeze.utils.exclude_from_matrix import excluded_combos, representative_combos
+
+
+@pytest.mark.parametrize(
+    "list_1, list_2, expected_representative_list",
+    [
+        (["3.6", "3.7"], ["1", "2"], [("3.6", "1"), ("3.7", "2")]),
+        (["3.6", "3.7"], ["1", "2", "3"], [("3.6", "1"), ("3.7", "2"), ("3.6", "3")]),
+        (["3.6", "3.7"], ["1", "2", "3", "4"], [("3.6", "1"), ("3.7", "2"), ("3.6", "3"), ("3.7", "4")]),
+        (
+            ["3.6", "3.7", "3.8"],
+            ["1", "2", "3", "4"],
+            [("3.6", "1"), ("3.7", "2"), ("3.8", "3"), ("3.6", "4")],
+        ),
+    ],
+)
+def test_exclude_from_matrix(
+    list_1: list[str],
+    list_2: list[str],
+    expected_representative_list: dict[str, str],
+):
+    representative_list = representative_combos(list_1, list_2)
+    exclusion_list = excluded_combos(list_1, list_2)
+    assert representative_list == expected_representative_list
+    assert len(representative_list) == len(list_1) * len(list_2) - len(exclusion_list)
+    assert len(set(representative_list) & set(exclusion_list)) == 0