Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,6 @@ repos:
language: node
files: >
(?x)
^task-sdk/src/airflow/sdk/execution_time/schema/schema\.json$|
^ts-sdk/src/generated/supervisor\.ts$|
^ts-sdk/scripts/generate-supervisor\.mjs$
additional_dependencies: ['pnpm@10.28.1']
Expand Down
4 changes: 4 additions & 0 deletions dev/breeze/doc/ci/04_selective_checks.md
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,10 @@ when some files are not changed. Those are the rules implemented:
* if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle
wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do
not touch `java-sdk/`)
* if no `TS SDK files` (`ts-sdk/`) changed - `check-ts-sdk-supervisor-schema` check is
skipped (it regenerates and diffs the generated ts-sdk file; a change to the supervisor
wire schema alone deliberately does not trigger it - regenerating the ts-sdk types is
the ts-sdk follow-up PR's job, not the schema author's)
* if no `All Providers Python files` and no `All Providers Yaml files` are changed -
`check-provider-yaml-valid` check is skipped

Expand Down
9 changes: 9 additions & 0 deletions dev/breeze/src/airflow_breeze/utils/selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ class FileGroupForCi(Enum):
TASK_SDK_INTEGRATION_TEST_FILES = auto()
GO_SDK_FILES = auto()
JAVA_SDK_FILES = auto()
TS_SDK_FILES = auto()
AIRFLOW_CTL_FILES = auto()
AIRFLOW_CTL_INTEGRATION_TEST_FILES = auto()
BREEZE_INTEGRATION_TEST_FILES = auto()
Expand Down Expand Up @@ -436,6 +437,9 @@ def __hash__(self):
FileGroupForCi.JAVA_SDK_FILES: [
r"^java-sdk/",
],
FileGroupForCi.TS_SDK_FILES: [
r"^ts-sdk/",
],
FileGroupForCi.ASSET_FILES: [
r"^airflow-core/src/airflow/assets/",
r"^airflow-core/src/airflow/models/assets/",
Expand Down Expand Up @@ -1612,6 +1616,11 @@ def skip_prek_hooks(self) -> str:
# on a cold cache. Skip it when no java-sdk files changed so unrelated PRs do not
# depend on that (intermittently failing) download.
prek_hooks_to_skip.add("ktlint")
if not self._matching_files(FileGroupForCi.TS_SDK_FILES, CI_FILE_GROUP_MATCHES):
# This hook regenerates ts-sdk/src/generated/supervisor.ts from the wire schema and
# diffs it. Schema-only changes deliberately do not trigger it: regenerating the
# ts-sdk types is the ts-sdk follow-up PR's job, not the schema author's.
prek_hooks_to_skip.add("check-ts-sdk-supervisor-schema")
if not (
self._matching_files(
FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES
Expand Down
84 changes: 61 additions & 23 deletions dev/breeze/tests/test_selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@


ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = (
"check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -115,7 +115,7 @@
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock"

ALL_SKIPPED_COMMITS_IF_NO_UI = (
"identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
Expand All @@ -125,7 +125,7 @@
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
"identity,ktlint,lint-helm-chart,"
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -136,7 +136,7 @@
)

ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
"identity,ktlint,lint-helm-chart,"
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -151,7 +151,7 @@
# forced. airflow-core Python changed (so mypy-airflow-core + flynt run); no
# provider.yaml, helm, or UI files changed, so those checks stay skipped.
ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -164,7 +164,7 @@
)

ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
"check-provider-yaml-valid,identity,ktlint,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -176,7 +176,7 @@
)

ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -189,7 +189,7 @@


ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -201,7 +201,7 @@
)

ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
"check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand All @@ -212,7 +212,7 @@
)

ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = (
"check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -457,7 +457,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "true",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -503,7 +503,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"run-api-tests": "true",
"docs-build": "true",
"skip-prek-hooks": (
"identity,ktlint,lint-helm-chart,"
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -754,7 +754,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"docs-build": "true",
"full-tests-needed": "false",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -792,7 +792,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"docs-build": "false",
"full-tests-needed": "false",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -829,7 +829,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"docs-build": "true",
"full-tests-needed": "false",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -865,7 +865,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"docs-build": "false",
"full-tests-needed": "false",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -1095,7 +1095,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"run-unit-tests": "true",
"docs-build": "false",
# no python files changed so flynt should not run
"skip-prek-hooks": "flynt," + ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
"skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS.replace(
"check-ts-sdk-supervisor-schema,", "check-ts-sdk-supervisor-schema,flynt,", 1
),
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
Expand Down Expand Up @@ -1206,7 +1208,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"docs-build": "false",
"run-kubernetes-tests": "false",
"skip-prek-hooks": (
"identity,ktlint,lint-helm-chart,"
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -1370,7 +1372,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "true",
"skip-prek-hooks": (
"check-provider-yaml-valid,flynt,identity,ktlint,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -1584,7 +1586,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
("shared/logging/src/airflow_shared/logging/remote.py",),
{
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,"
"mypy-airflow-e2e-tests,mypy-dev,mypy-devel-common,mypy-docker-tests,"
"mypy-helm-tests,mypy-kubernetes-tests,mypy-scripts,"
Expand Down Expand Up @@ -1644,6 +1646,42 @@ def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli
assert ("ktlint" in skipped_hooks) is ktlint_skipped


@pytest.mark.parametrize(
("files", "hook_skipped"),
[
pytest.param(
("ts-sdk/src/generated/supervisor.ts",),
False,
id="runs when ts-sdk files change",
),
pytest.param(
("task-sdk/src/airflow/sdk/execution_time/schema/schema.json",),
True,
id="skipped when only the supervisor schema changes",
),
pytest.param(
("SECURITY.md",),
True,
id="skipped when no ts-sdk files change",
),
],
)
def test_check_ts_sdk_supervisor_schema_hook_only_runs_for_relevant_changes(
files: tuple[str, ...], hook_skipped: bool
):
# This hook regenerates and diffs ts-sdk/src/generated/supervisor.ts. Only ts-sdk changes
# trigger it; regenerating after a wire-schema bump is the ts-sdk follow-up PR's job.
stderr = SelectiveChecks(
files=files,
commit_ref=NEUTRAL_COMMIT,
github_event=GithubEvents.PULL_REQUEST,
pr_labels=tuple(),
default_branch="main",
)
skipped_hooks = get_outputs_from_stderr(str(stderr))["skip-prek-hooks"].split(",")
assert ("check-ts-sdk-supervisor-schema" in skipped_hooks) is hook_skipped


def test_java_sdk_version_is_emitted_as_output():
# The lang-SDK k8s job reads this to pick the JDK for the native Java build via actions/setup-java.
stderr = SelectiveChecks(
Expand Down Expand Up @@ -2406,7 +2444,7 @@ def test_expected_output_push(
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down Expand Up @@ -2448,7 +2486,7 @@ def test_expected_output_push(
"microsoft.mssql mongo mysql openlineage oracle postgres "
"presto salesforce samba sftp ssh standard trino",
"skip-prek-hooks": (
"identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
Expand Down Expand Up @@ -2493,7 +2531,7 @@ def test_expected_output_push(
"docs-build": "true",
"docs-list-as-string": "apache-airflow",
"skip-prek-hooks": (
"check-provider-yaml-valid,identity,ktlint,lint-helm-chart,"
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
Expand Down
8 changes: 8 additions & 0 deletions ts-sdk/src/generated/supervisor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,12 +372,18 @@ export type After = string | null;
export type Before = string | null;
export type Limit = number | null;
export type Ascending = boolean;
export type Extra7 = {
[k: string]: string;
} | null;
export type Type29 = "GetAssetEventByAsset";
export type AliasName = string;
export type After1 = string | null;
export type Before1 = string | null;
export type Limit1 = number | null;
export type Ascending1 = boolean;
export type Extra8 = {
[k: string]: string;
} | null;
export type Type30 = "GetAssetEventByAssetAlias";
export type Name11 = string;
export type Key6 = string;
Expand Down Expand Up @@ -1217,6 +1223,7 @@ export interface GetAssetEventByAsset {
before?: Before;
limit?: Limit;
ascending?: Ascending;
extra?: Extra7;
type?: Type29;
}
/**
Expand All @@ -1229,6 +1236,7 @@ export interface GetAssetEventByAssetAlias {
before?: Before1;
limit?: Limit1;
ascending?: Ascending1;
extra?: Extra8;
type?: Type30;
}
/**
Expand Down
Loading