diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1a0de273842c..50a49acb93adb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -270,7 +270,6 @@ repos: language: node files: > (?x) - ^task-sdk/src/airflow/sdk/execution_time/schema/schema\.json$| ^ts-sdk/src/generated/supervisor\.ts$| ^ts-sdk/scripts/generate-supervisor\.mjs$ additional_dependencies: ['pnpm@10.28.1'] diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index eecf35b853cd6..1f6c9ef2bb059 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -480,6 +480,10 @@ when some files are not changed. Those are the rules implemented: * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do not touch `java-sdk/`) + * if no `TS SDK files` (`ts-sdk/`) changed - `check-ts-sdk-supervisor-schema` check is + skipped (it regenerates and diffs the generated ts-sdk file; a change to the supervisor + wire schema alone deliberately does not trigger it - regenerating the ts-sdk types is + the ts-sdk follow-up PR's job, not the schema author's) * if no `All Providers Python files` and no `All Providers Yaml files` are changed - `check-provider-yaml-valid` check is skipped diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 0648b83515c79..1cb4ded14969b 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -120,6 +120,7 @@ class FileGroupForCi(Enum): TASK_SDK_INTEGRATION_TEST_FILES = auto() GO_SDK_FILES = auto() JAVA_SDK_FILES = auto() + TS_SDK_FILES = auto() AIRFLOW_CTL_FILES = auto() AIRFLOW_CTL_INTEGRATION_TEST_FILES = auto() BREEZE_INTEGRATION_TEST_FILES = auto() @@ -436,6 +437,9 @@ def __hash__(self): FileGroupForCi.JAVA_SDK_FILES: [ r"^java-sdk/", ], + FileGroupForCi.TS_SDK_FILES: [ + r"^ts-sdk/", + ], FileGroupForCi.ASSET_FILES: [ r"^airflow-core/src/airflow/assets/", r"^airflow-core/src/airflow/models/assets/", @@ -1612,6 +1616,11 @@ def skip_prek_hooks(self) -> str: # on a cold cache. Skip it when no java-sdk files changed so unrelated PRs do not # depend on that (intermittently failing) download. prek_hooks_to_skip.add("ktlint") + if not self._matching_files(FileGroupForCi.TS_SDK_FILES, CI_FILE_GROUP_MATCHES): + # This hook regenerates ts-sdk/src/generated/supervisor.ts from the wire schema and + # diffs it. Schema-only changes deliberately do not trigger it: regenerating the + # ts-sdk types is the ts-sdk follow-up PR's job, not the schema author's. + prek_hooks_to_skip.add("check-ts-sdk-supervisor-schema") if not ( self._matching_files( FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index cbecbdd94429d..694ecdbc301e7 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -101,7 +101,7 @@ ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = ( - "check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -115,7 +115,7 @@ ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI = ( - "identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," @@ -125,7 +125,7 @@ "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = ( - "identity,ktlint,lint-helm-chart," + "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -136,7 +136,7 @@ ) ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = ( - "identity,ktlint,lint-helm-chart," + "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -151,7 +151,7 @@ # forced. airflow-core Python changed (so mypy-airflow-core + flynt run); no # provider.yaml, helm, or UI files changed, so those checks stay skipped. ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -164,7 +164,7 @@ ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( - "check-provider-yaml-valid,identity,ktlint," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -176,7 +176,7 @@ ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -189,7 +189,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -201,7 +201,7 @@ ) ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = ( - "check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -212,7 +212,7 @@ ) ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = ( - "check-provider-yaml-valid,flynt,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -457,7 +457,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -503,7 +503,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-api-tests": "true", "docs-build": "true", "skip-prek-hooks": ( - "identity,ktlint,lint-helm-chart," + "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -754,7 +754,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "full-tests-needed": "false", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -792,7 +792,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "false", "full-tests-needed": "false", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -829,7 +829,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "full-tests-needed": "false", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -865,7 +865,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "false", "full-tests-needed": "false", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -1095,7 +1095,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-unit-tests": "true", "docs-build": "false", # no python files changed so flynt should not run - "skip-prek-hooks": "flynt," + ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS, + "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS.replace( + "check-ts-sdk-supervisor-schema,", "check-ts-sdk-supervisor-schema,flynt,", 1 + ), "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", @@ -1206,7 +1208,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "false", "run-kubernetes-tests": "false", "skip-prek-hooks": ( - "identity,ktlint,lint-helm-chart," + "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -1370,7 +1372,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-prek-hooks": ( - "check-provider-yaml-valid,flynt,identity,ktlint," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -1584,7 +1586,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("shared/logging/src/airflow_shared/logging/remote.py",), { "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests," "mypy-airflow-e2e-tests,mypy-dev,mypy-devel-common,mypy-docker-tests," "mypy-helm-tests,mypy-kubernetes-tests,mypy-scripts," @@ -1644,6 +1646,42 @@ def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli assert ("ktlint" in skipped_hooks) is ktlint_skipped +@pytest.mark.parametrize( + ("files", "hook_skipped"), + [ + pytest.param( + ("ts-sdk/src/generated/supervisor.ts",), + False, + id="runs when ts-sdk files change", + ), + pytest.param( + ("task-sdk/src/airflow/sdk/execution_time/schema/schema.json",), + True, + id="skipped when only the supervisor schema changes", + ), + pytest.param( + ("SECURITY.md",), + True, + id="skipped when no ts-sdk files change", + ), + ], +) +def test_check_ts_sdk_supervisor_schema_hook_only_runs_for_relevant_changes( + files: tuple[str, ...], hook_skipped: bool +): + # This hook regenerates and diffs ts-sdk/src/generated/supervisor.ts. Only ts-sdk changes + # trigger it; regenerating after a wire-schema bump is the ts-sdk follow-up PR's job. + stderr = SelectiveChecks( + files=files, + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PULL_REQUEST, + pr_labels=tuple(), + default_branch="main", + ) + skipped_hooks = get_outputs_from_stderr(str(stderr))["skip-prek-hooks"].split(",") + assert ("check-ts-sdk-supervisor-schema" in skipped_hooks) is hook_skipped + + def test_java_sdk_version_is_emitted_as_output(): # The lang-SDK k8s job reads this to pick the JDK for the native Java build via actions/setup-java. stderr = SelectiveChecks( @@ -2406,7 +2444,7 @@ def test_expected_output_push( "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," @@ -2448,7 +2486,7 @@ def test_expected_output_push( "microsoft.mssql mongo mysql openlineage oracle postgres " "presto salesforce samba sftp ssh standard trino", "skip-prek-hooks": ( - "identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," + "check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," "mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging," @@ -2493,7 +2531,7 @@ def test_expected_output_push( "docs-build": "true", "docs-list-as-string": "apache-airflow", "skip-prek-hooks": ( - "check-provider-yaml-valid,identity,ktlint,lint-helm-chart," + "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart," "mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests," "mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests," "mypy-scripts," diff --git a/ts-sdk/src/generated/supervisor.ts b/ts-sdk/src/generated/supervisor.ts index 004ad49e7baba..4766c0e8a7949 100644 --- a/ts-sdk/src/generated/supervisor.ts +++ b/ts-sdk/src/generated/supervisor.ts @@ -372,12 +372,18 @@ export type After = string | null; export type Before = string | null; export type Limit = number | null; export type Ascending = boolean; +export type Extra7 = { + [k: string]: string; +} | null; export type Type29 = "GetAssetEventByAsset"; export type AliasName = string; export type After1 = string | null; export type Before1 = string | null; export type Limit1 = number | null; export type Ascending1 = boolean; +export type Extra8 = { + [k: string]: string; +} | null; export type Type30 = "GetAssetEventByAssetAlias"; export type Name11 = string; export type Key6 = string; @@ -1217,6 +1223,7 @@ export interface GetAssetEventByAsset { before?: Before; limit?: Limit; ascending?: Ascending; + extra?: Extra7; type?: Type29; } /** @@ -1229,6 +1236,7 @@ export interface GetAssetEventByAssetAlias { before?: Before1; limit?: Limit1; ascending?: Ascending1; + extra?: Extra8; type?: Type30; } /**