From 3cab00dfc3ca253a0845e5b67b49201b5602faf4 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Tue, 26 Mar 2024 13:06:30 +0400 Subject: [PATCH 1/3] Replace to broad exceptions into the Dev --- .../airflow_breeze/commands/release_management_commands.py | 6 +++--- dev/breeze/src/airflow_breeze/global_constants.py | 2 +- dev/breeze/src/airflow_breeze/utils/docs_publisher.py | 4 ++-- dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py | 2 +- dev/breeze/src/airflow_breeze/utils/packages.py | 4 ++-- docs/conf.py | 2 +- docs/exts/docs_build/docs_builder.py | 2 +- docs/exts/operators_and_hooks_ref.py | 4 ++-- docs/exts/provider_yaml_utils.py | 2 +- hatch_build.py | 2 +- scripts/ci/pre_commit/generate_pypi_readme.py | 2 +- scripts/ci/pre_commit/insert_extras.py | 4 ++-- scripts/ci/pre_commit/json_schema.py | 4 ++-- scripts/ci/pre_commit/replace_bad_characters.py | 2 +- scripts/ci/pre_commit/update_chart_dependencies.py | 4 ++-- scripts/ci/pre_commit/update_versions.py | 2 +- scripts/in_container/run_provider_yaml_files_check.py | 4 ++-- scripts/in_container/update_quarantined_test_status.py | 4 ++-- scripts/tools/check_if_limited_dependencies.py | 2 +- scripts/tools/list-integrations.py | 2 +- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index e5ad5b042288d..ae45a14f628f2 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -1143,7 +1143,7 @@ def _get_all_providers_in_dist( for file in DIST_DIR.glob(f"{filename_prefix}*.tar.gz"): matched = filename_pattern.match(file.name) if not matched: - raise Exception(f"Cannot parse provider package name from {file.name}") + raise ValueError(f"Cannot parse provider package name from {file.name}") provider_package_id = matched.group(1).replace("_", ".") yield provider_package_id @@ -1168,7 +1168,7 @@ def get_all_providers_in_dist(package_format: str, install_selected_providers: s ) ) else: - raise Exception(f"Unknown package format {package_format}") + raise ValueError(f"Unknown package format {package_format}") if install_selected_providers: filter_list = install_selected_providers.split(",") return [provider for provider in all_found_providers if provider in filter_list] @@ -1293,7 +1293,7 @@ def install_provider_packages( if dependency not in chunk: chunk.append(dependency) if len(list_of_all_providers) != total_num_providers: - raise Exception( + raise ValueError( f"Total providers {total_num_providers} is different " f"than {len(list_of_all_providers)} (just to be sure" f" no rounding errors crippled in)" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index fbf2abc5c11c4..31565152664c2 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -363,7 +363,7 @@ def get_airflow_version(): airflow_version = line.split()[2][1:-1] break if airflow_version == "unknown": - raise Exception("Unable to determine Airflow version") + raise ValueError("Unable to determine Airflow version") return airflow_version diff --git a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py index 5150eead2bd61..f04ca0379f870 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py @@ -58,7 +58,7 @@ def _build_dir(self) -> str: @property def _current_version(self): if not self.is_versioned: - raise Exception("This documentation package is not versioned") + raise ValueError("This documentation package is not versioned") if self.package_name == "apache-airflow": return get_airflow_version() if self.package_name.startswith("apache-airflow-providers-"): @@ -66,7 +66,7 @@ def _current_version(self): return provider["versions"][0] if self.package_name == "helm-chart": return chart_version() - return Exception(f"Unsupported package: {self.package_name}") + return ValueError(f"Unsupported package: {self.package_name}") @property def _publish_dir(self) -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index 3812277d4b6ba..d14ca0553dfb8 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -92,7 +92,7 @@ def get_architecture_string_for_urls() -> str: return "amd64" if architecture == Architecture.ARM: return "arm64" - raise Exception(f"The architecture {architecture} is not supported when downloading kubernetes tools!") + raise ValueError(f"The architecture {architecture} is not supported when downloading kubernetes tools!") def _download_with_retries(num_tries, path, tool, url): diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 94ec01b419137..94c1ef8c53bd8 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -133,7 +133,7 @@ def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path): try: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") + raise ValueError(f"Unable to parse: {provider_yaml_path}.") except ImportError: # we only validate the schema if jsonschema is available. This is needed for autocomplete # to not fail with import error if jsonschema is not installed @@ -177,7 +177,7 @@ def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> jsonschema.validate(provider_info, schema=schema) except jsonschema.ValidationError as ex: get_console().print("[red]Provider info not validated against runtime schema[/]") - raise Exception( + raise ValueError( "Error when validating schema. The schema must be compatible with " "airflow/provider_info.schema.json.", ex, diff --git a/docs/conf.py b/docs/conf.py index 90c9b68e6f0ea..4dbd3f7fe4715 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ if provider_yaml["package-name"] == PACKAGE_NAME ) except StopIteration: - raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") + raise LookupError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"]) PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0] SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"] diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index 6bd2304832ef1..3a0acdbdf3f44 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -91,7 +91,7 @@ def log_build_warning_filename(self) -> str: @property def _current_version(self): if not self.is_versioned: - raise Exception("This documentation package is not versioned") + raise ValueError("This documentation package is not versioned") if self.package_name == "apache-airflow": from airflow.version import version as airflow_version diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index 3674b509efe6e..bf62a36d8a4b1 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -69,10 +69,10 @@ def _render_template(template_name, **kwargs): def _docs_path(filepath: str): if not filepath.startswith("/docs/"): - raise Exception(f"The path must starts with '/docs/'. Current value: {filepath}") + raise ValueError(f"The path must starts with '/docs/'. Current value: {filepath}") if not filepath.endswith(".rst"): - raise Exception(f"The path must ends with '.rst'. Current value: {filepath}") + raise ValueError(f"The path must ends with '.rst'. Current value: {filepath}") if filepath.startswith("/docs/apache-airflow-providers-"): _, _, provider, rest = filepath.split("/", maxsplit=3) diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index d417ba259dbdd..5a015cd518fa3 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -68,7 +68,7 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: try: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") + raise ValueError(f"Unable to parse: {provider_yaml_path}.") if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) diff --git a/hatch_build.py b/hatch_build.py index 439e486ec51df..fefaeaa625dcd 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -592,7 +592,7 @@ def get_provider_requirement(provider_spec: str) -> str: if PROVIDER_DEPENDENCIES[provider_id]["state"] not in ["ready", "suspended", "removed"]: for dependency in PROVIDER_DEPENDENCIES[provider_id]["deps"]: if dependency.startswith("apache-airflow-providers"): - raise Exception( + raise ValueError( f"The provider {provider_id} is pre-installed and it has as dependency " f"to another provider {dependency}. This is not allowed. Pre-installed" f"providers should only have 'apache-airflow' and regular dependencies." diff --git a/scripts/ci/pre_commit/generate_pypi_readme.py b/scripts/ci/pre_commit/generate_pypi_readme.py index 0223b0a46b3d4..042822b9ce086 100755 --- a/scripts/ci/pre_commit/generate_pypi_readme.py +++ b/scripts/ci/pre_commit/generate_pypi_readme.py @@ -54,7 +54,7 @@ def extract_section(content, section_name): if section_match: return section_match.group(1) else: - raise Exception(f"Cannot find section {section_name} in README.md") + raise LookupError(f"Cannot find section {section_name} in README.md") if __name__ == "__main__": diff --git a/scripts/ci/pre_commit/insert_extras.py b/scripts/ci/pre_commit/insert_extras.py index e32ad199b43e8..16faeaad2cac9 100755 --- a/scripts/ci/pre_commit/insert_extras.py +++ b/scripts/ci/pre_commit/insert_extras.py @@ -42,7 +42,7 @@ def get_header_and_footer(extra_type: str, file_format: str) -> tuple[str, str]: elif file_format == "txt": return f"# START {extra_type.upper()} HERE", f"# END {extra_type.upper()} HERE" else: - raise Exception(f"Bad format {format} passed. Only rst and txt are supported") + raise ValueError(f"Bad format {format} passed. Only rst and txt are supported") def get_wrapped_list(extras_set: list[str]) -> list[str]: @@ -81,7 +81,7 @@ def process_documentation_files() -> bool: extra_type_dict = get_extra_types_dict() for file, file_format, add_comment in FILES_TO_UPDATE: if not file.exists(): - raise Exception(f"File {file} does not exist") + raise FileNotFoundError(f"File {file} does not exist") for extra_type_description, extra_list in extra_type_dict.items(): header, footer = get_header_and_footer(extra_type_description, file_format) if insert_documentation( diff --git a/scripts/ci/pre_commit/json_schema.py b/scripts/ci/pre_commit/json_schema.py index 886ff13fe8b27..ef58ba3dd4217 100755 --- a/scripts/ci/pre_commit/json_schema.py +++ b/scripts/ci/pre_commit/json_schema.py @@ -30,7 +30,7 @@ from jsonschema.validators import extend, validator_for if __name__ != "__main__": - raise Exception( + raise RuntimeError( "This file is intended to be executed as an executable program. You cannot use it as a module." "To run this script, run the ./build_docs.py command" ) @@ -150,7 +150,7 @@ def _load_spec(spec_file: str | None, spec_url: str | None): if spec_url: spec_file = fetch_and_cache(url=spec_url, output_filename=re.sub(r"[^a-zA-Z0-9]", "-", spec_url)) if not spec_file: - raise Exception(f"The {spec_file} was None and {spec_url} did not lead to any file loading.") + raise ValueError(f"The {spec_file} was None and {spec_url} did not lead to any file loading.") with open(spec_file) as schema_file: schema = json.loads(schema_file.read()) return schema diff --git a/scripts/ci/pre_commit/replace_bad_characters.py b/scripts/ci/pre_commit/replace_bad_characters.py index 67377f997452d..4faec0fc95a1e 100755 --- a/scripts/ci/pre_commit/replace_bad_characters.py +++ b/scripts/ci/pre_commit/replace_bad_characters.py @@ -25,7 +25,7 @@ from rich.console import Console if __name__ != "__main__": - raise Exception( + raise RuntimeError( "This file is intended to be executed as an executable program. You cannot use it as a module." f"To run this script, run the {__file__} command" ) diff --git a/scripts/ci/pre_commit/update_chart_dependencies.py b/scripts/ci/pre_commit/update_chart_dependencies.py index 939c2278972df..c86bb866d6750 100755 --- a/scripts/ci/pre_commit/update_chart_dependencies.py +++ b/scripts/ci/pre_commit/update_chart_dependencies.py @@ -36,7 +36,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str: for version in quay_data["tags"]: if version["name"].startswith("v"): return version["name"] - raise Exception("ERROR! No version found") + raise ValueError("ERROR! No version found") if __name__ == "__main__": @@ -64,7 +64,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str: next_line = f" tag: {latest_prometheus_statsd_exporter_version}" replace = False else: - raise Exception( + raise ValueError( f"ERROR! The next line after repository: should be tag: - " f"index {index} in {VALUES_YAML_FILE}" ) diff --git a/scripts/ci/pre_commit/update_versions.py b/scripts/ci/pre_commit/update_versions.py index a2e3361eacb66..cb9727e1747e0 100755 --- a/scripts/ci/pre_commit/update_versions.py +++ b/scripts/ci/pre_commit/update_versions.py @@ -31,7 +31,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path): with file_path.open("r+") as f: file_content = f.read() if not pattern.search(file_content): - raise Exception(f"Pattern {pattern!r} doesn't found in {file_path!r} file") + raise LookupError(f"Pattern {pattern!r} doesn't found in {file_path!r} file") new_content = pattern.sub(rf"\g<1>{v}\g<2>", file_content) if file_content == new_content: return diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index b1608c25bcedd..49b01e769ceb1 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -62,7 +62,7 @@ from yaml import SafeLoader # type: ignore if __name__ != "__main__": - raise Exception( + raise RuntimeError( "This file is intended to be executed as an executable program. You cannot use it as a module." ) @@ -113,7 +113,7 @@ def _load_package_data(package_paths: Iterable[str]): try: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {rel_path}.") + raise ValueError(f"Unable to parse: {rel_path}.") if not provider["state"] == "suspended": result[rel_path] = provider else: diff --git a/scripts/in_container/update_quarantined_test_status.py b/scripts/in_container/update_quarantined_test_status.py index a3f045843cdfe..621e216f1cdd1 100755 --- a/scripts/in_container/update_quarantined_test_status.py +++ b/scripts/in_container/update_quarantined_test_status.py @@ -202,14 +202,14 @@ def get_table(history_map: dict[str, TestHistory]) -> str: print(f"Token: {token}") github_repository = os.environ.get("GITHUB_REPOSITORY") if not github_repository: - raise Exception("GitHub Repository must be defined!") + raise ValueError("GitHub Repository must be defined!") user, repo = github_repository.split("/") print(f"User: {user}, Repo: {repo}") issue_id = int(os.environ.get("ISSUE_ID", 0)) num_runs = int(os.environ.get("NUM_RUNS", 10)) if issue_id == 0: - raise Exception("You need to define ISSUE_ID as environment variable") + raise ValueError("You need to define ISSUE_ID as environment variable") gh = login(token=token) diff --git a/scripts/tools/check_if_limited_dependencies.py b/scripts/tools/check_if_limited_dependencies.py index 16f879d80eb64..35e64278f1692 100755 --- a/scripts/tools/check_if_limited_dependencies.py +++ b/scripts/tools/check_if_limited_dependencies.py @@ -27,7 +27,7 @@ program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise Exception( + raise RuntimeError( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" ) diff --git a/scripts/tools/list-integrations.py b/scripts/tools/list-integrations.py index d563869c5498e..c130f1b985c37 100755 --- a/scripts/tools/list-integrations.py +++ b/scripts/tools/list-integrations.py @@ -34,7 +34,7 @@ program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise Exception( + raise RuntimeError( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" ) From c8c978b50851a951b0590d8779b661cae17eee22 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Wed, 3 Apr 2024 13:14:14 +0400 Subject: [PATCH 2/3] Add changes according to the review Co-authored-by: Jarek Potiuk Co-authored-by: Tzu-ping Chung --- .../commands/release_management_commands.py | 7 ++++--- .../src/airflow_breeze/global_constants.py | 2 +- .../src/airflow_breeze/utils/docs_publisher.py | 8 ++++++-- .../src/airflow_breeze/utils/kubernetes_utils.py | 3 ++- dev/breeze/src/airflow_breeze/utils/packages.py | 15 ++++++++------- docs/conf.py | 2 +- docs/exts/docs_build/docs_builder.py | 16 ---------------- docs/exts/operators_and_hooks_ref.py | 4 ++-- docs/exts/provider_yaml_utils.py | 5 +++-- hatch_build.py | 3 ++- scripts/ci/pre_commit/generate_pypi_readme.py | 2 +- scripts/ci/pre_commit/json_schema.py | 2 +- scripts/ci/pre_commit/replace_bad_characters.py | 2 +- .../ci/pre_commit/update_chart_dependencies.py | 2 +- scripts/ci/pre_commit/update_versions.py | 2 +- .../run_provider_yaml_files_check.py | 7 ++++--- .../update_quarantined_test_status.py | 4 ++-- 17 files changed, 40 insertions(+), 46 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index ae45a14f628f2..0fae67e485673 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -1143,7 +1143,7 @@ def _get_all_providers_in_dist( for file in DIST_DIR.glob(f"{filename_prefix}*.tar.gz"): matched = filename_pattern.match(file.name) if not matched: - raise ValueError(f"Cannot parse provider package name from {file.name}") + raise SystemExit(f"Cannot parse provider package name from {file.name}") provider_package_id = matched.group(1).replace("_", ".") yield provider_package_id @@ -1168,7 +1168,7 @@ def get_all_providers_in_dist(package_format: str, install_selected_providers: s ) ) else: - raise ValueError(f"Unknown package format {package_format}") + raise SystemExit(f"Unknown package format {package_format}") if install_selected_providers: filter_list = install_selected_providers.split(",") return [provider for provider in all_found_providers if provider in filter_list] @@ -1293,11 +1293,12 @@ def install_provider_packages( if dependency not in chunk: chunk.append(dependency) if len(list_of_all_providers) != total_num_providers: - raise ValueError( + msg = ( f"Total providers {total_num_providers} is different " f"than {len(list_of_all_providers)} (just to be sure" f" no rounding errors crippled in)" ) + raise RuntimeError(msg) parallelism = min(parallelism, len(provider_chunks)) with ci_group(f"Installing providers in {parallelism} chunks"): all_params = [f"Chunk {n}" for n in range(parallelism)] diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 31565152664c2..d2c01a0bd6f07 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -363,7 +363,7 @@ def get_airflow_version(): airflow_version = line.split()[2][1:-1] break if airflow_version == "unknown": - raise ValueError("Unable to determine Airflow version") + raise RuntimeError("Unable to determine Airflow version") return airflow_version diff --git a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py index f04ca0379f870..1d7a88e2455de 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py @@ -58,7 +58,11 @@ def _build_dir(self) -> str: @property def _current_version(self): if not self.is_versioned: - raise ValueError("This documentation package is not versioned") + msg = ( + "This documentation package is not versioned. " + "Make sure to add version in `provider.yaml` for the package." + ) + raise RuntimeError(msg) if self.package_name == "apache-airflow": return get_airflow_version() if self.package_name.startswith("apache-airflow-providers-"): @@ -66,7 +70,7 @@ def _current_version(self): return provider["versions"][0] if self.package_name == "helm-chart": return chart_version() - return ValueError(f"Unsupported package: {self.package_name}") + raise SystemExit(f"Unsupported package: {self.package_name}") @property def _publish_dir(self) -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index d14ca0553dfb8..ef94500900024 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -92,7 +92,8 @@ def get_architecture_string_for_urls() -> str: return "amd64" if architecture == Architecture.ARM: return "arm64" - raise ValueError(f"The architecture {architecture} is not supported when downloading kubernetes tools!") + msg = f"The architecture {architecture} is not supported when downloading kubernetes tools!" + raise SystemExit(msg) def _download_with_retries(num_tries, path, tool, url): diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 94c1ef8c53bd8..68e91b248924e 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -132,8 +132,9 @@ def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path): try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise ValueError(f"Unable to parse: {provider_yaml_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) except ImportError: # we only validate the schema if jsonschema is available. This is needed for autocomplete # to not fail with import error if jsonschema is not installed @@ -176,12 +177,12 @@ def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> try: jsonschema.validate(provider_info, schema=schema) except jsonschema.ValidationError as ex: - get_console().print("[red]Provider info not validated against runtime schema[/]") - raise ValueError( - "Error when validating schema. The schema must be compatible with " - "airflow/provider_info.schema.json.", - ex, + get_console().print( + "[red]Error when validating schema. The schema must be compatible with " + "[bold]'airflow/provider_info.schema.json'[/bold].\n" + f"Original exception [bold]{type(ex).__name__}: {ex}[/]" ) + raise SystemExit(1) def get_provider_info_dict(provider_id: str) -> dict[str, Any]: diff --git a/docs/conf.py b/docs/conf.py index 4dbd3f7fe4715..91b4f23996085 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ if provider_yaml["package-name"] == PACKAGE_NAME ) except StopIteration: - raise LookupError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") + raise RuntimeError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}") PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"]) PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0] SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"] diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index 3a0acdbdf3f44..afc796a505510 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -33,7 +33,6 @@ PROCESS_TIMEOUT, ) from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings -from docs.exts.docs_build.helm_chart_utils import chart_version from docs.exts.docs_build.spelling_checks import SpellingError, parse_spelling_warnings console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH) @@ -88,21 +87,6 @@ def log_build_warning_filename(self) -> str: """Warnings from build job.""" return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log") - @property - def _current_version(self): - if not self.is_versioned: - raise ValueError("This documentation package is not versioned") - if self.package_name == "apache-airflow": - from airflow.version import version as airflow_version - - return airflow_version - if self.package_name.startswith("apache-airflow-providers-"): - provider = next(p for p in ALL_PROVIDER_YAMLS if p["package-name"] == self.package_name) - return provider["versions"][0] - if self.package_name == "helm-chart": - return chart_version() - return Exception(f"Unsupported package: {self.package_name}") - @property def _src_dir(self) -> str: return f"{DOCS_DIR}/{self.package_name}" diff --git a/docs/exts/operators_and_hooks_ref.py b/docs/exts/operators_and_hooks_ref.py index bf62a36d8a4b1..43f954ebb0c37 100644 --- a/docs/exts/operators_and_hooks_ref.py +++ b/docs/exts/operators_and_hooks_ref.py @@ -69,10 +69,10 @@ def _render_template(template_name, **kwargs): def _docs_path(filepath: str): if not filepath.startswith("/docs/"): - raise ValueError(f"The path must starts with '/docs/'. Current value: {filepath}") + raise RuntimeError(f"The path must starts with '/docs/'. Current value: {filepath}") if not filepath.endswith(".rst"): - raise ValueError(f"The path must ends with '.rst'. Current value: {filepath}") + raise RuntimeError(f"The path must ends with '.rst'. Current value: {filepath}") if filepath.startswith("/docs/apache-airflow-providers-"): _, _, provider, rest = filepath.split("/", maxsplit=3) diff --git a/docs/exts/provider_yaml_utils.py b/docs/exts/provider_yaml_utils.py index 5a015cd518fa3..1eaf93a6986e7 100644 --- a/docs/exts/provider_yaml_utils.py +++ b/docs/exts/provider_yaml_utils.py @@ -67,8 +67,9 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]: provider = yaml.safe_load(yaml_file) try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise ValueError(f"Unable to parse: {provider_yaml_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) if provider["state"] == "suspended" and not include_suspended: continue provider_yaml_dir = os.path.dirname(provider_yaml_path) diff --git a/hatch_build.py b/hatch_build.py index fefaeaa625dcd..0ace661347c34 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -592,11 +592,12 @@ def get_provider_requirement(provider_spec: str) -> str: if PROVIDER_DEPENDENCIES[provider_id]["state"] not in ["ready", "suspended", "removed"]: for dependency in PROVIDER_DEPENDENCIES[provider_id]["deps"]: if dependency.startswith("apache-airflow-providers"): - raise ValueError( + msg = ( f"The provider {provider_id} is pre-installed and it has as dependency " f"to another provider {dependency}. This is not allowed. Pre-installed" f"providers should only have 'apache-airflow' and regular dependencies." ) + raise SystemExit(msg) if not dependency.startswith("apache-airflow"): PREINSTALLED_NOT_READY_DEPS.append(dependency) diff --git a/scripts/ci/pre_commit/generate_pypi_readme.py b/scripts/ci/pre_commit/generate_pypi_readme.py index 042822b9ce086..e263e798ff6c3 100755 --- a/scripts/ci/pre_commit/generate_pypi_readme.py +++ b/scripts/ci/pre_commit/generate_pypi_readme.py @@ -54,7 +54,7 @@ def extract_section(content, section_name): if section_match: return section_match.group(1) else: - raise LookupError(f"Cannot find section {section_name} in README.md") + raise RuntimeError(f"Cannot find section {section_name} in README.md") if __name__ == "__main__": diff --git a/scripts/ci/pre_commit/json_schema.py b/scripts/ci/pre_commit/json_schema.py index ef58ba3dd4217..75b999977272b 100755 --- a/scripts/ci/pre_commit/json_schema.py +++ b/scripts/ci/pre_commit/json_schema.py @@ -30,7 +30,7 @@ from jsonschema.validators import extend, validator_for if __name__ != "__main__": - raise RuntimeError( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." "To run this script, run the ./build_docs.py command" ) diff --git a/scripts/ci/pre_commit/replace_bad_characters.py b/scripts/ci/pre_commit/replace_bad_characters.py index 4faec0fc95a1e..7144eb6e9adfe 100755 --- a/scripts/ci/pre_commit/replace_bad_characters.py +++ b/scripts/ci/pre_commit/replace_bad_characters.py @@ -25,7 +25,7 @@ from rich.console import Console if __name__ != "__main__": - raise RuntimeError( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." f"To run this script, run the {__file__} command" ) diff --git a/scripts/ci/pre_commit/update_chart_dependencies.py b/scripts/ci/pre_commit/update_chart_dependencies.py index c86bb866d6750..21673f503e896 100755 --- a/scripts/ci/pre_commit/update_chart_dependencies.py +++ b/scripts/ci/pre_commit/update_chart_dependencies.py @@ -36,7 +36,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str: for version in quay_data["tags"]: if version["name"].startswith("v"): return version["name"] - raise ValueError("ERROR! No version found") + raise RuntimeError("ERROR! No version found") if __name__ == "__main__": diff --git a/scripts/ci/pre_commit/update_versions.py b/scripts/ci/pre_commit/update_versions.py index cb9727e1747e0..60947789bcec0 100755 --- a/scripts/ci/pre_commit/update_versions.py +++ b/scripts/ci/pre_commit/update_versions.py @@ -31,7 +31,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path): with file_path.open("r+") as f: file_content = f.read() if not pattern.search(file_content): - raise LookupError(f"Pattern {pattern!r} doesn't found in {file_path!r} file") + raise RuntimeError(f"Pattern {pattern!r} doesn't found in {file_path!r} file") new_content = pattern.sub(rf"\g<1>{v}\g<2>", file_content) if file_content == new_content: return diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 49b01e769ceb1..29ef9812aad8b 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -62,7 +62,7 @@ from yaml import SafeLoader # type: ignore if __name__ != "__main__": - raise RuntimeError( + raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." ) @@ -112,8 +112,9 @@ def _load_package_data(package_paths: Iterable[str]): rel_path = pathlib.Path(provider_yaml_path).relative_to(ROOT_DIR).as_posix() try: jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise ValueError(f"Unable to parse: {rel_path}.") + except jsonschema.ValidationError as ex: + msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}" + raise RuntimeError(msg) if not provider["state"] == "suspended": result[rel_path] = provider else: diff --git a/scripts/in_container/update_quarantined_test_status.py b/scripts/in_container/update_quarantined_test_status.py index 621e216f1cdd1..72cd04c33ea36 100755 --- a/scripts/in_container/update_quarantined_test_status.py +++ b/scripts/in_container/update_quarantined_test_status.py @@ -202,14 +202,14 @@ def get_table(history_map: dict[str, TestHistory]) -> str: print(f"Token: {token}") github_repository = os.environ.get("GITHUB_REPOSITORY") if not github_repository: - raise ValueError("GitHub Repository must be defined!") + raise RuntimeError("GitHub Repository must be defined!") user, repo = github_repository.split("/") print(f"User: {user}, Repo: {repo}") issue_id = int(os.environ.get("ISSUE_ID", 0)) num_runs = int(os.environ.get("NUM_RUNS", 10)) if issue_id == 0: - raise ValueError("You need to define ISSUE_ID as environment variable") + raise RuntimeError("You need to define ISSUE_ID as environment variable") gh = login(token=token) From 74939c9321dd95c5a148969065d73c6a66b96414 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Wed, 3 Apr 2024 13:45:46 +0400 Subject: [PATCH 3/3] RuntimeError -> SystemExit if detect it run in __main__ Co-authored-by: Tzu-ping Chung --- scripts/tools/check_if_limited_dependencies.py | 2 +- scripts/tools/list-integrations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tools/check_if_limited_dependencies.py b/scripts/tools/check_if_limited_dependencies.py index 35e64278f1692..855c1b044a277 100755 --- a/scripts/tools/check_if_limited_dependencies.py +++ b/scripts/tools/check_if_limited_dependencies.py @@ -27,7 +27,7 @@ program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise RuntimeError( + raise SystemExit( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" ) diff --git a/scripts/tools/list-integrations.py b/scripts/tools/list-integrations.py index c130f1b985c37..8054d591597bc 100755 --- a/scripts/tools/list-integrations.py +++ b/scripts/tools/list-integrations.py @@ -34,7 +34,7 @@ program = f"./{__file__}" if not __file__.startswith("./") else __file__ if __name__ != "__main__": - raise RuntimeError( + raise SystemExit( "This file is intended to be used as an executable program. You cannot use it as a module." f"To execute this script, run the '{program}' command" )