From 5ad1921ad12ccbafcd356f23b65cc1b9d79ff635 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 10 Sep 2021 13:57:25 +0200 Subject: [PATCH] Add missing __init__.py files for some test packages Lacl of the __init__.py caused failures in some specific cases - especially when new providers have been added. This PR adds missing ``__init__.py`` files and modifies pre-commit check which was only implemented for main files and example_dags. It checks if those files are present and adds them if missing. --- .pre-commit-config.yaml | 8 ++++---- BREEZE.rst | 10 +++++----- STATIC_CODE_CHECKS.rst | 2 +- .../google/config_templates/__init__.py | 16 +++++++++++++++ breeze-complete | 2 +- ...ck_providers_subpackages_all_have_init.py} | 20 +++++++++---------- .../amazon/aws/config_templates/__init__.py | 16 +++++++++++++++ .../amazon/aws/infrastructure/__init__.py | 16 +++++++++++++++ .../example_s3_to_redshift/__init__.py | 16 +++++++++++++++ .../providers/amazon/aws/secrets/__init__.py | 16 +++++++++++++++ .../google/ads/operators/__init__.py | 16 +++++++++++++++ tests/providers/jenkins/hooks/__init__.py | 16 +++++++++++++++ .../microsoft/azure/secrets/__init__.py | 16 +++++++++++++++ .../microsoft/mssql/operators/__init__.py | 16 +++++++++++++++ tests/providers/odbc/__init__.py | 16 +++++++++++++++ tests/providers/odbc/hooks/__init__.py | 16 +++++++++++++++ tests/providers/plexus/hooks/__init__.py | 16 +++++++++++++++ tests/providers/tableau/__init__.py | 16 +++++++++++++++ 18 files changed, 229 insertions(+), 21 deletions(-) create mode 100644 airflow/providers/google/config_templates/__init__.py rename scripts/ci/pre_commit/{pre_commit_check_providers_example_dag_init.py => pre_commit_check_providers_subpackages_all_have_init.py} (70%) create mode 100644 tests/providers/amazon/aws/config_templates/__init__.py create mode 100644 tests/providers/amazon/aws/infrastructure/__init__.py create mode 100644 tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py create mode 100644 tests/providers/amazon/aws/secrets/__init__.py create mode 100644 tests/providers/google/ads/operators/__init__.py create mode 100644 tests/providers/jenkins/hooks/__init__.py create mode 100644 tests/providers/microsoft/azure/secrets/__init__.py create mode 100644 tests/providers/microsoft/mssql/operators/__init__.py create mode 100644 tests/providers/odbc/__init__.py create mode 100644 tests/providers/odbc/hooks/__init__.py create mode 100644 tests/providers/plexus/hooks/__init__.py create mode 100644 tests/providers/tableau/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5f0df7318198..212be6f4d1dc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -508,16 +508,16 @@ repos: # Keep dependency versions in sync w/ airflow/www/package.json additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0'] - id: providers-init-file - name: Provider init file + name: Provider init file is missing pass_filenames: false always_run: true entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.sh language: system - - id: providers-example-dags-init-file - name: Provider example_dags init file + - id: providers-subpackages-init-file + name: Provider subpackage init files are there pass_filenames: false always_run: true - entry: ./scripts/ci/pre_commit/pre_commit_check_providers_example_dag_init.py + entry: ./scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py language: python require_serial: true - id: provider-yamls diff --git a/BREEZE.rst b/BREEZE.rst index 738c1e1e31223..e0ce35ed66589 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2182,11 +2182,11 @@ This is the current syntax for `./breeze <./breeze>`_: insert-license isort json-schema language-matters lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending mypy mypy-helm no-providers-in-core-examples no-relative-imports pre-commit-descriptions pre-commit-hook-names pretty-format-json - provide-create-sessions providers-changelogs providers-example-dags-init-file - providers-init-file provider-yamls pydevd pydocstyle python-no-log-warn pyupgrade - restrict-start_date rst-backticks setup-order setup-extra-packages shellcheck - sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace ui-lint - update-breeze-file update-extras update-local-yml-file update-setup-cfg-file + provide-create-sessions providers-changelogs providers-init-file + providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn + pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages + shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace + ui-lint update-breeze-file update-extras update-local-yml-file update-setup-cfg-file verify-db-migrations-documented version-sync www-lint yamllint yesqa You can pass extra arguments including options to the pre-commit framework as diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index ea24e11f96c8b..8d533b8f8dda4 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -224,7 +224,7 @@ require Breeze Docker images to be installed locally. ------------------------------------ ---------------------------------------------------------------- ------------ ``providers-changelogs`` Updates documentation for providers changelogs ------------------------------------ ---------------------------------------------------------------- ------------ -``providers-example-dags-init-file`` Check that providers' example_dags __init__.py file is present +``providers-subpackages-init-file`` Check that providers' subpackages __init__.py files are there ------------------------------------ ---------------------------------------------------------------- ------------ ``providers-init-file`` Check that provider's __init__.py file is removed ------------------------------------ ---------------------------------------------------------------- ------------ diff --git a/airflow/providers/google/config_templates/__init__.py b/airflow/providers/google/config_templates/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/google/config_templates/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/breeze-complete b/breeze-complete index d116eafa5f353..aac8210b36e44 100644 --- a/breeze-complete +++ b/breeze-complete @@ -123,8 +123,8 @@ pre-commit-hook-names pretty-format-json provide-create-sessions providers-changelogs -providers-example-dags-init-file providers-init-file +providers-subpackages-init-file provider-yamls pydevd pydocstyle diff --git a/scripts/ci/pre_commit/pre_commit_check_providers_example_dag_init.py b/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py similarity index 70% rename from scripts/ci/pre_commit/pre_commit_check_providers_example_dag_init.py rename to scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py index bf5a7cc865103..0d06dc8fa6946 100755 --- a/scripts/ci/pre_commit/pre_commit_check_providers_example_dag_init.py +++ b/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py @@ -24,11 +24,11 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) -def check_example_dags_dir_init_file(example_dags_dirs: List[str]) -> None: +def check_dir_init_file(provider_files: List[str]) -> None: missing_init_dirs = [] - for example_dags_dir in example_dags_dirs: - if not os.path.exists(example_dags_dir + "__init__.py"): - missing_init_dirs.append(example_dags_dir) + for dags_file in provider_files: + if os.path.isdir(dags_file) and not os.path.exists(os.path.join(dags_file, "__init__.py")): + missing_init_dirs.append(dags_file) if missing_init_dirs: with open(os.path.join(ROOT_DIR, "license-templates/LICENSE.txt")) as license: @@ -36,17 +36,17 @@ def check_example_dags_dir_init_file(example_dags_dirs: List[str]) -> None: prefixed_licensed_txt = [f"# {line}" if line != "\n" else "#\n" for line in license_txt] for missing_init_dir in missing_init_dirs: - with open(missing_init_dir + "__init__.py", "w") as init_file: + with open(os.path.join(missing_init_dir, "__init__.py"), "w") as init_file: init_file.write("".join(prefixed_licensed_txt)) - print("No __init__.py file was found in the following provider example_dags directories:") + print("No __init__.py file was found in the following provider directories:") print("\n".join(missing_init_dirs)) print("\nThe missing __init__.py files have been created. Please add these new files to a commit.") sys.exit(1) if __name__ == "__main__": - all_provider_example_dags_dirs = sorted( - glob(f"{ROOT_DIR}/airflow/providers/**/example_dags/", recursive=True) - ) - check_example_dags_dir_init_file(all_provider_example_dags_dirs) + all_provider_subpackage_dirs = sorted(glob(f"{ROOT_DIR}/airflow/providers/**/*", recursive=True)) + check_dir_init_file(all_provider_subpackage_dirs) + all_test_provider_subpackage_dirs = sorted(glob(f"{ROOT_DIR}/tests/providers/**/*", recursive=True)) + check_dir_init_file(all_test_provider_subpackage_dirs) diff --git a/tests/providers/amazon/aws/config_templates/__init__.py b/tests/providers/amazon/aws/config_templates/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/amazon/aws/config_templates/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/amazon/aws/infrastructure/__init__.py b/tests/providers/amazon/aws/infrastructure/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/amazon/aws/infrastructure/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py b/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/amazon/aws/infrastructure/example_s3_to_redshift/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/amazon/aws/secrets/__init__.py b/tests/providers/amazon/aws/secrets/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/amazon/aws/secrets/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/google/ads/operators/__init__.py b/tests/providers/google/ads/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/google/ads/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/jenkins/hooks/__init__.py b/tests/providers/jenkins/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/jenkins/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/microsoft/azure/secrets/__init__.py b/tests/providers/microsoft/azure/secrets/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/microsoft/azure/secrets/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/microsoft/mssql/operators/__init__.py b/tests/providers/microsoft/mssql/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/microsoft/mssql/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/odbc/__init__.py b/tests/providers/odbc/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/odbc/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/odbc/hooks/__init__.py b/tests/providers/odbc/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/odbc/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/plexus/hooks/__init__.py b/tests/providers/plexus/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/plexus/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/tableau/__init__.py b/tests/providers/tableau/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/tableau/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License.