From f39be410450cd00d30e6872d6c74c3bb078fc08c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 7 Dec 2022 21:02:03 +0100 Subject: [PATCH] Merge redis and rabbitmq integration into celery Previously we had separate integrations per service, but since we are moving the integration tests into separate job, it will be easier if the celery integration is a single one - this way we will have 1-1 relationship between tests to run and enabled integrations. The checks for various integrations were not really working recently and this change is introducing them back. --- .../src/airflow_breeze/global_constants.py | 6 +- images/breeze/output-commands-hash.txt | 10 +- images/breeze/output-commands.svg | 98 +++++++------- images/breeze/output_shell.svg | 110 ++++++++-------- images/breeze/output_start-airflow.svg | 120 +++++++++--------- images/breeze/output_testing.svg | 22 ++-- images/breeze/output_testing_tests.svg | 90 ++++++------- ...ation-redis.yml => integration-celery.yml} | 17 ++- .../docker-compose/integration-rabbitmq.yml | 40 ------ scripts/in_container/check_environment.sh | 75 +++++------ .../cli/commands/test_celery_command.py | 3 +- .../executors/test_celery_executor.py | 6 +- .../providers/redis/hooks/test_redis.py | 2 +- .../redis/operators/test_redis_publish.py | 2 +- .../providers/redis/sensors/test_redis_key.py | 2 +- .../redis/sensors/test_redis_pub_sub.py | 2 +- 16 files changed, 275 insertions(+), 330 deletions(-) rename scripts/ci/docker-compose/{integration-redis.yml => integration-celery.yml} (75%) delete mode 100644 scripts/ci/docker-compose/integration-rabbitmq.yml diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 670d202782734..e080cdf37b9a0 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -49,8 +49,7 @@ "mongo", "openldap", "pinot", - "rabbitmq", - "redis", + "celery", "statsd", "trino", ] @@ -199,8 +198,7 @@ def get_airflow_extras(): "mongo", "openldap", "pinot", - "rabbitmq", - "redis", + "celery", "statsd", "trino", ] diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 27e90061eb515..2908d23bcc0dd 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:11fedd2637e0e3b31936c4f015ab0d19 +main:0e7677cd7035e01b5260b9ac4889fd90 build-docs:80555245ea1142991ce1d63c3bf8ce74 ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd @@ -48,11 +48,11 @@ setup:regenerate-command-images:20016a5ea492f214692c4b57c4fa9c06 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 setup:a3bd246c3a425f3e586d11bbdc8937cb -shell:51149096ed3fb28d2a01e0a5ad0146ef -start-airflow:4bec5d26386dc268ec2de2f637c85272 +shell:6c4dc12f3ff82a1e16aa3818d4181966 +start-airflow:d9ccad39c8551d32aa6d2dd9eee8d573 static-checks:f45ad432bdc47a2256fdb0277b19d816 stop:8969537ccdd799f692ccb8600a7bbed6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a -testing:tests:6f6b7f18cde20255fe988f3a26f340cf -testing:2a93cd42229622d0eaa445c96fd1246d +testing:tests:d2961459b8b39377dd2bc6a06a1bacf0 +testing:9d83bc79d12d10f15402cb848bebcc01 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 763f7d8c8051d..76f63c36952a0 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - + @@ -174,12 +174,9 @@ - - - - Breeze commands + Breeze commands @@ -190,50 +187,49 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index a0814d0de1970..91b60abfbb607 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - + - + @@ -198,12 +198,9 @@ - - - - Command: shell + Command: shell @@ -214,58 +211,57 @@ -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... Enter breeze environment. this is the default command use when no other is selected. -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ -│--max-timeMaximum time that the command should take - if it takes longer, the command    │ -│will fail.                                                                     │ -│(INTEGER RANGE)                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ +│--max-timeMaximum time that the command should take - if it takes longer, the command    │ +│will fail.                                                                     │ +│(INTEGER RANGE)                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_start-airflow.svg b/images/breeze/output_start-airflow.svg index 76128052183c3..075ad7041b6ec 100644 --- a/images/breeze/output_start-airflow.svg +++ b/images/breeze/output_start-airflow.svg @@ -1,4 +1,4 @@ - + - + @@ -216,12 +216,9 @@ - - - - Command: start-airflow + Command: start-airflow @@ -232,64 +229,63 @@ -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... Enter breeze environment and starts all Airflow components in the tmux session. Compile assets if contents of www  directory changed. -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ -│(mutually exclusive with --dev-mode).                                                   │ -│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ -│(mutually exclusive with --skip-asset-compilation).                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ +│(mutually exclusive with --dev-mode).                                                   │ +│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ +│(mutually exclusive with --skip-asset-compilation).                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index bedaaf4ac7086..28eee47f3e614 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 1b23af7398977..de29b87a3ec26 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -35,8 +35,8 @@ .breeze-testing-tests-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-tests-r2 { fill: #c5c8c6 } .breeze-testing-tests-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-tests-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-tests-r5 { fill: #868887 } +.breeze-testing-tests-r4 { fill: #868887 } +.breeze-testing-tests-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-tests-r6 { fill: #8d7b39 } .breeze-testing-tests-r7 { fill: #98a84b;font-weight: bold } @@ -196,52 +196,52 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ -│MySQL | Quarantine)                                                                         │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Integration Other Providers WWW]  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--full-tests-neededWhether full set of tests is run.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ -│(selected | all | skip | remove)                                                               │ -│[default: selected]                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ +│MySQL | Quarantine)                                                                         │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Integration Other Providers WWW]  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--full-tests-neededWhether full set of tests is run.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│(selected | all | skip | remove)                                                               │ +│[default: selected]                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/integration-redis.yml b/scripts/ci/docker-compose/integration-celery.yml similarity index 75% rename from scripts/ci/docker-compose/integration-redis.yml rename to scripts/ci/docker-compose/integration-celery.yml index abefa80ff3387..4c3f4a07229b5 100644 --- a/scripts/ci/docker-compose/integration-redis.yml +++ b/scripts/ci/docker-compose/integration-celery.yml @@ -17,6 +17,17 @@ --- version: "3.7" services: + rabbitmq: + image: rabbitmq:3.7 + volumes: + - /dev/urandom:/dev/random # Required to get non-blocking entropy source + - rabbitmq-db-volume:/var/lib/rabbitmq + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 5s + timeout: 30s + retries: 50 + restart: "on-failure" redis: image: redis:5.0.1 volumes: @@ -30,13 +41,15 @@ services: timeout: 30s retries: 50 restart: "on-failure" - airflow: environment: - - INTEGRATION_REDIS=true + - INTEGRATION_CELERY=true depends_on: redis: condition: service_healthy + rabbitmq: + condition: service_healthy volumes: + rabbitmq-db-volume: redis-db-volume: diff --git a/scripts/ci/docker-compose/integration-rabbitmq.yml b/scripts/ci/docker-compose/integration-rabbitmq.yml deleted file mode 100644 index bf04b4e84fac2..0000000000000 --- a/scripts/ci/docker-compose/integration-rabbitmq.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -version: "3.7" -services: - rabbitmq: - image: rabbitmq:3.7 - volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - - rabbitmq-db-volume:/var/lib/rabbitmq - healthcheck: - test: rabbitmq-diagnostics -q ping - interval: 5s - timeout: 30s - retries: 50 - restart: "on-failure" - - airflow: - environment: - - INTEGRATION_RABBITMQ=true - depends_on: - rabbitmq: - condition: service_healthy - -volumes: - rabbitmq-db-volume: diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index de9f45cdef47c..5f18ad24eff19 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -19,8 +19,6 @@ # shellcheck source=scripts/in_container/_in_container_script_init.sh EXIT_CODE=0 -DISABLED_INTEGRATIONS="" - # We want to avoid misleading messages and perform only forward lookup of the service IP address. # Netcat when run without -n performs both forward and reverse lookup and fails if the reverse # lookup name does not match the original name even if the host is reachable via IP. This happens @@ -76,23 +74,6 @@ function check_service { fi } -function check_integration { - local integration_label=$1 - local integration_name=$2 - local call=$3 - local max_check=${4:=1} - - local env_var_name - env_var_name=INTEGRATION_${integration_name^^} - if [[ ${!env_var_name:=} != "true" || ${!env_var_name} != "True" ]]; then - if [[ ! ${DISABLED_INTEGRATIONS} == *" ${integration_name}"* ]]; then - DISABLED_INTEGRATIONS="${DISABLED_INTEGRATIONS} ${integration_name}" - fi - return - fi - check_service "${integration_label}" "${call}" "${max_check}" -} - function check_db_backend { local max_check=${1:=1} @@ -144,30 +125,44 @@ function startairflow_if_requested() { } echo -echo "${COLOR_BLUE}Checking integrations and backends.${COLOR_RESET}" +echo "${COLOR_BLUE}Checking backend and integrations.${COLOR_RESET}" echo if [[ -n ${BACKEND=} ]]; then check_db_backend 50 fi echo -check_integration "Kerberos" "kerberos" "run_nc kdc-server-example-com 88" 50 -check_integration "MongoDB" "mongo" "run_nc mongo 27017" 50 -check_integration "Redis" "redis" "run_nc redis 6379" 50 -check_integration "Cassandra" "cassandra" "run_nc cassandra 9042" 50 -check_integration "OpenLDAP" "openldap" "run_nc openldap 389" 50 -check_integration "Trino (HTTP)" "trino" "run_nc trino 8080" 50 -check_integration "Trino (HTTPS)" "trino" "run_nc trino 7778" 50 -check_integration "Trino (API)" "trino" \ - "curl --max-time 1 http://trino:8080/v1/info/ | grep '\"starting\":false'" 50 -check_integration "Pinot (HTTP)" "pinot" "run_nc pinot 9000" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:9000/health' -H 'accept: text/plain' | grep OK" -check_integration "Pinot (Controller API)" "pinot" "${CMD}" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:9000/pinot-controller/admin' -H 'accept: text/plain' | grep GOOD" -check_integration "Pinot (Controller API)" "pinot" "${CMD}" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:8000/health' -H 'accept: text/plain' | grep OK" -check_integration "Pinot (Broker API)" "pinot" "${CMD}" 50 -check_integration "RabbitMQ" "rabbitmq" "run_nc rabbitmq 5672" 50 + +if [[ ${INTEGRATION_KERBEROS} == "true" ]]; then + check_service "Kerberos" "run_nc kdc-server-example-com 88" 50 +fi +if [[ ${INTEGRATION_MONGO} == "true" ]]; then + check_service "MongoDB" "run_nc mongo 27017" 50 +fi +if [[ ${INTEGRATION_CELERY} == "true" ]]; then + check_service "Redis" "run_nc redis 6379" 50 + check_service "RabbitMQ" "run_nc rabbitmq 5672" 50 +fi +if [[ ${INTEGRATION_CASSANDRA} == "true" ]]; then + check_service "Cassandra" "run_nc cassandra 9042" 50 +fi +if [[ ${INTEGRATION_OPENLDAP} == "true" ]]; then + check_service "OpenLDAP" "run_nc openldap 389" 50 +fi +if [[ ${INTEGRATION_TRINO} == "true" ]]; then + check_service "Trino (HTTP)" "run_nc trino 8080" 50 + check_service "Trino (HTTPS)" "run_nc trino 7778" 50 + check_service "Trino (API)" "curl --max-time 1 http://trino:8080/v1/info/ | grep '\"starting\":false'" 50 +fi +if [[ ${INTEGRATION_PINOT} == "true" ]]; then + check_service "Pinot (HTTP)" "run_nc pinot 9000" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:9000/health' -H 'accept: text/plain' | grep OK" + check_service "Pinot (Controller API)" "${CMD}" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:9000/pinot-controller/admin' -H 'accept: text/plain' | grep GOOD" + check_service "Pinot (Controller API)" "${CMD}" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:8000/health' -H 'accept: text/plain' | grep OK" + check_service "Pinot (Broker API)" "${CMD}" 50 +fi if [[ ${EXIT_CODE} != 0 ]]; then echo @@ -180,9 +175,3 @@ fi resetdb_if_requested startairflow_if_requested - -if [[ -n ${DISABLED_INTEGRATIONS=} && (${VERBOSE=} == "true" || ${VERBOSE} == "True") ]]; then - echo - echo "${COLOR_BLUE}Those integrations are disabled: ${DISABLED_INTEGRATIONS}${COLOR_RESET}" - echo -fi diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 3cc9da0a13473..306d5fcfd88d9 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -26,8 +26,7 @@ from tests.test_utils.config import conf_vars -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestWorkerServeLogs: @classmethod diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index f7f69fcfce789..2c0e578750f45 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -93,8 +93,7 @@ def _prepare_app(broker_url=None, execute=None): set_event_loop(None) -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestCeleryExecutor: def setup_method(self) -> None: @@ -260,8 +259,7 @@ def __ne__(self, other): return not self.__eq__(other) -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestBulkStateFetcher(unittest.TestCase): @mock.patch( diff --git a/tests/integration/providers/redis/hooks/test_redis.py b/tests/integration/providers/redis/hooks/test_redis.py index eac17ee676edc..d7760933f2143 100644 --- a/tests/integration/providers/redis/hooks/test_redis.py +++ b/tests/integration/providers/redis/hooks/test_redis.py @@ -22,7 +22,7 @@ from airflow.providers.redis.hooks.redis import RedisHook -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisHook: def test_real_ping(self): hook = RedisHook(redis_conn_id="redis_default") diff --git a/tests/integration/providers/redis/operators/test_redis_publish.py b/tests/integration/providers/redis/operators/test_redis_publish.py index c5ea8a65bd626..44678369124d6 100644 --- a/tests/integration/providers/redis/operators/test_redis_publish.py +++ b/tests/integration/providers/redis/operators/test_redis_publish.py @@ -29,7 +29,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisPublishOperator: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE} diff --git a/tests/integration/providers/redis/sensors/test_redis_key.py b/tests/integration/providers/redis/sensors/test_redis_key.py index f54e816c29a50..76034a1be77dc 100644 --- a/tests/integration/providers/redis/sensors/test_redis_key.py +++ b/tests/integration/providers/redis/sensors/test_redis_key.py @@ -27,7 +27,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisSensor: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE} diff --git a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py index e99f2a38d9914..e50967328f3ef 100644 --- a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py +++ b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py @@ -30,7 +30,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisPubSubSensor: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE}