diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 670d202782734..e080cdf37b9a0 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -49,8 +49,7 @@ "mongo", "openldap", "pinot", - "rabbitmq", - "redis", + "celery", "statsd", "trino", ] @@ -199,8 +198,7 @@ def get_airflow_extras(): "mongo", "openldap", "pinot", - "rabbitmq", - "redis", + "celery", "statsd", "trino", ] diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 27e90061eb515..2908d23bcc0dd 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,7 +1,7 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:11fedd2637e0e3b31936c4f015ab0d19 +main:0e7677cd7035e01b5260b9ac4889fd90 build-docs:80555245ea1142991ce1d63c3bf8ce74 ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a ci:fix-ownership:fee2c9ec9ef19686792002ae054fecdd @@ -48,11 +48,11 @@ setup:regenerate-command-images:20016a5ea492f214692c4b57c4fa9c06 setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 setup:a3bd246c3a425f3e586d11bbdc8937cb -shell:51149096ed3fb28d2a01e0a5ad0146ef -start-airflow:4bec5d26386dc268ec2de2f637c85272 +shell:6c4dc12f3ff82a1e16aa3818d4181966 +start-airflow:d9ccad39c8551d32aa6d2dd9eee8d573 static-checks:f45ad432bdc47a2256fdb0277b19d816 stop:8969537ccdd799f692ccb8600a7bbed6 testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c testing:helm-tests:94a442e7f3f63b34c4831a84d165690a -testing:tests:6f6b7f18cde20255fe988f3a26f340cf -testing:2a93cd42229622d0eaa445c96fd1246d +testing:tests:d2961459b8b39377dd2bc6a06a1bacf0 +testing:9d83bc79d12d10f15402cb848bebcc01 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 763f7d8c8051d..76f63c36952a0 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - + @@ -174,12 +174,9 @@ - - - - Breeze commands + Breeze commands @@ -190,50 +187,49 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index a0814d0de1970..91b60abfbb607 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - + - + @@ -198,12 +198,9 @@ - - - - Command: shell + Command: shell @@ -214,58 +211,57 @@ -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... Enter breeze environment. this is the default command use when no other is selected. -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ -│--max-timeMaximum time that the command should take - if it takes longer, the command    │ -│will fail.                                                                     │ -│(INTEGER RANGE)                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ +│--max-timeMaximum time that the command should take - if it takes longer, the command    │ +│will fail.                                                                     │ +│(INTEGER RANGE)                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_start-airflow.svg b/images/breeze/output_start-airflow.svg index 76128052183c3..075ad7041b6ec 100644 --- a/images/breeze/output_start-airflow.svg +++ b/images/breeze/output_start-airflow.svg @@ -1,4 +1,4 @@ - + - + @@ -216,12 +216,9 @@ - - - - Command: start-airflow + Command: start-airflow @@ -232,64 +229,63 @@ -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... Enter breeze environment and starts all Airflow components in the tmux session. Compile assets if contents of www  directory changed. -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ -│(mutually exclusive with --dev-mode).                                                   │ -│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ -│(mutually exclusive with --skip-asset-compilation).                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ +│(mutually exclusive with --dev-mode).                                                   │ +│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ +│(mutually exclusive with --skip-asset-compilation).                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index bedaaf4ac7086..28eee47f3e614 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .breeze-testing-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-r2 { fill: #c5c8c6 } .breeze-testing-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-r5 { fill: #868887 } +.breeze-testing-r4 { fill: #868887 } +.breeze-testing-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 1b23af7398977..de29b87a3ec26 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -35,8 +35,8 @@ .breeze-testing-tests-r1 { fill: #c5c8c6;font-weight: bold } .breeze-testing-tests-r2 { fill: #c5c8c6 } .breeze-testing-tests-r3 { fill: #d0b344;font-weight: bold } -.breeze-testing-tests-r4 { fill: #68a0b3;font-weight: bold } -.breeze-testing-tests-r5 { fill: #868887 } +.breeze-testing-tests-r4 { fill: #868887 } +.breeze-testing-tests-r5 { fill: #68a0b3;font-weight: bold } .breeze-testing-tests-r6 { fill: #8d7b39 } .breeze-testing-tests-r7 { fill: #98a84b;font-weight: bold } @@ -196,52 +196,52 @@ -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... Run the specified unit test targets. -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ -│MySQL | Quarantine)                                                                         │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Integration Other Providers WWW]  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--full-tests-neededWhether full set of tests is run.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ -│(selected | all | skip | remove)                                                               │ -│[default: selected]                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                    │ +│(cassandra | kerberos | mongo | openldap | pinot | celery | statsd | trino | all)│ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ +│MySQL | Quarantine)                                                                         │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Integration Other Providers WWW]  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--full-tests-neededWhether full set of tests is run.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│(selected | all | skip | remove)                                                               │ +│[default: selected]                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/integration-redis.yml b/scripts/ci/docker-compose/integration-celery.yml similarity index 75% rename from scripts/ci/docker-compose/integration-redis.yml rename to scripts/ci/docker-compose/integration-celery.yml index abefa80ff3387..4c3f4a07229b5 100644 --- a/scripts/ci/docker-compose/integration-redis.yml +++ b/scripts/ci/docker-compose/integration-celery.yml @@ -17,6 +17,17 @@ --- version: "3.7" services: + rabbitmq: + image: rabbitmq:3.7 + volumes: + - /dev/urandom:/dev/random # Required to get non-blocking entropy source + - rabbitmq-db-volume:/var/lib/rabbitmq + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 5s + timeout: 30s + retries: 50 + restart: "on-failure" redis: image: redis:5.0.1 volumes: @@ -30,13 +41,15 @@ services: timeout: 30s retries: 50 restart: "on-failure" - airflow: environment: - - INTEGRATION_REDIS=true + - INTEGRATION_CELERY=true depends_on: redis: condition: service_healthy + rabbitmq: + condition: service_healthy volumes: + rabbitmq-db-volume: redis-db-volume: diff --git a/scripts/ci/docker-compose/integration-rabbitmq.yml b/scripts/ci/docker-compose/integration-rabbitmq.yml deleted file mode 100644 index bf04b4e84fac2..0000000000000 --- a/scripts/ci/docker-compose/integration-rabbitmq.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -version: "3.7" -services: - rabbitmq: - image: rabbitmq:3.7 - volumes: - - /dev/urandom:/dev/random # Required to get non-blocking entropy source - - rabbitmq-db-volume:/var/lib/rabbitmq - healthcheck: - test: rabbitmq-diagnostics -q ping - interval: 5s - timeout: 30s - retries: 50 - restart: "on-failure" - - airflow: - environment: - - INTEGRATION_RABBITMQ=true - depends_on: - rabbitmq: - condition: service_healthy - -volumes: - rabbitmq-db-volume: diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index de9f45cdef47c..5f18ad24eff19 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -19,8 +19,6 @@ # shellcheck source=scripts/in_container/_in_container_script_init.sh EXIT_CODE=0 -DISABLED_INTEGRATIONS="" - # We want to avoid misleading messages and perform only forward lookup of the service IP address. # Netcat when run without -n performs both forward and reverse lookup and fails if the reverse # lookup name does not match the original name even if the host is reachable via IP. This happens @@ -76,23 +74,6 @@ function check_service { fi } -function check_integration { - local integration_label=$1 - local integration_name=$2 - local call=$3 - local max_check=${4:=1} - - local env_var_name - env_var_name=INTEGRATION_${integration_name^^} - if [[ ${!env_var_name:=} != "true" || ${!env_var_name} != "True" ]]; then - if [[ ! ${DISABLED_INTEGRATIONS} == *" ${integration_name}"* ]]; then - DISABLED_INTEGRATIONS="${DISABLED_INTEGRATIONS} ${integration_name}" - fi - return - fi - check_service "${integration_label}" "${call}" "${max_check}" -} - function check_db_backend { local max_check=${1:=1} @@ -144,30 +125,44 @@ function startairflow_if_requested() { } echo -echo "${COLOR_BLUE}Checking integrations and backends.${COLOR_RESET}" +echo "${COLOR_BLUE}Checking backend and integrations.${COLOR_RESET}" echo if [[ -n ${BACKEND=} ]]; then check_db_backend 50 fi echo -check_integration "Kerberos" "kerberos" "run_nc kdc-server-example-com 88" 50 -check_integration "MongoDB" "mongo" "run_nc mongo 27017" 50 -check_integration "Redis" "redis" "run_nc redis 6379" 50 -check_integration "Cassandra" "cassandra" "run_nc cassandra 9042" 50 -check_integration "OpenLDAP" "openldap" "run_nc openldap 389" 50 -check_integration "Trino (HTTP)" "trino" "run_nc trino 8080" 50 -check_integration "Trino (HTTPS)" "trino" "run_nc trino 7778" 50 -check_integration "Trino (API)" "trino" \ - "curl --max-time 1 http://trino:8080/v1/info/ | grep '\"starting\":false'" 50 -check_integration "Pinot (HTTP)" "pinot" "run_nc pinot 9000" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:9000/health' -H 'accept: text/plain' | grep OK" -check_integration "Pinot (Controller API)" "pinot" "${CMD}" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:9000/pinot-controller/admin' -H 'accept: text/plain' | grep GOOD" -check_integration "Pinot (Controller API)" "pinot" "${CMD}" 50 -CMD="curl --max-time 1 -X GET 'http://pinot:8000/health' -H 'accept: text/plain' | grep OK" -check_integration "Pinot (Broker API)" "pinot" "${CMD}" 50 -check_integration "RabbitMQ" "rabbitmq" "run_nc rabbitmq 5672" 50 + +if [[ ${INTEGRATION_KERBEROS} == "true" ]]; then + check_service "Kerberos" "run_nc kdc-server-example-com 88" 50 +fi +if [[ ${INTEGRATION_MONGO} == "true" ]]; then + check_service "MongoDB" "run_nc mongo 27017" 50 +fi +if [[ ${INTEGRATION_CELERY} == "true" ]]; then + check_service "Redis" "run_nc redis 6379" 50 + check_service "RabbitMQ" "run_nc rabbitmq 5672" 50 +fi +if [[ ${INTEGRATION_CASSANDRA} == "true" ]]; then + check_service "Cassandra" "run_nc cassandra 9042" 50 +fi +if [[ ${INTEGRATION_OPENLDAP} == "true" ]]; then + check_service "OpenLDAP" "run_nc openldap 389" 50 +fi +if [[ ${INTEGRATION_TRINO} == "true" ]]; then + check_service "Trino (HTTP)" "run_nc trino 8080" 50 + check_service "Trino (HTTPS)" "run_nc trino 7778" 50 + check_service "Trino (API)" "curl --max-time 1 http://trino:8080/v1/info/ | grep '\"starting\":false'" 50 +fi +if [[ ${INTEGRATION_PINOT} == "true" ]]; then + check_service "Pinot (HTTP)" "run_nc pinot 9000" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:9000/health' -H 'accept: text/plain' | grep OK" + check_service "Pinot (Controller API)" "${CMD}" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:9000/pinot-controller/admin' -H 'accept: text/plain' | grep GOOD" + check_service "Pinot (Controller API)" "${CMD}" 50 + CMD="curl --max-time 1 -X GET 'http://pinot:8000/health' -H 'accept: text/plain' | grep OK" + check_service "Pinot (Broker API)" "${CMD}" 50 +fi if [[ ${EXIT_CODE} != 0 ]]; then echo @@ -180,9 +175,3 @@ fi resetdb_if_requested startairflow_if_requested - -if [[ -n ${DISABLED_INTEGRATIONS=} && (${VERBOSE=} == "true" || ${VERBOSE} == "True") ]]; then - echo - echo "${COLOR_BLUE}Those integrations are disabled: ${DISABLED_INTEGRATIONS}${COLOR_RESET}" - echo -fi diff --git a/tests/integration/cli/commands/test_celery_command.py b/tests/integration/cli/commands/test_celery_command.py index 3cc9da0a13473..306d5fcfd88d9 100644 --- a/tests/integration/cli/commands/test_celery_command.py +++ b/tests/integration/cli/commands/test_celery_command.py @@ -26,8 +26,7 @@ from tests.test_utils.config import conf_vars -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestWorkerServeLogs: @classmethod diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index f7f69fcfce789..2c0e578750f45 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -93,8 +93,7 @@ def _prepare_app(broker_url=None, execute=None): set_event_loop(None) -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestCeleryExecutor: def setup_method(self) -> None: @@ -260,8 +259,7 @@ def __ne__(self, other): return not self.__eq__(other) -@pytest.mark.integration("redis") -@pytest.mark.integration("rabbitmq") +@pytest.mark.integration("celery") @pytest.mark.backend("mysql", "postgres") class TestBulkStateFetcher(unittest.TestCase): @mock.patch( diff --git a/tests/integration/providers/redis/hooks/test_redis.py b/tests/integration/providers/redis/hooks/test_redis.py index eac17ee676edc..d7760933f2143 100644 --- a/tests/integration/providers/redis/hooks/test_redis.py +++ b/tests/integration/providers/redis/hooks/test_redis.py @@ -22,7 +22,7 @@ from airflow.providers.redis.hooks.redis import RedisHook -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisHook: def test_real_ping(self): hook = RedisHook(redis_conn_id="redis_default") diff --git a/tests/integration/providers/redis/operators/test_redis_publish.py b/tests/integration/providers/redis/operators/test_redis_publish.py index c5ea8a65bd626..44678369124d6 100644 --- a/tests/integration/providers/redis/operators/test_redis_publish.py +++ b/tests/integration/providers/redis/operators/test_redis_publish.py @@ -29,7 +29,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisPublishOperator: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE} diff --git a/tests/integration/providers/redis/sensors/test_redis_key.py b/tests/integration/providers/redis/sensors/test_redis_key.py index f54e816c29a50..76034a1be77dc 100644 --- a/tests/integration/providers/redis/sensors/test_redis_key.py +++ b/tests/integration/providers/redis/sensors/test_redis_key.py @@ -27,7 +27,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisSensor: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE} diff --git a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py index e99f2a38d9914..e50967328f3ef 100644 --- a/tests/integration/providers/redis/sensors/test_redis_pub_sub.py +++ b/tests/integration/providers/redis/sensors/test_redis_pub_sub.py @@ -30,7 +30,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) -@pytest.mark.integration("redis") +@pytest.mark.integration("celery") class TestRedisPubSubSensor: def setup_method(self): args = {"owner": "airflow", "start_date": DEFAULT_DATE}