From 3ef995c5a28f457a1543c780f42f16f1b04591c4 Mon Sep 17 00:00:00 2001 From: Daniel Standish <15932138+dstandish@users.noreply.github.com> Date: Sat, 26 Feb 2022 11:19:48 -0800 Subject: [PATCH] Autogenerate migration reference doc * document airflow version in each alembic migration module and use this to autogen the doc * update each migration module to have the same description used in migration ref (so it can be used in autogen) See PR #21596 for reference. --- .pre-commit-config.yaml | 14 +- BREEZE.rst | 21 +- STATIC_CODE_CHECKS.rst | 2 + ...1210f153_increase_queue_name_size_limit.py | 3 +- ...02_increase_length_of_fab_ab_view_menu_.py | 7 +- .../versions/03bc53e68815_add_sm_dag_index.py | 3 +- .../versions/05f30312d566_merge_heads.py | 3 +- .../versions/08364691d074_merge_heads.py | 3 +- ...add_max_active_runs_column_to_dagmodel_.py | 7 +- .../0a2a5b66e19d_add_task_reschedule_table.py | 3 +- .../0e2a74e0fc9f_add_time_zone_awareness.py | 1 + ...add_dag_id_state_index_on_dag_run_table.py | 3 +- .../13eb55f81627_for_compatibility.py | 3 +- ...terval_start_end_to_dagmodel_and_dagrun.py | 1 + .../1507a7289a2f_create_is_encrypted.py | 3 +- ...e3_add_is_encrypted_column_to_variable_.py | 3 +- .../versions/1b38cef5b76e_add_dagrun.py | 3 +- .../211e584da130_add_ti_state_index.py | 3 +- ...24_add_executor_config_to_task_instance.py | 3 +- ...2c6edca13270_resource_based_permissions.py | 1 + ...497a22_rename_last_scheduler_run_column.py | 7 +- .../versions/2e541a1dcfed_task_duration.py | 3 +- .../2e82aab8ef20_rename_user_table.py | 3 +- ...rename_concurrency_column_in_dag_table_.py | 7 +- ...0f54d61_more_logging_into_task_isntance.py | 3 +- ...4_add_kubernetes_resource_checkpointing.py | 3 +- ...364159666cbd_add_job_id_to_dagrun_table.py | 5 +- .../3c20cacc0044_add_dagrun_run_type.py | 7 +- .../versions/40e67319e3a9_dagrun_config.py | 3 +- .../41f5f12752f8_add_superuser_field.py | 3 +- .../versions/4446e08588_dagrun_start_end.py | 3 +- ...ncrease_size_of_connection_extra_field_.py | 3 +- ...93b9_add_k8s_yaml_to_rendered_templates.py | 1 + ..._add_fractional_seconds_to_mysql_tables.py | 3 +- .../502898887f84_adding_extra_to_log.py | 3 +- ..._mssql_exec_date_rendered_task_instance.py | 1 + .../versions/52d714495f0_job_id_indices.py | 3 +- ...d308c5f_add_trigger_table_and_task_info.py | 7 +- ...61833c1c74b_add_password_column_to_user.py | 3 +- ...df053233_adding_index_for_dag_id_in_job.py | 7 +- ...3_increase_length_of_email_and_username.py | 3 +- ...d17757c7a_add_pid_field_to_taskinstance.py | 3 +- ...01f_add_description_field_to_connection.py | 7 +- ...fix_description_field_in_connection_to_.py | 7 +- ...de9cddf6c9_add_task_fails_journal_table.py | 3 +- ...4a4_make_taskinstance_pool_not_nullable.py | 3 +- ...hange_datetime_to_datetime2_6_on_mssql_.py | 1 + ...b18f_added_timetable_description_column.py | 7 +- .../7939bcff74ba_add_dagtags_table.py | 7 +- ...2661a43ba3_taskinstance_keyed_to_dagrun.py | 7 +- ...f_remove_can_read_permission_on_config_.py | 3 +- ...3f031fd9f1c_improve_mssql_compatibility.py | 3 +- .../849da589634d_prefix_dag_permissions.py | 1 + .../8504051e801b_xcom_dag_task_indices.py | 3 +- ...add_rendered_task_instance_fields_table.py | 5 +- .../856955da8476_fix_sqlite_foreign_key.py | 3 +- ...922c8a04_change_default_pool_slots_to_1.py | 7 +- ...5c0_add_kubernetes_scheduler_uniqueness.py | 3 +- ...3f6d53_add_unique_constraint_to_conn_id.py | 7 +- ...66b9c467a_set_conn_type_as_non_nullable.py | 7 +- ...increase_pool_name_size_in_taskinstance.py | 7 +- .../versions/92c57b58940d_add_fab_tables.py | 1 + ...c8_task_reschedule_fk_on_cascade_delete.py | 3 +- .../947454bf1dff_add_ti_job_id_index.py | 3 +- .../952da73b5eff_add_dag_code_table.py | 3 +- .../versions/9635ae0956e7_index_taskfail.py | 3 +- ...b8_add_queued_at_column_to_dagrun_table.py | 7 +- ..._add_scheduling_decision_to_dagrun_and_.py | 7 +- ...resource_based_permissions_for_default_.py | 3 +- ...2_add_data_compressed_to_serialized_dag.py | 1 + ...b_add_pool_slots_field_to_task_instance.py | 3 +- .../a56c9515abdc_remove_dag_stat_table.py | 3 +- ...dd_precision_to_execution_date_in_mysql.py | 7 +- ...b247b1e3d1ed_add_queued_by_job_id_to_ti.py | 1 + ...5a55525161_increase_length_of_pool_name.py | 1 + .../b3b105409875_add_root_dag_id_to_dag.py | 7 +- ...6_add_a_column_to_track_the_encryption_.py | 1 + ...dd_notification_sent_column_to_sla_miss.py | 3 +- ...bbf4a7ad0465_remove_id_column_from_xcom.py | 1 + ...6_make_xcom_value_column_a_large_binary.py | 1 + ...dd_has_import_errors_column_to_dagmodel.py | 1 + ...4f3d11e8b_drop_kuberesourceversion_and_.py | 7 +- .../bf00311e1990_add_index_to_taskinstance.py | 3 +- ...5b5ae4a_switch_xcom_table_to_use_run_id.py | 1 + .../c381b21cb7e4_add_session_table_to_db.py | 7 +- .../c8ffec048a3b_add_fields_to_dag.py | 3 +- ...7_add_max_tries_column_to_task_instance.py | 3 +- ...8_add_index_on_state_dag_id_for_queued_.py | 7 +- .../cf5dc11e79ad_drop_user_and_chart.py | 3 +- ...ae31099d61_increase_text_size_for_mysql.py | 1 + .../d38e04c12aa2_add_serialized_dag_table.py | 3 +- ..._add_dag_hash_column_to_serialized_dag_.py | 7 +- .../versions/dd25f486b8ea_add_idx_log_dag.py | 3 +- ...4ecb8fbee3_add_schedule_interval_to_dag.py | 1 + ...55d70_add_description_field_to_variable.py | 7 +- ...1ece99cc_add_external_executor_id_to_ti.py | 1 + ...e357a868_update_schema_for_smart_sensor.py | 3 +- .../versions/e3a246e0dc1_current_schema.py | 1 + ..._add_taskmap_and_map_id_on_taskinstance.py | 9 +- ...1f0_make_xcom_pkey_columns_non_nullable.py | 7 +- ..._change_field_in_dagcode_to_mediumtext_.py | 3 +- ...433877c24_fix_mysql_not_null_constraint.py | 3 +- .../f2ca10b85618_add_dag_stats_table.py | 3 +- ...89_add_task_log_filename_template_model.py | 3 +- ...increase_length_for_connection_password.py | 7 +- breeze-complete | 1 + .../src/airflow_breeze/pre_commit_ids.py | 2 +- docs/apache-airflow/migrations-ref.rst | 450 +++++++++--------- .../pre_commit_migration_documented.py | 67 --- .../pre_commit_migration_reference.py | 134 ++++++ .../pre_commit_migration_reference.sh | 42 ++ .../in_container/run_migration_reference.sh | 20 + 112 files changed, 697 insertions(+), 440 deletions(-) delete mode 100755 scripts/ci/pre_commit/pre_commit_migration_documented.py create mode 100755 scripts/ci/pre_commit/pre_commit_migration_reference.py create mode 100755 scripts/ci/pre_commit/pre_commit_migration_reference.sh create mode 100755 scripts/in_container/run_migration_reference.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7ef96a857c73..6c5f76e6ea3cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -488,13 +488,6 @@ repos: pass_filenames: false require_serial: true additional_dependencies: ['pyyaml'] - - id: verify-db-migrations-documented - name: Verify that DB Migrations have been documented - language: python - entry: ./scripts/ci/pre_commit/pre_commit_migration_documented.py - files: ^airflow/migrations/versions/.*\.(py)$ - pass_filenames: false - require_serial: true - id: boring-cyborg name: Checks for Boring Cyborg configuration consistency language: python @@ -738,4 +731,11 @@ repos: files: \.py$ pass_filenames: true exclude: ^airflow/_vendor/ + - id: migration-reference + name: Update migration ref doc + language: system + entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.sh + files: \.py$ + pass_filenames: false + files: ^airflow/migrations/versions/.*\.(py)$|^docs/apache-airflow/migrations-ref.rst$ ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE diff --git a/BREEZE.rst b/BREEZE.rst index 3fa459f218130..e9bb46f9a1343 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2240,16 +2240,17 @@ This is the current syntax for `./breeze <./breeze>`_: debug-statements detect-private-key docstring-params doctoc dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin insert-license isort json-schema - language-matters lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending - mypy mypy-helm no-providers-in-core-examples no-relative-imports - persist-credentials-disabled pre-commit-descriptions pre-commit-hook-names - pretty-format-json provide-create-sessions providers-changelogs providers-init-file - providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn - pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages - shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace - ui-lint update-breeze-file update-extras update-local-yml-file update-setup-cfg-file - update-supported-versions update-versions vendor-k8s-json-schema - verify-db-migrations-documented version-sync www-lint yamllint yesqa + language-matters lint-dockerfile lint-openapi markdownlint mermaid + migration-reference mixed-line-ending mypy mypy-helm no-providers-in-core-examples + no-relative-imports persist-credentials-disabled pre-commit-descriptions + pre-commit-hook-names pretty-format-json provide-create-sessions + providers-changelogs providers-init-file providers-subpackages-init-file + provider-yamls pydevd pydocstyle python-no-log-warn pyupgrade restrict-start_date + rst-backticks setup-order setup-extra-packages shellcheck sort-in-the-wild + sort-spelling-wordlist stylelint trailing-whitespace ui-lint update-breeze-file + update-extras update-local-yml-file update-setup-cfg-file update-supported-versions + update-versions vendor-k8s-json-schema verify-db-migrations-documented version-sync + www-lint yamllint yesqa You can pass extra arguments including options to the pre-commit framework as passed after --. For example: diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index cd22d2d6b6a05..087b0716a5f6a 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -216,6 +216,8 @@ require Breeze Docker images to be installed locally. ------------------------------------ ---------------------------------------------------------------- ------------ ``markdownlint`` Lints Markdown files ------------------------------------ ---------------------------------------------------------------- ------------ +``migration-reference`` Update migration reference doc and revision metadata +------------------------------------ ---------------------------------------------------------------- ------------ ``mixed-line-ending`` Detects if mixed line ending is used (\r vs. \r\n) ------------------------------------ ---------------------------------------------------------------- ------------ ``mypy`` Runs mypy * diff --git a/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py b/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py index b7253686c3acc..50dbc43caea89 100644 --- a/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py +++ b/airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""increase queue name size limit +"""Increase queue name size limit Revision ID: 004c1210f153 Revises: 939bb1e647c8 @@ -32,6 +32,7 @@ down_revision = '939bb1e647c8' branch_labels = None depends_on = None +airflow_version = '1.10.4' def upgrade(): diff --git a/airflow/migrations/versions/03afc6b6f902_increase_length_of_fab_ab_view_menu_.py b/airflow/migrations/versions/03afc6b6f902_increase_length_of_fab_ab_view_menu_.py index 3b78f02c7c9f7..b03d4019879cf 100644 --- a/airflow/migrations/versions/03afc6b6f902_increase_length_of_fab_ab_view_menu_.py +++ b/airflow/migrations/versions/03afc6b6f902_increase_length_of_fab_ab_view_menu_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Increase length of FAB ab_view_menu.name column +"""Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column Revision ID: 03afc6b6f902 Revises: 92c57b58940d @@ -35,10 +35,11 @@ down_revision = '92c57b58940d' branch_labels = None depends_on = None +airflow_version = '1.10.13' def upgrade(): - """Apply Increase length of FAB ab_view_menu.name column""" + """Apply Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column""" conn = op.get_bind() inspector = Inspector.from_engine(conn) tables = inspector.get_table_names() @@ -69,7 +70,7 @@ def upgrade(): def downgrade(): - """Unapply Increase length of FAB ab_view_menu.name column""" + """Unapply Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column""" conn = op.get_bind() inspector = Inspector.from_engine(conn) tables = inspector.get_table_names() diff --git a/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py b/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py index fc8468155c235..a4bf9a03e1985 100644 --- a/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py +++ b/airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""merge_heads_2 +"""Merge migrations Heads Revision ID: 03bc53e68815 Revises: 0a2a5b66e19d, bf00311e1990 @@ -30,6 +30,7 @@ down_revision = ('0a2a5b66e19d', 'bf00311e1990') branch_labels = None depends_on = None +airflow_version = '1.10.2' def upgrade(): diff --git a/airflow/migrations/versions/05f30312d566_merge_heads.py b/airflow/migrations/versions/05f30312d566_merge_heads.py index 36940c67d8605..33691b7b04ec3 100644 --- a/airflow/migrations/versions/05f30312d566_merge_heads.py +++ b/airflow/migrations/versions/05f30312d566_merge_heads.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""merge heads +"""Merge migrations Heads Revision ID: 05f30312d566 Revises: 86770d1215c0, 0e2a74e0fc9f @@ -28,6 +28,7 @@ down_revision = ('86770d1215c0', '0e2a74e0fc9f') branch_labels = None depends_on = None +airflow_version = '1.10.0' def upgrade(): diff --git a/airflow/migrations/versions/08364691d074_merge_heads.py b/airflow/migrations/versions/08364691d074_merge_heads.py index bba9e73cdf8c3..1f589e993fe7a 100644 --- a/airflow/migrations/versions/08364691d074_merge_heads.py +++ b/airflow/migrations/versions/08364691d074_merge_heads.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Merge the four heads back together +"""Straighten out the migrations Revision ID: 08364691d074 Revises: a56c9515abdc, 004c1210f153, 74effc47d867, b3b105409875 @@ -29,6 +29,7 @@ down_revision = ('a56c9515abdc', '004c1210f153', '74effc47d867', 'b3b105409875') branch_labels = None depends_on = None +airflow_version = '1.10.7' def upgrade(): diff --git a/airflow/migrations/versions/092435bf5d12_add_max_active_runs_column_to_dagmodel_.py b/airflow/migrations/versions/092435bf5d12_add_max_active_runs_column_to_dagmodel_.py index a9f612d591ef8..1b759ac27da2d 100644 --- a/airflow/migrations/versions/092435bf5d12_add_max_active_runs_column_to_dagmodel_.py +++ b/airflow/migrations/versions/092435bf5d12_add_max_active_runs_column_to_dagmodel_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add max_active_runs column to dagmodel table +"""Add ``max_active_runs`` column to ``dag_model`` table Revision ID: 092435bf5d12 Revises: 97cdd93827b8 @@ -33,10 +33,11 @@ down_revision = '97cdd93827b8' branch_labels = None depends_on = None +airflow_version = '2.1.4' def upgrade(): - """Apply Add max_active_runs column to dagmodel table""" + """Apply Add ``max_active_runs`` column to ``dag_model`` table""" op.add_column('dag', sa.Column('max_active_runs', sa.Integer(), nullable=True)) with op.batch_alter_table('dag_run', schema=None) as batch_op: # Add index to dag_run.dag_id and also add index to dag_run.state where state==running @@ -51,7 +52,7 @@ def upgrade(): def downgrade(): - """Unapply Add max_active_runs column to dagmodel table""" + """Unapply Add ``max_active_runs`` column to ``dag_model`` table""" op.drop_column('dag', 'max_active_runs') with op.batch_alter_table('dag_run', schema=None) as batch_op: # Drop index to dag_run.dag_id and also drop index to dag_run.state where state==running diff --git a/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py index b7e62413ec1a3..a26204fff122e 100644 --- a/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py +++ b/airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""add task_reschedule table +"""Add ``task_reschedule`` table Revision ID: 0a2a5b66e19d Revises: 9635ae0956e7 @@ -32,6 +32,7 @@ down_revision = '9635ae0956e7' branch_labels = None depends_on = None +airflow_version = '1.10.2' TABLE_NAME = 'task_reschedule' INDEX_NAME = 'idx_' + TABLE_NAME + '_dag_task_date' diff --git a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py index e4d35c047ca7e..8945ff06c99ca 100644 --- a/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py +++ b/airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py @@ -32,6 +32,7 @@ down_revision = "d2ae31099d61" branch_labels = None depends_on = None +airflow_version = '1.10.0' def upgrade(): diff --git a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py index 288a0b60aa821..a7acdb2e7b187 100644 --- a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py +++ b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add dag_id/state index on dag_run table +"""Add ``dag_id``/``state`` index on ``dag_run`` table Revision ID: 127d2bf2dfa7 Revises: 5e7d17757c7a @@ -30,6 +30,7 @@ down_revision = '5e7d17757c7a' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/13eb55f81627_for_compatibility.py b/airflow/migrations/versions/13eb55f81627_for_compatibility.py index a15b20c6c6ad8..cd45e1fa745da 100644 --- a/airflow/migrations/versions/13eb55f81627_for_compatibility.py +++ b/airflow/migrations/versions/13eb55f81627_for_compatibility.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""maintain history for compatibility with earlier migrations +"""Maintain history for compatibility with earlier migrations Revision ID: 13eb55f81627 Revises: 1507a7289a2f @@ -29,6 +29,7 @@ down_revision = '1507a7289a2f' branch_labels = None depends_on = None +airflow_version = '1.5.0' def upgrade(): diff --git a/airflow/migrations/versions/142555e44c17_add_data_interval_start_end_to_dagmodel_and_dagrun.py b/airflow/migrations/versions/142555e44c17_add_data_interval_start_end_to_dagmodel_and_dagrun.py index 6be37326cd29c..1aaa2230ee571 100644 --- a/airflow/migrations/versions/142555e44c17_add_data_interval_start_end_to_dagmodel_and_dagrun.py +++ b/airflow/migrations/versions/142555e44c17_add_data_interval_start_end_to_dagmodel_and_dagrun.py @@ -34,6 +34,7 @@ down_revision = "54bebd308c5f" branch_labels = None depends_on = None +airflow_version = '2.2.0' def upgrade(): diff --git a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py index 1a539c603e5a1..ffade38b51e76 100644 --- a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py +++ b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""create is_encrypted +"""Add ``is_encrypted`` column in ``connection`` table Revision ID: 1507a7289a2f Revises: e3a246e0dc1 @@ -32,6 +32,7 @@ down_revision = 'e3a246e0dc1' branch_labels = None depends_on = None +airflow_version = '1.5.0' connectionhelper = sa.Table( 'connection', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True), sa.Column('is_encrypted') diff --git a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py index 3e7c694eb018a..e72260ec2089d 100644 --- a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py +++ b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add is_encrypted column to variable table +"""Add ``is_encrypted`` column to variable table Revision ID: 1968acfc09e3 Revises: bba5a7cfc896 @@ -31,6 +31,7 @@ down_revision = 'bba5a7cfc896' branch_labels = None depends_on = None +airflow_version = '1.7.0' def upgrade(): diff --git a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py index 441a4d8e7159b..66a65fd3c3d3c 100644 --- a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py +++ b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add dagrun +"""Add ``dag_run`` table Revision ID: 1b38cef5b76e Revises: 52d714495f0 @@ -34,6 +34,7 @@ down_revision = '502898887f84' branch_labels = None depends_on = None +airflow_version = '1.6.0' def upgrade(): diff --git a/airflow/migrations/versions/211e584da130_add_ti_state_index.py b/airflow/migrations/versions/211e584da130_add_ti_state_index.py index 86c2e7de3ef75..eae9bfd06450d 100644 --- a/airflow/migrations/versions/211e584da130_add_ti_state_index.py +++ b/airflow/migrations/versions/211e584da130_add_ti_state_index.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add TI state index +"""Add TI state index Revision ID: 211e584da130 Revises: 2e82aab8ef20 @@ -30,6 +30,7 @@ down_revision = '2e82aab8ef20' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py index c02a1f5242eac..67bda056ecb51 100644 --- a/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py +++ b/airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py @@ -16,7 +16,7 @@ # under the License. -"""kubernetes_resource_checkpointing +"""Add ``executor_config`` column to ``task_instance`` table Revision ID: 33ae817a1ff4 Revises: 947454bf1dff @@ -33,6 +33,7 @@ down_revision = '33ae817a1ff4' branch_labels = None depends_on = None +airflow_version = '1.10.0' TASK_INSTANCE_TABLE = "task_instance" NEW_COLUMN = "executor_config" diff --git a/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py b/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py index 774109cd01424..62121a35455a0 100644 --- a/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py +++ b/airflow/migrations/versions/2c6edca13270_resource_based_permissions.py @@ -33,6 +33,7 @@ down_revision = '849da589634d' branch_labels = None depends_on = None +airflow_version = '2.0.0' mapping = { diff --git a/airflow/migrations/versions/2e42bb497a22_rename_last_scheduler_run_column.py b/airflow/migrations/versions/2e42bb497a22_rename_last_scheduler_run_column.py index 97d8ff6211afc..487abdb11994b 100644 --- a/airflow/migrations/versions/2e42bb497a22_rename_last_scheduler_run_column.py +++ b/airflow/migrations/versions/2e42bb497a22_rename_last_scheduler_run_column.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""rename last_scheduler_run column +"""Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time`` Revision ID: 2e42bb497a22 Revises: 8646922c8a04 @@ -33,10 +33,11 @@ down_revision = '8646922c8a04' branch_labels = None depends_on = None +airflow_version = '2.0.2' def upgrade(): - """Apply rename last_scheduler_run column""" + """Apply Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time``""" conn = op.get_bind() if conn.dialect.name == "mssql": with op.batch_alter_table('dag') as batch_op: @@ -51,7 +52,7 @@ def upgrade(): def downgrade(): - """Unapply rename last_scheduler_run column""" + """Unapply Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time``""" conn = op.get_bind() if conn.dialect.name == "mssql": with op.batch_alter_table('dag') as batch_op: diff --git a/airflow/migrations/versions/2e541a1dcfed_task_duration.py b/airflow/migrations/versions/2e541a1dcfed_task_duration.py index e7535bdf254ed..9fa217d1aa181 100644 --- a/airflow/migrations/versions/2e541a1dcfed_task_duration.py +++ b/airflow/migrations/versions/2e541a1dcfed_task_duration.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""task_duration +"""Change ``task_instance.task_duration`` type to ``FLOAT`` Revision ID: 2e541a1dcfed Revises: 1b38cef5b76e @@ -33,6 +33,7 @@ down_revision = '1b38cef5b76e' branch_labels = None depends_on = None +airflow_version = '1.6.0' def upgrade(): diff --git a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py index cf461c0323031..7259eb5c1821c 100644 --- a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py +++ b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""rename user table +"""Rename user table Revision ID: 2e82aab8ef20 Revises: 1968acfc09e3 @@ -30,6 +30,7 @@ down_revision = '1968acfc09e3' branch_labels = None depends_on = None +airflow_version = '1.7.1' def upgrade(): diff --git a/airflow/migrations/versions/30867afad44a_rename_concurrency_column_in_dag_table_.py b/airflow/migrations/versions/30867afad44a_rename_concurrency_column_in_dag_table_.py index 384fbc41ce306..f71fdf38821c8 100644 --- a/airflow/migrations/versions/30867afad44a_rename_concurrency_column_in_dag_table_.py +++ b/airflow/migrations/versions/30867afad44a_rename_concurrency_column_in_dag_table_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Rename concurrency column in dag table to max_active_tasks +"""Rename ``concurrency`` column in ``dag`` table to`` max_active_tasks`` Revision ID: 30867afad44a Revises: e9304a3141f0 @@ -32,10 +32,11 @@ down_revision = 'e9304a3141f0' branch_labels = None depends_on = None +airflow_version = '2.2.0' def upgrade(): - """Apply Rename concurrency column in dag table to max_active_tasks""" + """Apply Rename ``concurrency`` column in ``dag`` table to`` max_active_tasks``""" conn = op.get_bind() is_sqlite = bool(conn.dialect.name == "sqlite") @@ -53,7 +54,7 @@ def upgrade(): def downgrade(): - """Unapply Rename concurrency column in dag table to max_active_tasks""" + """Unapply Rename ``concurrency`` column in ``dag`` table to`` max_active_tasks``""" with op.batch_alter_table('dag') as batch_op: batch_op.alter_column( 'max_active_tasks', diff --git a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py index b8ab2908c404b..2eb793addaed5 100644 --- a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py +++ b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""More logging into task_instance +"""Add ``operator`` and ``queued_dttm`` to ``task_instance`` table Revision ID: 338e90f54d61 Revises: 13eb55f81627 @@ -31,6 +31,7 @@ down_revision = '13eb55f81627' branch_labels = None depends_on = None +airflow_version = '1.5.0' def upgrade(): diff --git a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py index d8d600be0ccee..6273c01f2633c 100644 --- a/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py +++ b/airflow/migrations/versions/33ae817a1ff4_add_kubernetes_resource_checkpointing.py @@ -16,7 +16,7 @@ # under the License. -"""kubernetes_resource_checkpointing +"""Add Kubernetes resource check-pointing Revision ID: 33ae817a1ff4 Revises: 947454bf1dff @@ -32,6 +32,7 @@ down_revision = 'd2ae31099d61' branch_labels = None depends_on = None +airflow_version = '1.10.0' RESOURCE_TABLE = "kube_resource_version" diff --git a/airflow/migrations/versions/364159666cbd_add_job_id_to_dagrun_table.py b/airflow/migrations/versions/364159666cbd_add_job_id_to_dagrun_table.py index 8e6989a65ccc8..c42308393fef2 100644 --- a/airflow/migrations/versions/364159666cbd_add_job_id_to_dagrun_table.py +++ b/airflow/migrations/versions/364159666cbd_add_job_id_to_dagrun_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add creating_job_id to DagRun table +"""Add ``creating_job_id`` to ``DagRun`` table Revision ID: 364159666cbd Revises: 52d53670a240 @@ -32,10 +32,11 @@ down_revision = '52d53670a240' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply Add creating_job_id to DagRun table""" + """Apply Add ``creating_job_id`` to ``DagRun`` table""" op.add_column('dag_run', sa.Column('creating_job_id', sa.Integer)) diff --git a/airflow/migrations/versions/3c20cacc0044_add_dagrun_run_type.py b/airflow/migrations/versions/3c20cacc0044_add_dagrun_run_type.py index c2a636a45a696..19c310b62da1b 100644 --- a/airflow/migrations/versions/3c20cacc0044_add_dagrun_run_type.py +++ b/airflow/migrations/versions/3c20cacc0044_add_dagrun_run_type.py @@ -17,7 +17,7 @@ # under the License. """ -Add DagRun run_type +Add ``run_type`` column in ``dag_run`` table Revision ID: 3c20cacc0044 Revises: b25a55525161 @@ -38,6 +38,7 @@ down_revision = "b25a55525161" branch_labels = None depends_on = None +airflow_version = '2.0.0' Base = declarative_base() @@ -53,7 +54,7 @@ class DagRun(Base): # type: ignore def upgrade(): - """Apply Add DagRun run_type""" + """Apply Add ``run_type`` column in ``dag_run`` table""" run_type_col_type = sa.String(length=50) conn = op.get_bind() @@ -88,5 +89,5 @@ def upgrade(): def downgrade(): - """Unapply Add DagRun run_type""" + """Unapply Add ``run_type`` column in ``dag_run`` table""" op.drop_column("dag_run", "run_type") diff --git a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py index 7ce183827fbe8..ae7790713797c 100644 --- a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py +++ b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""dagrun_config +"""Add ``conf`` column in ``dag_run`` table Revision ID: 40e67319e3a9 Revises: 2e541a1dcfed @@ -31,6 +31,7 @@ down_revision = '2e541a1dcfed' branch_labels = None depends_on = None +airflow_version = '1.6.0' def upgrade(): diff --git a/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py index 8b3e9b4a41de5..11ea2938deb7d 100644 --- a/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py +++ b/airflow/migrations/versions/41f5f12752f8_add_superuser_field.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""add superuser field +"""Add superuser field Revision ID: 41f5f12752f8 Revises: 03bc53e68815 @@ -31,6 +31,7 @@ down_revision = '03bc53e68815' branch_labels = None depends_on = None +airflow_version = '1.10.2' def upgrade(): diff --git a/airflow/migrations/versions/4446e08588_dagrun_start_end.py b/airflow/migrations/versions/4446e08588_dagrun_start_end.py index 48c85d5c94b70..ae471443b04e8 100644 --- a/airflow/migrations/versions/4446e08588_dagrun_start_end.py +++ b/airflow/migrations/versions/4446e08588_dagrun_start_end.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""dagrun start end +"""Add ``start_date`` and ``end_date`` in ``dag_run`` table Revision ID: 4446e08588 Revises: 561833c1c74b @@ -32,6 +32,7 @@ down_revision = '561833c1c74b' branch_labels = None depends_on = None +airflow_version = '1.6.2' def upgrade(): diff --git a/airflow/migrations/versions/449b4072c2da_increase_size_of_connection_extra_field_.py b/airflow/migrations/versions/449b4072c2da_increase_size_of_connection_extra_field_.py index 808d435870dd6..77873664f6c20 100644 --- a/airflow/migrations/versions/449b4072c2da_increase_size_of_connection_extra_field_.py +++ b/airflow/migrations/versions/449b4072c2da_increase_size_of_connection_extra_field_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Increase size of connection.extra field to handle multiple RSA keys +"""Increase size of ``connection.extra`` field to handle multiple RSA keys Revision ID: 449b4072c2da Revises: 82b7c48c147f @@ -32,6 +32,7 @@ down_revision = '82b7c48c147f' branch_labels = None depends_on = None +airflow_version = '2.0.2' def upgrade(): diff --git a/airflow/migrations/versions/45ba3f1493b9_add_k8s_yaml_to_rendered_templates.py b/airflow/migrations/versions/45ba3f1493b9_add_k8s_yaml_to_rendered_templates.py index f78402e867066..675df14259230 100644 --- a/airflow/migrations/versions/45ba3f1493b9_add_k8s_yaml_to_rendered_templates.py +++ b/airflow/migrations/versions/45ba3f1493b9_add_k8s_yaml_to_rendered_templates.py @@ -35,6 +35,7 @@ down_revision = '364159666cbd' branch_labels = None depends_on = None +airflow_version = '2.0.0' __tablename__ = "rendered_task_instance_fields" k8s_pod_yaml = Column('k8s_pod_yaml', sqlalchemy_jsonfield.JSONField(json=json), nullable=True) diff --git a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py index 00ec5df75b86a..9cee531e749d5 100644 --- a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py +++ b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add fractional seconds to mysql tables +"""Add fractional seconds to MySQL tables Revision ID: 4addfa1236f1 Revises: f2ca10b85618 @@ -32,6 +32,7 @@ down_revision = 'f2ca10b85618' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py index 1bf5f0d3924ae..5bc28ad372633 100644 --- a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py +++ b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Adding extra to Log +"""Adding ``extra`` column to ``Log`` table Revision ID: 502898887f84 Revises: 52d714495f0 @@ -31,6 +31,7 @@ down_revision = '52d714495f0' branch_labels = None depends_on = None +airflow_version = '1.6.0' def upgrade(): diff --git a/airflow/migrations/versions/52d53670a240_fix_mssql_exec_date_rendered_task_instance.py b/airflow/migrations/versions/52d53670a240_fix_mssql_exec_date_rendered_task_instance.py index 122821b8d896b..dd14290729997 100644 --- a/airflow/migrations/versions/52d53670a240_fix_mssql_exec_date_rendered_task_instance.py +++ b/airflow/migrations/versions/52d53670a240_fix_mssql_exec_date_rendered_task_instance.py @@ -32,6 +32,7 @@ down_revision = '98271e7606e2' branch_labels = None depends_on = None +airflow_version = '2.0.0' TABLE_NAME = 'rendered_task_instance_fields' diff --git a/airflow/migrations/versions/52d714495f0_job_id_indices.py b/airflow/migrations/versions/52d714495f0_job_id_indices.py index abf4e2d00c2ef..e6ba4fd226670 100644 --- a/airflow/migrations/versions/52d714495f0_job_id_indices.py +++ b/airflow/migrations/versions/52d714495f0_job_id_indices.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""job_id indices +"""Add indices in ``job`` table Revision ID: 52d714495f0 Revises: 338e90f54d61 @@ -30,6 +30,7 @@ down_revision = '338e90f54d61' branch_labels = None depends_on = None +airflow_version = '1.5.2' def upgrade(): diff --git a/airflow/migrations/versions/54bebd308c5f_add_trigger_table_and_task_info.py b/airflow/migrations/versions/54bebd308c5f_add_trigger_table_and_task_info.py index 8647403857260..0f83d4ff8dd59 100644 --- a/airflow/migrations/versions/54bebd308c5f_add_trigger_table_and_task_info.py +++ b/airflow/migrations/versions/54bebd308c5f_add_trigger_table_and_task_info.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add trigger table and task info +"""Adds ``trigger`` table and deferrable operator columns to task instance Revision ID: 54bebd308c5f Revises: 30867afad44a @@ -34,10 +34,11 @@ down_revision = '30867afad44a' branch_labels = None depends_on = None +airflow_version = '2.2.0' def upgrade(): - """Apply Add trigger table and task info""" + """Apply Adds ``trigger`` table and deferrable operator columns to task instance""" op.create_table( 'trigger', sa.Column('id', sa.Integer(), primary_key=True, nullable=False), @@ -58,7 +59,7 @@ def upgrade(): def downgrade(): - """Unapply Add trigger table and task info""" + """Unapply Adds ``trigger`` table and deferrable operator columns to task instance""" with op.batch_alter_table('task_instance', schema=None) as batch_op: batch_op.drop_constraint('task_instance_trigger_id_fkey', type_='foreignkey') batch_op.drop_index('ti_trigger_id') diff --git a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py index 5a83b113b8ea8..d1004ba52a8d3 100644 --- a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py +++ b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add password column to user +"""Add ``password`` column to ``user`` table Revision ID: 561833c1c74b Revises: 40e67319e3a9 @@ -31,6 +31,7 @@ down_revision = '40e67319e3a9' branch_labels = None depends_on = None +airflow_version = '1.6.2' def upgrade(): diff --git a/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py index c643a6298442a..693984b6edbfc 100644 --- a/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py +++ b/airflow/migrations/versions/587bdf053233_adding_index_for_dag_id_in_job.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""adding index for dag_id in job +"""Add index for ``dag_id`` column in ``job`` table. Revision ID: 587bdf053233 Revises: c381b21cb7e4 @@ -31,13 +31,14 @@ down_revision = 'c381b21cb7e4' branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): - """Apply adding index for dag_id in job""" + """Apply Add index for ``dag_id`` column in ``job`` table.""" op.create_index('idx_job_dag_id', 'job', ['dag_id'], unique=False) def downgrade(): - """Unapply adding index for dag_id in job""" + """Unapply Add index for ``dag_id`` column in ``job`` table.""" op.drop_index('idx_job_dag_id', table_name='job') diff --git a/airflow/migrations/versions/5e3ec427fdd3_increase_length_of_email_and_username.py b/airflow/migrations/versions/5e3ec427fdd3_increase_length_of_email_and_username.py index bd78fdd28af9b..54087dfe02d1d 100644 --- a/airflow/migrations/versions/5e3ec427fdd3_increase_length_of_email_and_username.py +++ b/airflow/migrations/versions/5e3ec427fdd3_increase_length_of_email_and_username.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Increase length of email and username +"""Increase length of email and username in ``ab_user`` and ``ab_register_user`` table to ``256`` characters Revision ID: 5e3ec427fdd3 Revises: 587bdf053233 @@ -32,6 +32,7 @@ down_revision = '587bdf053233' branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): diff --git a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py index 7078f8132be25..cea1d1010a888 100644 --- a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py +++ b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add pid field to TaskInstance +"""Add ``pid`` field to ``TaskInstance`` Revision ID: 5e7d17757c7a Revises: 8504051e801b @@ -32,6 +32,7 @@ down_revision = '8504051e801b' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/61ec73d9401f_add_description_field_to_connection.py b/airflow/migrations/versions/61ec73d9401f_add_description_field_to_connection.py index ff480c7d5e0e1..4c3f5835dcbfd 100644 --- a/airflow/migrations/versions/61ec73d9401f_add_description_field_to_connection.py +++ b/airflow/migrations/versions/61ec73d9401f_add_description_field_to_connection.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add description field to connection +"""Add description field to ``connection`` table Revision ID: 61ec73d9401f Revises: 2c6edca13270 @@ -32,10 +32,11 @@ down_revision = '2c6edca13270' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply Add description field to connection""" + """Apply Add description field to ``connection`` table""" conn = op.get_bind() with op.batch_alter_table('connection') as batch_op: @@ -49,6 +50,6 @@ def upgrade(): def downgrade(): - """Unapply Add description field to connection""" + """Unapply Add description field to ``connection`` table""" with op.batch_alter_table('connection', schema=None) as batch_op: batch_op.drop_column('description') diff --git a/airflow/migrations/versions/64a7d6477aae_fix_description_field_in_connection_to_.py b/airflow/migrations/versions/64a7d6477aae_fix_description_field_in_connection_to_.py index fa7b2b21e2c22..dba78c04c461a 100644 --- a/airflow/migrations/versions/64a7d6477aae_fix_description_field_in_connection_to_.py +++ b/airflow/migrations/versions/64a7d6477aae_fix_description_field_in_connection_to_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""fix description field in connection to be text +"""Fix description field in ``connection`` to be ``text`` Revision ID: 64a7d6477aae Revises: f5b5ec089444 @@ -32,10 +32,11 @@ down_revision = '61ec73d9401f' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply fix description field in connection to be text""" + """Apply Fix description field in ``connection`` to be ``text``""" conn = op.get_bind() if conn.dialect.name == "sqlite": # in sqlite TEXT and STRING column types are the same @@ -54,7 +55,7 @@ def upgrade(): def downgrade(): - """Unapply fix description field in connection to be text""" + """Unapply Fix description field in ``connection`` to be ``text``""" conn = op.get_bind() if conn.dialect.name == "sqlite": # in sqlite TEXT and STRING column types are the same diff --git a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py index c2dbb647089a2..679efb70a1818 100644 --- a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py +++ b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add task fails journal table +"""Add ``task_fail`` table Revision ID: 64de9cddf6c9 Revises: 211e584da130 @@ -33,6 +33,7 @@ down_revision = '211e584da130' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py b/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py index cdbe686a74a93..53261af4a87e8 100644 --- a/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py +++ b/airflow/migrations/versions/6e96a59344a4_make_taskinstance_pool_not_nullable.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Make TaskInstance.pool not nullable +"""Make ``TaskInstance.pool`` not nullable Revision ID: 6e96a59344a4 Revises: 939bb1e647c8 @@ -37,6 +37,7 @@ down_revision = '939bb1e647c8' branch_labels = None depends_on = None +airflow_version = '1.10.4' Base = declarative_base() ID_LEN = 250 diff --git a/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py b/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py index 7fa4ab97208d5..49e3e9b84a7e0 100644 --- a/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py +++ b/airflow/migrations/versions/74effc47d867_change_datetime_to_datetime2_6_on_mssql_.py @@ -34,6 +34,7 @@ down_revision = '6e96a59344a4' branch_labels = None depends_on = None +airflow_version = '1.10.5' def upgrade(): diff --git a/airflow/migrations/versions/786e3737b18f_added_timetable_description_column.py b/airflow/migrations/versions/786e3737b18f_added_timetable_description_column.py index 3741822a20692..ca569cd12b70d 100644 --- a/airflow/migrations/versions/786e3737b18f_added_timetable_description_column.py +++ b/airflow/migrations/versions/786e3737b18f_added_timetable_description_column.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Added timetable description column +"""Add ``timetable_description`` column to DagModel for UI. Revision ID: 786e3737b18f Revises: 5e3ec427fdd3 @@ -32,15 +32,16 @@ down_revision = '5e3ec427fdd3' branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): - """Apply Added timetable description column""" + """Apply Add ``timetable_description`` column to DagModel for UI.""" with op.batch_alter_table('dag', schema=None) as batch_op: batch_op.add_column(sa.Column('timetable_description', sa.String(length=1000), nullable=True)) def downgrade(): - """Unapply Added timetable description column""" + """Unapply Add ``timetable_description`` column to DagModel for UI.""" with op.batch_alter_table('dag', schema=None) as batch_op: batch_op.drop_column('timetable_description') diff --git a/airflow/migrations/versions/7939bcff74ba_add_dagtags_table.py b/airflow/migrations/versions/7939bcff74ba_add_dagtags_table.py index a19fee24f3772..a7ae10860e8aa 100644 --- a/airflow/migrations/versions/7939bcff74ba_add_dagtags_table.py +++ b/airflow/migrations/versions/7939bcff74ba_add_dagtags_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add DagTags table +"""Add ``DagTags`` table Revision ID: 7939bcff74ba Revises: fe461863935f @@ -34,10 +34,11 @@ down_revision = 'fe461863935f' branch_labels = None depends_on = None +airflow_version = '1.10.8' def upgrade(): - """Apply Add DagTags table""" + """Apply Add ``DagTags`` table""" op.create_table( 'dag_tag', sa.Column('name', sa.String(length=100), nullable=False), @@ -51,5 +52,5 @@ def upgrade(): def downgrade(): - """Unapply Add DagTags table""" + """Unapply Add ``DagTags`` table""" op.drop_table('dag_tag') diff --git a/airflow/migrations/versions/7b2661a43ba3_taskinstance_keyed_to_dagrun.py b/airflow/migrations/versions/7b2661a43ba3_taskinstance_keyed_to_dagrun.py index ac50fa746423b..eee0bd23a1eac 100644 --- a/airflow/migrations/versions/7b2661a43ba3_taskinstance_keyed_to_dagrun.py +++ b/airflow/migrations/versions/7b2661a43ba3_taskinstance_keyed_to_dagrun.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""TaskInstance keyed to DagRun +"""Change ``TaskInstance`` and ``TaskReschedule`` tables from execution_date to run_id. Revision ID: 7b2661a43ba3 Revises: 142555e44c17 @@ -39,6 +39,7 @@ down_revision = '142555e44c17' branch_labels = None depends_on = None +airflow_version = '2.2.0' # Just Enough Table to run the conditions for update. @@ -90,7 +91,7 @@ def get_table_constraints(conn, table_name): def upgrade(): - """Apply TaskInstance keyed to DagRun""" + """Apply Change ``TaskInstance`` and ``TaskReschedule`` tables from execution_date to run_id.""" conn = op.get_bind() dialect_name = conn.dialect.name @@ -311,7 +312,7 @@ def upgrade(): def downgrade(): - """Unapply TaskInstance keyed to DagRun""" + """Unapply Change ``TaskInstance`` and ``TaskReschedule`` tables from execution_date to run_id.""" dialect_name = op.get_bind().dialect.name dt_type = TIMESTAMP string_id_col_type = StringID() diff --git a/airflow/migrations/versions/82b7c48c147f_remove_can_read_permission_on_config_.py b/airflow/migrations/versions/82b7c48c147f_remove_can_read_permission_on_config_.py index 1926e4d34b9bb..51e8f20dbaf6c 100644 --- a/airflow/migrations/versions/82b7c48c147f_remove_can_read_permission_on_config_.py +++ b/airflow/migrations/versions/82b7c48c147f_remove_can_read_permission_on_config_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Remove can_read permission on config resource for User and Viewer role +"""Remove ``can_read`` permission on config resource for ``User`` and ``Viewer`` role Revision ID: 82b7c48c147f Revises: e959f08ac86c @@ -33,6 +33,7 @@ down_revision = 'e959f08ac86c' branch_labels = None depends_on = None +airflow_version = '2.0.1' def upgrade(): diff --git a/airflow/migrations/versions/83f031fd9f1c_improve_mssql_compatibility.py b/airflow/migrations/versions/83f031fd9f1c_improve_mssql_compatibility.py index 97d0019ac44cc..66fe513713fc3 100644 --- a/airflow/migrations/versions/83f031fd9f1c_improve_mssql_compatibility.py +++ b/airflow/migrations/versions/83f031fd9f1c_improve_mssql_compatibility.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""improve mssql compatibility +"""Improve MSSQL compatibility Revision ID: 83f031fd9f1c Revises: ccde3e26fe78 @@ -37,6 +37,7 @@ down_revision = 'ccde3e26fe78' branch_labels = None depends_on = None +airflow_version = '2.2.0' def is_table_empty(conn, table_name): diff --git a/airflow/migrations/versions/849da589634d_prefix_dag_permissions.py b/airflow/migrations/versions/849da589634d_prefix_dag_permissions.py index 15da1113d4c61..2538a12a89494 100644 --- a/airflow/migrations/versions/849da589634d_prefix_dag_permissions.py +++ b/airflow/migrations/versions/849da589634d_prefix_dag_permissions.py @@ -35,6 +35,7 @@ down_revision = '45ba3f1493b9' branch_labels = None depends_on = None +airflow_version = '2.0.0' def prefix_individual_dag_permissions(session): diff --git a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py index 6f60ac6b12335..ea2aba8e32620 100644 --- a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py +++ b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""xcom dag task indices +"""Add indices on ``xcom`` table Revision ID: 8504051e801b Revises: 4addfa1236f1 @@ -31,6 +31,7 @@ down_revision = '4addfa1236f1' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/852ae6c715af_add_rendered_task_instance_fields_table.py b/airflow/migrations/versions/852ae6c715af_add_rendered_task_instance_fields_table.py index 3a32841dba0f1..2027dd4745fb7 100644 --- a/airflow/migrations/versions/852ae6c715af_add_rendered_task_instance_fields_table.py +++ b/airflow/migrations/versions/852ae6c715af_add_rendered_task_instance_fields_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add RenderedTaskInstanceFields table +"""Add ``RenderedTaskInstanceFields`` table Revision ID: 852ae6c715af Revises: a4c2fd67d16b @@ -34,12 +34,13 @@ down_revision = 'a4c2fd67d16b' branch_labels = None depends_on = None +airflow_version = '1.10.10' TABLE_NAME = 'rendered_task_instance_fields' def upgrade(): - """Apply Add RenderedTaskInstanceFields table""" + """Apply Add ``RenderedTaskInstanceFields`` table""" json_type = sa.JSON conn = op.get_bind() diff --git a/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py index 92a97fa8c2ae7..abf289572fbda 100644 --- a/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py +++ b/airflow/migrations/versions/856955da8476_fix_sqlite_foreign_key.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""fix sqlite foreign key +"""Fix Sqlite foreign key Revision ID: 856955da8476 Revises: f23433877c24 @@ -31,6 +31,7 @@ down_revision = 'f23433877c24' branch_labels = None depends_on = None +airflow_version = '1.10.0' def upgrade(): diff --git a/airflow/migrations/versions/8646922c8a04_change_default_pool_slots_to_1.py b/airflow/migrations/versions/8646922c8a04_change_default_pool_slots_to_1.py index 9dea4d64ca9a2..f5ae34c2977c9 100644 --- a/airflow/migrations/versions/8646922c8a04_change_default_pool_slots_to_1.py +++ b/airflow/migrations/versions/8646922c8a04_change_default_pool_slots_to_1.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Change default pool_slots to 1 +"""Change default ``pool_slots`` to ``1`` Revision ID: 8646922c8a04 Revises: 449b4072c2da @@ -32,15 +32,16 @@ down_revision = '449b4072c2da' branch_labels = None depends_on = None +airflow_version = '2.0.2' def upgrade(): - """Change default pool_slots to 1 and make pool_slots not nullable""" + """Change default ``pool_slots`` to ``1`` and make pool_slots not nullable""" with op.batch_alter_table("task_instance", schema=None) as batch_op: batch_op.alter_column("pool_slots", existing_type=sa.Integer, nullable=False, server_default='1') def downgrade(): - """Unapply Change default pool_slots to 1""" + """Unapply Change default ``pool_slots`` to ``1``""" with op.batch_alter_table("task_instance", schema=None) as batch_op: batch_op.alter_column("pool_slots", existing_type=sa.Integer, nullable=True, server_default=None) diff --git a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py index bad9f5ddf52df..4dd1864580a9d 100644 --- a/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py +++ b/airflow/migrations/versions/86770d1215c0_add_kubernetes_scheduler_uniqueness.py @@ -16,7 +16,7 @@ # under the License. -"""add kubernetes scheduler uniqueness +"""Add kubernetes scheduler uniqueness Revision ID: 86770d1215c0 Revises: 27c6a30d7c24 @@ -31,6 +31,7 @@ down_revision = '27c6a30d7c24' branch_labels = None depends_on = None +airflow_version = '1.10.0' RESOURCE_TABLE = "kube_worker_uuid" diff --git a/airflow/migrations/versions/8d48763f6d53_add_unique_constraint_to_conn_id.py b/airflow/migrations/versions/8d48763f6d53_add_unique_constraint_to_conn_id.py index b06c8687c5285..e3172ddf34172 100644 --- a/airflow/migrations/versions/8d48763f6d53_add_unique_constraint_to_conn_id.py +++ b/airflow/migrations/versions/8d48763f6d53_add_unique_constraint_to_conn_id.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add unique constraint to conn_id +"""Add unique constraint to ``conn_id`` Revision ID: 8d48763f6d53 Revises: 8f966b9c467a @@ -34,10 +34,11 @@ down_revision = '8f966b9c467a' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply add unique constraint to conn_id and set it as non-nullable""" + """Apply Add unique constraint to ``conn_id`` and set it as non-nullable""" try: with op.batch_alter_table('connection') as batch_op: batch_op.alter_column("conn_id", nullable=False, existing_type=sa.String(250, **COLLATION_ARGS)) @@ -48,7 +49,7 @@ def upgrade(): def downgrade(): - """Unapply add unique constraint to conn_id and set it as non-nullable""" + """Unapply Add unique constraint to ``conn_id`` and set it as non-nullable""" with op.batch_alter_table('connection') as batch_op: batch_op.drop_constraint(constraint_name="unique_conn_id", type_="unique") diff --git a/airflow/migrations/versions/8f966b9c467a_set_conn_type_as_non_nullable.py b/airflow/migrations/versions/8f966b9c467a_set_conn_type_as_non_nullable.py index c912c3de0b6a2..1df5bfbfe324c 100644 --- a/airflow/migrations/versions/8f966b9c467a_set_conn_type_as_non_nullable.py +++ b/airflow/migrations/versions/8f966b9c467a_set_conn_type_as_non_nullable.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Set conn_type as non-nullable +"""Set ``conn_type`` as non-nullable Revision ID: 8f966b9c467a Revises: 3c20cacc0044 @@ -33,10 +33,11 @@ down_revision = "3c20cacc0044" branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply Set conn_type as non-nullable""" + """Apply Set ``conn_type`` as non-nullable""" Base = declarative_base() class Connection(Base): @@ -62,6 +63,6 @@ class Connection(Base): def downgrade(): - """Unapply Set conn_type as non-nullable""" + """Unapply Set ``conn_type`` as non-nullable""" with op.batch_alter_table("connection", schema=None) as batch_op: batch_op.alter_column("conn_type", existing_type=sa.VARCHAR(length=500), nullable=True) diff --git a/airflow/migrations/versions/90d1635d7b86_increase_pool_name_size_in_taskinstance.py b/airflow/migrations/versions/90d1635d7b86_increase_pool_name_size_in_taskinstance.py index 940840b61d309..bc7d61f859ab3 100644 --- a/airflow/migrations/versions/90d1635d7b86_increase_pool_name_size_in_taskinstance.py +++ b/airflow/migrations/versions/90d1635d7b86_increase_pool_name_size_in_taskinstance.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Increase pool name size in TaskInstance +"""Increase maximum length of pool name in ``task_instance`` table to ``256`` characters Revision ID: 90d1635d7b86 Revises: 2e42bb497a22 @@ -32,15 +32,16 @@ down_revision = '2e42bb497a22' branch_labels = None depends_on = None +airflow_version = '2.1.0' def upgrade(): - """Apply Increase pool name size in TaskInstance""" + """Apply Increase maximum length of pool name in ``task_instance`` table to ``256`` characters""" with op.batch_alter_table('task_instance') as batch_op: batch_op.alter_column('pool', type_=sa.String(256), nullable=False) def downgrade(): - """Unapply Increase pool name size in TaskInstance""" + """Unapply Increase maximum length of pool name in ``task_instance`` table to ``256`` characters""" with op.batch_alter_table('task_instance') as batch_op: batch_op.alter_column('pool', type_=sa.String(50), nullable=False) diff --git a/airflow/migrations/versions/92c57b58940d_add_fab_tables.py b/airflow/migrations/versions/92c57b58940d_add_fab_tables.py index 38f3c618eb8e6..775f7140fe94f 100644 --- a/airflow/migrations/versions/92c57b58940d_add_fab_tables.py +++ b/airflow/migrations/versions/92c57b58940d_add_fab_tables.py @@ -33,6 +33,7 @@ down_revision = 'da3f683c3a5a' branch_labels = None depends_on = None +airflow_version = '1.10.13' def upgrade(): diff --git a/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py b/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py index 7c51df4b5ffb1..9a535cbc99c48 100644 --- a/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py +++ b/airflow/migrations/versions/939bb1e647c8_task_reschedule_fk_on_cascade_delete.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""task reschedule fk on cascade delete +"""task reschedule foreign key on cascade delete Revision ID: 939bb1e647c8 Revises: dd4ecb8fbee3 @@ -30,6 +30,7 @@ down_revision = 'dd4ecb8fbee3' branch_labels = None depends_on = None +airflow_version = '1.10.3' def upgrade(): diff --git a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py index e7c948e66afdc..364e57d50360d 100644 --- a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py +++ b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add ti job_id index +"""Create index on ``job_id`` column in ``task_instance`` table Revision ID: 947454bf1dff Revises: bdaa763e6c56 @@ -30,6 +30,7 @@ down_revision = 'bdaa763e6c56' branch_labels = None depends_on = None +airflow_version = '1.8.2' def upgrade(): diff --git a/airflow/migrations/versions/952da73b5eff_add_dag_code_table.py b/airflow/migrations/versions/952da73b5eff_add_dag_code_table.py index cfd5d079c9017..523a1a0f8c0e5 100644 --- a/airflow/migrations/versions/952da73b5eff_add_dag_code_table.py +++ b/airflow/migrations/versions/952da73b5eff_add_dag_code_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add dag_code table +"""Add ``dag_code`` table Revision ID: 952da73b5eff Revises: 852ae6c715af @@ -34,6 +34,7 @@ down_revision = '852ae6c715af' branch_labels = None depends_on = None +airflow_version = '1.10.10' def upgrade(): diff --git a/airflow/migrations/versions/9635ae0956e7_index_taskfail.py b/airflow/migrations/versions/9635ae0956e7_index_taskfail.py index 84579f4a59085..1323e77f7f040 100644 --- a/airflow/migrations/versions/9635ae0956e7_index_taskfail.py +++ b/airflow/migrations/versions/9635ae0956e7_index_taskfail.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""index-taskfail +"""Create index on ``task_fail`` table Revision ID: 9635ae0956e7 Revises: 856955da8476 @@ -29,6 +29,7 @@ down_revision = '856955da8476' branch_labels = None depends_on = None +airflow_version = '1.10.0' def upgrade(): diff --git a/airflow/migrations/versions/97cdd93827b8_add_queued_at_column_to_dagrun_table.py b/airflow/migrations/versions/97cdd93827b8_add_queued_at_column_to_dagrun_table.py index db44b0fde326b..7561837c53e99 100644 --- a/airflow/migrations/versions/97cdd93827b8_add_queued_at_column_to_dagrun_table.py +++ b/airflow/migrations/versions/97cdd93827b8_add_queued_at_column_to_dagrun_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add queued_at column to dagrun table +"""Add ``queued_at`` column in ``dag_run`` table Revision ID: 97cdd93827b8 Revises: a13f7613ad25 @@ -34,13 +34,14 @@ down_revision = 'a13f7613ad25' branch_labels = None depends_on = None +airflow_version = '2.1.3' def upgrade(): - """Apply Add queued_at column to dagrun table""" + """Apply Add ``queued_at`` column in ``dag_run`` table""" op.add_column('dag_run', sa.Column('queued_at', TIMESTAMP, nullable=True)) def downgrade(): - """Unapply Add queued_at column to dagrun table""" + """Unapply Add ``queued_at`` column in ``dag_run`` table""" op.drop_column('dag_run', "queued_at") diff --git a/airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py b/airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py index c27f30fc3156c..66f25cf29fac8 100644 --- a/airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py +++ b/airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add scheduling_decision to DagRun and DAG +"""Add ``scheduling_decision`` to ``DagRun`` and ``DAG`` Revision ID: 98271e7606e2 Revises: bef4f3d11e8b @@ -34,10 +34,11 @@ down_revision = 'bef4f3d11e8b' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): - """Apply Add scheduling_decision to DagRun and DAG""" + """Apply Add ``scheduling_decision`` to ``DagRun`` and ``DAG``""" conn = op.get_bind() is_sqlite = bool(conn.dialect.name == "sqlite") is_mssql = bool(conn.dialect.name == "mssql") @@ -87,7 +88,7 @@ def upgrade(): def downgrade(): - """Unapply Add scheduling_decision to DagRun and DAG""" + """Unapply Add ``scheduling_decision`` to ``DagRun`` and ``DAG``""" conn = op.get_bind() is_sqlite = bool(conn.dialect.name == "sqlite") diff --git a/airflow/migrations/versions/a13f7613ad25_resource_based_permissions_for_default_.py b/airflow/migrations/versions/a13f7613ad25_resource_based_permissions_for_default_.py index 5ccde155c08fb..cd162de566e88 100644 --- a/airflow/migrations/versions/a13f7613ad25_resource_based_permissions_for_default_.py +++ b/airflow/migrations/versions/a13f7613ad25_resource_based_permissions_for_default_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Resource based permissions for default FAB views. +"""Resource based permissions for default ``Flask-AppBuilder`` views Revision ID: a13f7613ad25 Revises: e165e7455d70 @@ -33,6 +33,7 @@ down_revision = 'e165e7455d70' branch_labels = None depends_on = None +airflow_version = '2.1.0' mapping = { diff --git a/airflow/migrations/versions/a3bcd0914482_add_data_compressed_to_serialized_dag.py b/airflow/migrations/versions/a3bcd0914482_add_data_compressed_to_serialized_dag.py index 9e19e80de1ae4..f7276d8b2cac2 100644 --- a/airflow/migrations/versions/a3bcd0914482_add_data_compressed_to_serialized_dag.py +++ b/airflow/migrations/versions/a3bcd0914482_add_data_compressed_to_serialized_dag.py @@ -32,6 +32,7 @@ down_revision = 'e655c0453f75' branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): diff --git a/airflow/migrations/versions/a4c2fd67d16b_add_pool_slots_field_to_task_instance.py b/airflow/migrations/versions/a4c2fd67d16b_add_pool_slots_field_to_task_instance.py index 4dd825e8b7de4..26bff68ba2141 100644 --- a/airflow/migrations/versions/a4c2fd67d16b_add_pool_slots_field_to_task_instance.py +++ b/airflow/migrations/versions/a4c2fd67d16b_add_pool_slots_field_to_task_instance.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add pool_slots field to task_instance +"""Add ``pool_slots`` field to ``task_instance`` Revision ID: a4c2fd67d16b Revises: 7939bcff74ba @@ -32,6 +32,7 @@ down_revision = '7939bcff74ba' branch_labels = None depends_on = None +airflow_version = '1.10.10' def upgrade(): diff --git a/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py index a9ea301e2737f..454482e71877a 100644 --- a/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py +++ b/airflow/migrations/versions/a56c9515abdc_remove_dag_stat_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Remove dag_stat table +"""Remove ``dag_stat`` table Revision ID: a56c9515abdc Revises: c8ffec048a3b @@ -32,6 +32,7 @@ down_revision = 'c8ffec048a3b' branch_labels = None depends_on = None +airflow_version = '1.10.3' def upgrade(): diff --git a/airflow/migrations/versions/a66efa278eea_add_precision_to_execution_date_in_mysql.py b/airflow/migrations/versions/a66efa278eea_add_precision_to_execution_date_in_mysql.py index 29cce5feee310..873ba148d03d6 100644 --- a/airflow/migrations/versions/a66efa278eea_add_precision_to_execution_date_in_mysql.py +++ b/airflow/migrations/versions/a66efa278eea_add_precision_to_execution_date_in_mysql.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add Precision to execution_date in RenderedTaskInstanceFields table +"""Add Precision to ``execution_date`` in ``RenderedTaskInstanceFields`` table Revision ID: a66efa278eea Revises: 952da73b5eff @@ -32,13 +32,14 @@ down_revision = '952da73b5eff' branch_labels = None depends_on = None +airflow_version = '1.10.11' TABLE_NAME = 'rendered_task_instance_fields' COLUMN_NAME = 'execution_date' def upgrade(): - """Add Precision to execution_date in RenderedTaskInstanceFields table for MySQL""" + """Add Precision to ``execution_date`` in ``RenderedTaskInstanceFields`` table for MySQL""" conn = op.get_bind() if conn.dialect.name == "mysql": op.alter_column( @@ -47,7 +48,7 @@ def upgrade(): def downgrade(): - """Unapply Add Precision to execution_date in RenderedTaskInstanceFields table""" + """Unapply Add Precision to ``execution_date`` in ``RenderedTaskInstanceFields`` table""" conn = op.get_bind() if conn.dialect.name == "mysql": op.alter_column( diff --git a/airflow/migrations/versions/b247b1e3d1ed_add_queued_by_job_id_to_ti.py b/airflow/migrations/versions/b247b1e3d1ed_add_queued_by_job_id_to_ti.py index 794f64ec9967e..90247d02ef1b5 100644 --- a/airflow/migrations/versions/b247b1e3d1ed_add_queued_by_job_id_to_ti.py +++ b/airflow/migrations/versions/b247b1e3d1ed_add_queued_by_job_id_to_ti.py @@ -32,6 +32,7 @@ down_revision = 'e38be357a868' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/b25a55525161_increase_length_of_pool_name.py b/airflow/migrations/versions/b25a55525161_increase_length_of_pool_name.py index 1989af7c2dede..b25eae2b3b780 100644 --- a/airflow/migrations/versions/b25a55525161_increase_length_of_pool_name.py +++ b/airflow/migrations/versions/b25a55525161_increase_length_of_pool_name.py @@ -34,6 +34,7 @@ down_revision = 'bbf4a7ad0465' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/b3b105409875_add_root_dag_id_to_dag.py b/airflow/migrations/versions/b3b105409875_add_root_dag_id_to_dag.py index a193f9fed9706..3d184e6b9bd1d 100644 --- a/airflow/migrations/versions/b3b105409875_add_root_dag_id_to_dag.py +++ b/airflow/migrations/versions/b3b105409875_add_root_dag_id_to_dag.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add root_dag_id to DAG +"""Add ``root_dag_id`` to ``DAG`` Revision ID: b3b105409875 Revises: d38e04c12aa2 @@ -32,15 +32,16 @@ down_revision = 'd38e04c12aa2' branch_labels = None depends_on = None +airflow_version = '1.10.7' def upgrade(): - """Apply add root_dag_id to DAG""" + """Apply Add ``root_dag_id`` to ``DAG``""" op.add_column('dag', sa.Column('root_dag_id', sa.String(length=250), nullable=True)) op.create_index('idx_root_dag_id', 'dag', ['root_dag_id'], unique=False) def downgrade(): - """Unapply add root_dag_id to DAG""" + """Unapply Add ``root_dag_id`` to ``DAG``""" op.drop_index('idx_root_dag_id', table_name='dag') op.drop_column('dag', 'root_dag_id') diff --git a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py index d559fafb1ec26..4e0d53e062654 100644 --- a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py +++ b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py @@ -32,6 +32,7 @@ down_revision = 'bbc73705a13e' branch_labels = None depends_on = None +airflow_version = '1.7.0' def upgrade(): diff --git a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py index 4c6393836bfff..0e3548257c7a8 100644 --- a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py +++ b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add notification_sent column to sla_miss +"""Add ``notification_sent`` column to ``sla_miss`` table Revision ID: bbc73705a13e Revises: 4446e08588 @@ -31,6 +31,7 @@ down_revision = '4446e08588' branch_labels = None depends_on = None +airflow_version = '1.7.0' def upgrade(): diff --git a/airflow/migrations/versions/bbf4a7ad0465_remove_id_column_from_xcom.py b/airflow/migrations/versions/bbf4a7ad0465_remove_id_column_from_xcom.py index 1c52da90471a3..0c191641b7263 100644 --- a/airflow/migrations/versions/bbf4a7ad0465_remove_id_column_from_xcom.py +++ b/airflow/migrations/versions/bbf4a7ad0465_remove_id_column_from_xcom.py @@ -35,6 +35,7 @@ down_revision = 'cf5dc11e79ad' branch_labels = None depends_on = None +airflow_version = '2.0.0' def get_table_constraints(conn, table_name): diff --git a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py index d69d24aed6e91..00a21bf543125 100644 --- a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py +++ b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py @@ -32,6 +32,7 @@ down_revision = 'cc1e65623dc7' branch_labels = None depends_on = None +airflow_version = '1.8.2' def upgrade(): diff --git a/airflow/migrations/versions/be2bfac3da23_add_has_import_errors_column_to_dagmodel.py b/airflow/migrations/versions/be2bfac3da23_add_has_import_errors_column_to_dagmodel.py index 9edef0b7db828..3758805d068f4 100644 --- a/airflow/migrations/versions/be2bfac3da23_add_has_import_errors_column_to_dagmodel.py +++ b/airflow/migrations/versions/be2bfac3da23_add_has_import_errors_column_to_dagmodel.py @@ -32,6 +32,7 @@ down_revision = '7b2661a43ba3' branch_labels = None depends_on = None +airflow_version = '2.2.3' def upgrade(): diff --git a/airflow/migrations/versions/bef4f3d11e8b_drop_kuberesourceversion_and_.py b/airflow/migrations/versions/bef4f3d11e8b_drop_kuberesourceversion_and_.py index cfc201de1dec2..b11b67b436e76 100644 --- a/airflow/migrations/versions/bef4f3d11e8b_drop_kuberesourceversion_and_.py +++ b/airflow/migrations/versions/bef4f3d11e8b_drop_kuberesourceversion_and_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Drop KubeResourceVersion and KubeWorkerId +"""Drop ``KubeResourceVersion`` and ``KubeWorkerId`` Revision ID: bef4f3d11e8b Revises: e1a11ece99cc @@ -33,6 +33,7 @@ down_revision = 'e1a11ece99cc' branch_labels = None depends_on = None +airflow_version = '2.0.0' WORKER_UUID_TABLE = "kube_worker_uuid" @@ -40,7 +41,7 @@ def upgrade(): - """Apply Drop KubeResourceVersion and KubeWorkerIdentifier tables""" + """Apply Drop ``KubeResourceVersion`` and ``KubeWorkerId``entifier tables""" conn = op.get_bind() inspector = Inspector.from_engine(conn) tables = inspector.get_table_names() @@ -52,7 +53,7 @@ def upgrade(): def downgrade(): - """Unapply Drop KubeResourceVersion and KubeWorkerIdentifier tables""" + """Unapply Drop ``KubeResourceVersion`` and ``KubeWorkerId``entifier tables""" conn = op.get_bind() inspector = Inspector.from_engine(conn) tables = inspector.get_table_names() diff --git a/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py b/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py index 7fa96ea276b7b..a52663dcc9dea 100644 --- a/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py +++ b/airflow/migrations/versions/bf00311e1990_add_index_to_taskinstance.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add index to taskinstance +"""Add index to ``task_instance`` table Revision ID: bf00311e1990 Revises: dd25f486b8ea @@ -31,6 +31,7 @@ down_revision = 'dd25f486b8ea' branch_labels = None depends_on = None +airflow_version = '1.10.2' def upgrade(): diff --git a/airflow/migrations/versions/c306b5b5ae4a_switch_xcom_table_to_use_run_id.py b/airflow/migrations/versions/c306b5b5ae4a_switch_xcom_table_to_use_run_id.py index c4d5f2143b380..4e380628e61bc 100644 --- a/airflow/migrations/versions/c306b5b5ae4a_switch_xcom_table_to_use_run_id.py +++ b/airflow/migrations/versions/c306b5b5ae4a_switch_xcom_table_to_use_run_id.py @@ -34,6 +34,7 @@ down_revision = "a3bcd0914482" branch_labels = None depends_on = None +airflow_version = '2.3.0' metadata = MetaData() diff --git a/airflow/migrations/versions/c381b21cb7e4_add_session_table_to_db.py b/airflow/migrations/versions/c381b21cb7e4_add_session_table_to_db.py index cc6b9ab35f0b2..3b70acbdab529 100644 --- a/airflow/migrations/versions/c381b21cb7e4_add_session_table_to_db.py +++ b/airflow/migrations/versions/c381b21cb7e4_add_session_table_to_db.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add session table to db +"""Create a ``session`` table to store web session data Revision ID: c381b21cb7e4 Revises: be2bfac3da23 @@ -32,12 +32,13 @@ down_revision = 'be2bfac3da23' branch_labels = None depends_on = None +airflow_version = '2.2.4' TABLE_NAME = 'session' def upgrade(): - """Apply add session table to db""" + """Apply Create a ``session`` table to store web session data""" op.create_table( TABLE_NAME, sa.Column('id', sa.Integer()), @@ -50,5 +51,5 @@ def upgrade(): def downgrade(): - """Unapply add session table to db""" + """Unapply Create a ``session`` table to store web session data""" op.drop_table(TABLE_NAME) diff --git a/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py index ff02efbf9f1a0..a1eb00f14fb52 100644 --- a/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py +++ b/airflow/migrations/versions/c8ffec048a3b_add_fields_to_dag.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add fields to dag +"""Add ``description`` and ``default_view`` column to ``dag`` table Revision ID: c8ffec048a3b Revises: 41f5f12752f8 @@ -32,6 +32,7 @@ down_revision = '41f5f12752f8' branch_labels = None depends_on = None +airflow_version = '1.10.3' def upgrade(): diff --git a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py index b4078e19606f9..a2a89fe690019 100644 --- a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py +++ b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""add max tries column to task instance +"""Add ``max_tries`` column to ``task_instance`` Revision ID: cc1e65623dc7 Revises: 127d2bf2dfa7 @@ -37,6 +37,7 @@ down_revision = '127d2bf2dfa7' branch_labels = None depends_on = None +airflow_version = '1.8.2' Base = declarative_base() BATCH_SIZE = 5000 diff --git a/airflow/migrations/versions/ccde3e26fe78_add_index_on_state_dag_id_for_queued_.py b/airflow/migrations/versions/ccde3e26fe78_add_index_on_state_dag_id_for_queued_.py index 6a1cbe65329da..f9e55fce83550 100644 --- a/airflow/migrations/versions/ccde3e26fe78_add_index_on_state_dag_id_for_queued_.py +++ b/airflow/migrations/versions/ccde3e26fe78_add_index_on_state_dag_id_for_queued_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add index on state, dag_id for queued dagrun +"""Add index on state, dag_id for queued ``dagrun`` Revision ID: ccde3e26fe78 Revises: 092435bf5d12 @@ -32,10 +32,11 @@ down_revision = '092435bf5d12' branch_labels = None depends_on = None +airflow_version = '2.1.4' def upgrade(): - """Apply Add index on state, dag_id for queued dagrun""" + """Apply Add index on state, dag_id for queued ``dagrun``""" with op.batch_alter_table('dag_run') as batch_op: batch_op.create_index( 'idx_dag_run_queued_dags', @@ -47,6 +48,6 @@ def upgrade(): def downgrade(): - """Unapply Add index on state, dag_id for queued dagrun""" + """Unapply Add index on state, dag_id for queued ``dagrun``""" with op.batch_alter_table('dag_run') as batch_op: batch_op.drop_index('idx_dag_run_queued_dags') diff --git a/airflow/migrations/versions/cf5dc11e79ad_drop_user_and_chart.py b/airflow/migrations/versions/cf5dc11e79ad_drop_user_and_chart.py index e7252520be2e8..db51a32be210d 100644 --- a/airflow/migrations/versions/cf5dc11e79ad_drop_user_and_chart.py +++ b/airflow/migrations/versions/cf5dc11e79ad_drop_user_and_chart.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""drop_user_and_chart +"""Drop ``user`` and ``chart`` table Revision ID: cf5dc11e79ad Revises: 03afc6b6f902 @@ -32,6 +32,7 @@ down_revision = '03afc6b6f902' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py index ece0234ee55a9..41450aec0f9c2 100644 --- a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py +++ b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py @@ -31,6 +31,7 @@ down_revision = '947454bf1dff' branch_labels = None depends_on = None +airflow_version = '1.8.2' def upgrade(): diff --git a/airflow/migrations/versions/d38e04c12aa2_add_serialized_dag_table.py b/airflow/migrations/versions/d38e04c12aa2_add_serialized_dag_table.py index 9b96a56db3ecc..38179f2a7033a 100644 --- a/airflow/migrations/versions/d38e04c12aa2_add_serialized_dag_table.py +++ b/airflow/migrations/versions/d38e04c12aa2_add_serialized_dag_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add serialized_dag table +"""Add ``serialized_dag`` table Revision ID: d38e04c12aa2 Revises: 6e96a59344a4 @@ -34,6 +34,7 @@ down_revision = '6e96a59344a4' branch_labels = None depends_on = None +airflow_version = '1.10.7' def upgrade(): diff --git a/airflow/migrations/versions/da3f683c3a5a_add_dag_hash_column_to_serialized_dag_.py b/airflow/migrations/versions/da3f683c3a5a_add_dag_hash_column_to_serialized_dag_.py index 8d4723cc8d9f8..5113e2f28cec0 100644 --- a/airflow/migrations/versions/da3f683c3a5a_add_dag_hash_column_to_serialized_dag_.py +++ b/airflow/migrations/versions/da3f683c3a5a_add_dag_hash_column_to_serialized_dag_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add dag_hash Column to serialized_dag table +"""Add ``dag_hash`` Column to ``serialized_dag`` table Revision ID: da3f683c3a5a Revises: a66efa278eea @@ -32,10 +32,11 @@ down_revision = 'a66efa278eea' branch_labels = None depends_on = None +airflow_version = '1.10.12' def upgrade(): - """Apply Add dag_hash Column to serialized_dag table""" + """Apply Add ``dag_hash`` Column to ``serialized_dag`` table""" op.add_column( 'serialized_dag', sa.Column('dag_hash', sa.String(32), nullable=False, server_default='Hash not calculated yet'), @@ -43,5 +44,5 @@ def upgrade(): def downgrade(): - """Unapply Add dag_hash Column to serialized_dag table""" + """Unapply Add ``dag_hash`` Column to ``serialized_dag`` table""" op.drop_column('serialized_dag', 'dag_hash') diff --git a/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py b/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py index 560b763963dd6..76fcfd375b9e2 100644 --- a/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py +++ b/airflow/migrations/versions/dd25f486b8ea_add_idx_log_dag.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""add idx_log_dag +"""Add index on ``log`` table Revision ID: dd25f486b8ea Revises: 9635ae0956e7 @@ -29,6 +29,7 @@ down_revision = '9635ae0956e7' branch_labels = None depends_on = None +airflow_version = '1.10.2' def upgrade(): diff --git a/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py index 776527e16633b..42e0a5db92882 100644 --- a/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py +++ b/airflow/migrations/versions/dd4ecb8fbee3_add_schedule_interval_to_dag.py @@ -32,6 +32,7 @@ down_revision = 'c8ffec048a3b' branch_labels = None depends_on = None +airflow_version = '1.10.3' def upgrade(): diff --git a/airflow/migrations/versions/e165e7455d70_add_description_field_to_variable.py b/airflow/migrations/versions/e165e7455d70_add_description_field_to_variable.py index 7e1185971a236..761a718f32237 100644 --- a/airflow/migrations/versions/e165e7455d70_add_description_field_to_variable.py +++ b/airflow/migrations/versions/e165e7455d70_add_description_field_to_variable.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add description field to variable +"""Add description field to ``Variable`` model Revision ID: e165e7455d70 Revises: 90d1635d7b86 @@ -32,15 +32,16 @@ down_revision = '90d1635d7b86' branch_labels = None depends_on = None +airflow_version = '2.1.0' def upgrade(): - """Apply add description field to variable""" + """Apply Add description field to ``Variable`` model""" with op.batch_alter_table('variable', schema=None) as batch_op: batch_op.add_column(sa.Column('description', sa.Text(), nullable=True)) def downgrade(): - """Unapply add description field to variable""" + """Unapply Add description field to ``Variable`` model""" with op.batch_alter_table('variable', schema=None) as batch_op: batch_op.drop_column('description') diff --git a/airflow/migrations/versions/e1a11ece99cc_add_external_executor_id_to_ti.py b/airflow/migrations/versions/e1a11ece99cc_add_external_executor_id_to_ti.py index 8662ee418949e..a9b6030caff8f 100644 --- a/airflow/migrations/versions/e1a11ece99cc_add_external_executor_id_to_ti.py +++ b/airflow/migrations/versions/e1a11ece99cc_add_external_executor_id_to_ti.py @@ -32,6 +32,7 @@ down_revision = 'b247b1e3d1ed' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/e38be357a868_update_schema_for_smart_sensor.py b/airflow/migrations/versions/e38be357a868_update_schema_for_smart_sensor.py index 213299a2d0c97..aae2113a52919 100644 --- a/airflow/migrations/versions/e38be357a868_update_schema_for_smart_sensor.py +++ b/airflow/migrations/versions/e38be357a868_update_schema_for_smart_sensor.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add sensor_instance table +"""Add ``sensor_instance`` table Revision ID: e38be357a868 Revises: 8d48763f6d53 @@ -35,6 +35,7 @@ down_revision = '8d48763f6d53' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/e3a246e0dc1_current_schema.py b/airflow/migrations/versions/e3a246e0dc1_current_schema.py index 49e73591887bf..b4edbd627405d 100644 --- a/airflow/migrations/versions/e3a246e0dc1_current_schema.py +++ b/airflow/migrations/versions/e3a246e0dc1_current_schema.py @@ -36,6 +36,7 @@ down_revision = None branch_labels = None depends_on = None +airflow_version = '1.5.0' def upgrade(): diff --git a/airflow/migrations/versions/e655c0453f75_add_taskmap_and_map_id_on_taskinstance.py b/airflow/migrations/versions/e655c0453f75_add_taskmap_and_map_id_on_taskinstance.py index f7aca8250a262..acd4b81e19a52 100644 --- a/airflow/migrations/versions/e655c0453f75_add_taskmap_and_map_id_on_taskinstance.py +++ b/airflow/migrations/versions/e655c0453f75_add_taskmap_and_map_id_on_taskinstance.py @@ -16,7 +16,8 @@ # specific language governing permissions and limitations # under the License. -"""Add TaskMap and map_index on TaskInstance. +"""Add ``map_index`` column to TaskInstance to identify task-mapping, +and a ``task_map`` table to track mapping values from XCom. Revision ID: e655c0453f75 Revises: f9da662e7089 @@ -34,10 +35,14 @@ down_revision = "f9da662e7089" branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): - """Add TaskMap and map_index on TaskInstance.""" + """ + Add ``map_index`` column to TaskInstance to identify task-mapping, + and a ``task_map`` table to track mapping values from XCom. + """ # We need to first remove constraints on task_reschedule since they depend on task_instance. with op.batch_alter_table("task_reschedule") as batch_op: batch_op.drop_constraint("task_reschedule_ti_fkey", "foreignkey") diff --git a/airflow/migrations/versions/e9304a3141f0_make_xcom_pkey_columns_non_nullable.py b/airflow/migrations/versions/e9304a3141f0_make_xcom_pkey_columns_non_nullable.py index 7aee0f26d1d2d..7873c57b09ff9 100644 --- a/airflow/migrations/versions/e9304a3141f0_make_xcom_pkey_columns_non_nullable.py +++ b/airflow/migrations/versions/e9304a3141f0_make_xcom_pkey_columns_non_nullable.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""make xcom pkey columns non-nullable +"""Make XCom primary key columns non-nullable Revision ID: e9304a3141f0 Revises: 83f031fd9f1c @@ -32,10 +32,11 @@ down_revision = '83f031fd9f1c' branch_labels = None depends_on = None +airflow_version = '2.2.0' def upgrade(): - """Apply make xcom pkey columns non-nullable""" + """Apply Make XCom primary key columns non-nullable""" conn = op.get_bind() with op.batch_alter_table('xcom') as bop: bop.alter_column("key", type_=StringID(length=512), nullable=False) @@ -45,7 +46,7 @@ def upgrade(): def downgrade(): - """Unapply make xcom pkey columns non-nullable""" + """Unapply Make XCom primary key columns non-nullable""" conn = op.get_bind() with op.batch_alter_table('xcom') as bop: if conn.dialect.name == 'mssql': diff --git a/airflow/migrations/versions/e959f08ac86c_change_field_in_dagcode_to_mediumtext_.py b/airflow/migrations/versions/e959f08ac86c_change_field_in_dagcode_to_mediumtext_.py index 559aa467ecff9..1faa9f336e20b 100644 --- a/airflow/migrations/versions/e959f08ac86c_change_field_in_dagcode_to_mediumtext_.py +++ b/airflow/migrations/versions/e959f08ac86c_change_field_in_dagcode_to_mediumtext_.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Change field in DagCode to MEDIUMTEXT for MySql +"""Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql Revision ID: e959f08ac86c Revises: 64a7d6477aae @@ -31,6 +31,7 @@ down_revision = '64a7d6477aae' branch_labels = None depends_on = None +airflow_version = '2.0.0' def upgrade(): diff --git a/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py b/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py index 77424ec76505c..765d43762e1aa 100644 --- a/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py +++ b/airflow/migrations/versions/f23433877c24_fix_mysql_not_null_constraint.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""fix mysql not null constraint +"""Fix MySQL not null constraint Revision ID: f23433877c24 Revises: 05f30312d566 @@ -30,6 +30,7 @@ down_revision = '05f30312d566' branch_labels = None depends_on = None +airflow_version = '1.10.0' def upgrade(): diff --git a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py index 644c138eba9ab..9cb37d2a41cbc 100644 --- a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py +++ b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""add dag_stats table +"""Add ``dag_stats`` table Revision ID: f2ca10b85618 Revises: 64de9cddf6c9 @@ -33,6 +33,7 @@ down_revision = '64de9cddf6c9' branch_labels = None depends_on = None +airflow_version = '1.7.1.3' def upgrade(): diff --git a/airflow/migrations/versions/f9da662e7089_add_task_log_filename_template_model.py b/airflow/migrations/versions/f9da662e7089_add_task_log_filename_template_model.py index 1a61c4bb0fc9a..51d2a2b63fe7a 100644 --- a/airflow/migrations/versions/f9da662e7089_add_task_log_filename_template_model.py +++ b/airflow/migrations/versions/f9da662e7089_add_task_log_filename_template_model.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Add model for task log filename template. +"""Add ``LogTemplate`` table to track changes to config values ``log_filename_template`` Revision ID: f9da662e7089 Revises: 786e3737b18f @@ -33,6 +33,7 @@ down_revision = "786e3737b18f" branch_labels = None depends_on = None +airflow_version = '2.3.0' def upgrade(): diff --git a/airflow/migrations/versions/fe461863935f_increase_length_for_connection_password.py b/airflow/migrations/versions/fe461863935f_increase_length_for_connection_password.py index 0e97630248a9b..955e1a668623c 100644 --- a/airflow/migrations/versions/fe461863935f_increase_length_for_connection_password.py +++ b/airflow/migrations/versions/fe461863935f_increase_length_for_connection_password.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""increase_length_for_connection_password +"""Increase length for connection password Revision ID: fe461863935f Revises: 08364691d074 @@ -32,10 +32,11 @@ down_revision = '08364691d074' branch_labels = None depends_on = None +airflow_version = '1.10.7' def upgrade(): - """Apply increase_length_for_connection_password""" + """Apply Increase length for connection password""" with op.batch_alter_table('connection', schema=None) as batch_op: batch_op.alter_column( 'password', @@ -46,7 +47,7 @@ def upgrade(): def downgrade(): - """Unapply increase_length_for_connection_password""" + """Unapply Increase length for connection password""" with op.batch_alter_table('connection', schema=None) as batch_op: batch_op.alter_column( 'password', diff --git a/breeze-complete b/breeze-complete index 203a3131ff76c..ef4e95a4143e8 100644 --- a/breeze-complete +++ b/breeze-complete @@ -119,6 +119,7 @@ lint-dockerfile lint-openapi markdownlint mermaid +migration-reference mixed-line-ending mypy mypy-helm diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 6e7c188f9c4bd..4786fce4a83ad 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -63,6 +63,7 @@ 'lint-dockerfile', 'lint-openapi', 'markdownlint', + 'migration-reference', 'mixed-line-ending', 'mypy', 'no-providers-in-core-examples', @@ -96,7 +97,6 @@ 'update-supported-versions', 'update-version', 'vendor-k8s-json-schema', - 'verify-db-migrations-documented', 'www-lint', 'yamllint', 'yesqa', diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 8526930fd1874..c3898af85547b 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -20,222 +20,240 @@ Reference for Database Migrations Here's the list of all the Database Migrations that are executed via when you run ``airflow db upgrade``: -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| Revision ID | Revises ID | Airflow Version | Description | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``c306b5b5ae4a`` (head) | ``a3bcd0914482`` | ``2.3.0`` | Switch ``XCom`` primary key from ``execution_date`` to ``run_id``. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | Add ``data_compressed`` to serialized_dag and make data column nullable. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e655c0453f75`` | ``587bdf053233`` | ``2.3.0`` | Add ``map_index`` column to TaskInstance to identify task-mapping, and a ``task_map`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add ``map_index`` column to TaskInstance to identify task-mapping, and a ``task_map`` | -| | | | table to track mapping values from XCom. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``f9da662e7089`` | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values ``log_filename_template`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``786e3737b18f`` | ``5e3ec427fdd3`` | ``2.3.0`` | Add ``timetable_description`` column to DagModel for UI. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``5e3ec427fdd3`` | ``587bdf053233`` | ``2.3.0`` | Increase length of email and username in ``ab_user`` and ``ab_register_user`` table | -| | | | to ``256`` characters | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``587bdf053233`` | ``c381b21cb7e4`` | ``2.2.4`` | Add index for ``dag_id`` column in ``job`` table. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``7b2661a43ba3`` | ``142555e44c17`` | ``2.2.0`` | Change ``TaskInstance`` and ``TaskReschedule`` tables from execution_date to run_id. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``142555e44c17`` | ``54bebd308c5f`` | ``2.2.0`` | Add ``data_interval_start`` and ``data_interval_end`` to ``DagRun`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``54bebd308c5f`` | ``30867afad44a`` | ``2.2.0`` | Adds ``trigger`` table and deferrable operator columns to task instance | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``30867afad44a`` | ``e9304a3141f0`` | ``2.2.0`` | Rename ``concurrency`` column in ``dag`` table to ``max_active_tasks`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``a13f7613ad25`` | ``e165e7455d70`` | ``2.1.0`` | Resource based permissions for default ``Flask-AppBuilder`` views | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``90d1635d7b86`` | ``2e42bb497a22`` | ``2.1.0`` | Increase maximum length of pool name in ``task_instance`` table to ``256`` characters | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``2e42bb497a22`` | ``8646922c8a04`` | ``2.0.2`` | Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``449b4072c2da`` | ``82b7c48c147f`` | ``2.0.2`` | Increase size of ``connection.extra`` field to handle multiple RSA keys | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``82b7c48c147f`` | ``e959f08ac86c`` | ``2.0.1`` | Remove ``can_read`` permission on config resource for ``User`` and ``Viewer`` role | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``2c6edca13270`` | ``849da589634d`` | ``2.0.0`` | Resource based permissions. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``849da589634d`` | ``45ba3f1493b9`` | ``2.0.0`` | Prefix DAG permissions. | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``03afc6b6f902`` | ``92c57b58940d`` | ``1.10.13`` | Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``a66efa278eea`` | ``952da73b5eff`` | ``1.10.11`` | Add Precision to ``execution_date`` in ``RenderedTaskInstanceFields`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``08364691d074`` (mergepoint) | ``a56c9515abdc``,| ``1.10.7`` | Straighten out the migrations | -| | ``004c1210f153``,| | | -| | ``74effc47d867``,| | | -| | ``b3b105409875`` | | | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``74effc47d867`` | ``6e96a59344a4`` | ``1.10.5`` | Change ``datetime`` to ``datetime2(6)`` on MSSQL tables | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``6e96a59344a4`` (branchpoint) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``939bb1e647c8`` (branchpoint) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``c8ffec048a3b`` (branchpoint) | ``41f5f12752f8`` | ``1.10.3`` | Add ``description`` and ``default_view`` column to ``dag`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``03bc53e68815`` (mergepoint) | ``0a2a5b66e19d``,| ``1.10.2`` | Merge migrations Heads | -| | ``bf00311e1990`` | | | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``9635ae0956e7`` (branchpoint) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``05f30312d566`` (mergepoint) | ``0e2a74e0fc9f`` | ``1.10.0`` | Merge migrations Heads | -| | ``86770d1215c0`` | | | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``d2ae31099d61`` (branchpoint) | ``947454bf1dff`` | ``1.8.2`` | Increase text size for MySQL (not relevant for other DBs' text types) | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bba5a7cfc896`` | ``bbc73705a13e`` | ``1.7.0`` | Add a column to track the encryption state of the 'Extra' field in connection | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT`` | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``338e90f54d61`` | ``13eb55f81627`` | ``1.5.0`` | Add ``operator`` and ``queued_dttm`` to ``task_instance`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ -| ``e3a246e0dc1`` | | ``1.5.0`` | Create initial schema | -+--------------------------------+------------------+-----------------+---------------------------------------------------------------------------------------+ + .. This table is automatically updated by pre-commit by ``scripts/ci/pre_commit/pre_commit_migration_reference.py`` + .. All table elements are scraped from migration files + .. Beginning of auto-generated table + ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``c306b5b5ae4a`` (head) | ``a3bcd0914482`` | ``2.3.0`` | Switch XCom table to use ``run_id``. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | add data_compressed to serialized_dag | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add ``map_index`` column to TaskInstance to identify task- | +| | | | mapping, and a ``task_map`` table to track mapping values | +| | | | from XCom. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``f9da662e7089`` | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values | +| | | | ``log_filename_template`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``786e3737b18f`` | ``5e3ec427fdd3`` | ``2.3.0`` | Add ``timetable_description`` column to DagModel for UI. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``5e3ec427fdd3`` | ``587bdf053233`` | ``2.3.0`` | Increase length of email and username in ``ab_user`` and | +| | | | ``ab_register_user`` table to ``256`` characters | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``587bdf053233`` | ``c381b21cb7e4`` | ``2.3.0`` | Add index for ``dag_id`` column in ``job`` table. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``7b2661a43ba3`` | ``142555e44c17`` | ``2.2.0`` | Change ``TaskInstance`` and ``TaskReschedule`` tables from | +| | | | execution_date to run_id. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``142555e44c17`` | ``54bebd308c5f`` | ``2.2.0`` | Add data_interval_[start|end] to DagModel and DagRun. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``54bebd308c5f`` | ``30867afad44a`` | ``2.2.0`` | Adds ``trigger`` table and deferrable operator columns to | +| | | | task instance | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``30867afad44a`` | ``e9304a3141f0`` | ``2.2.0`` | Rename ``concurrency`` column in ``dag`` table to`` | +| | | | max_active_tasks`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``a13f7613ad25`` | ``e165e7455d70`` | ``2.1.0`` | Resource based permissions for default ``Flask-AppBuilder`` | +| | | | views | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``90d1635d7b86`` | ``2e42bb497a22`` | ``2.1.0`` | Increase maximum length of pool name in ``task_instance`` | +| | | | table to ``256`` characters | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``2e42bb497a22`` | ``8646922c8a04`` | ``2.0.2`` | Rename ``last_scheduler_run`` column in ``DAG`` table to | +| | | | ``last_parsed_time`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``449b4072c2da`` | ``82b7c48c147f`` | ``2.0.2`` | Increase size of ``connection.extra`` field to handle | +| | | | multiple RSA keys | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``82b7c48c147f`` | ``e959f08ac86c`` | ``2.0.1`` | Remove ``can_read`` permission on config resource for | +| | | | ``User`` and ``Viewer`` role | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``2c6edca13270`` | ``849da589634d`` | ``2.0.0`` | Resource based permissions. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``849da589634d`` | ``45ba3f1493b9`` | ``2.0.0`` | Prefix DAG permissions. | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``03afc6b6f902`` | ``92c57b58940d`` | ``1.10.13`` | Increase length of ``Flask-AppBuilder`` | +| | | | ``ab_view_menu.name`` column | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``a66efa278eea`` | ``952da73b5eff`` | ``1.10.11`` | Add Precision to ``execution_date`` in | +| | | | ``RenderedTaskInstanceFields`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``08364691d074`` (merge_point) | ``a56c9515abdc``, | ``1.10.7`` | Straighten out the migrations | +| | ``004c1210f153``, | | | +| | ``74effc47d867``, | | | +| | ``b3b105409875`` | | | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``74effc47d867`` | ``6e96a59344a4`` | ``1.10.5`` | change datetime to datetime2(6) on MSSQL tables | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``6e96a59344a4`` (branch_point) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``939bb1e647c8`` (branch_point) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``c8ffec048a3b`` (branch_point) | ``41f5f12752f8`` | ``1.10.3`` | Add ``description`` and ``default_view`` column to ``dag`` | +| | | | table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``03bc53e68815`` (merge_point) | ``0a2a5b66e19d``, | ``1.10.2`` | Merge migrations Heads | +| | ``bf00311e1990`` | | | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``9635ae0956e7`` (branch_point) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``05f30312d566`` (merge_point) | ``86770d1215c0``, | ``1.10.0`` | Merge migrations Heads | +| | ``0e2a74e0fc9f`` | | | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``d2ae31099d61`` (branch_point) | ``947454bf1dff`` | ``1.8.2`` | Increase text size for MySQL (not relevant for other DBs' | +| | | | text types) | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bba5a7cfc896`` | ``bbc73705a13e`` | ``1.7.0`` | Add a column to track the encryption state of the 'Extra' | +| | | | field in connection | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT`` | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``338e90f54d61`` | ``13eb55f81627`` | ``1.5.0`` | Add ``operator`` and ``queued_dttm`` to ``task_instance`` | +| | | | table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ +| ``e3a246e0dc1`` (base) | ``None`` | ``1.5.0`` | current schema | ++---------------------------------+-------------------+-------------+--------------------------------------------------------------+ + + .. End of auto-generated table .. spelling:: branchpoint diff --git a/scripts/ci/pre_commit/pre_commit_migration_documented.py b/scripts/ci/pre_commit/pre_commit_migration_documented.py deleted file mode 100755 index 4508458afa8fe..0000000000000 --- a/scripts/ci/pre_commit/pre_commit_migration_documented.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Module to verify that migrations added have been documented in -docs/apache-airflow/migrations-ref.rst -""" -import glob -import os -import re -import sys - - -def check_migration_is_documented(migration_file, doc_path): - - with open(migration_file) as m_file: - file_contents = m_file.read() - match = re.search(r"revision\s*=\s*['\"](\w*)['\"]", file_contents) - if match: - revision_id = match.group(1) - with open(doc_path) as doc_file: - doc_file_contents = doc_file.read() - if revision_id not in doc_file_contents: - return False, revision_id - return True, None - - -if __name__ == '__main__': - project_root = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) - - airflow_migrations_dir = os.path.join(project_root, "airflow/migrations/versions") - - doc_path = os.path.abspath(os.path.join(project_root, "docs/apache-airflow/migrations-ref.rst")) - assert os.path.isfile(doc_path) - - airflow_migrations_dir = os.path.abspath(airflow_migrations_dir) - migration_files = [f for f in glob.glob(f"{airflow_migrations_dir}/*.py")] - - undocumented_migrations = [] - - for migration_file in migration_files: - is_documented, rev = check_migration_is_documented(migration_file=migration_file, doc_path=doc_path) - if not is_documented: - undocumented_migrations.append((migration_file, rev)) - - if undocumented_migrations: - print() - print(f"DB Migrations in following files have not been documented in '{doc_path}'") - print() - for undocumented_migration, rev in undocumented_migrations: - print(f"\t- {undocumented_migration} (Revision ID: '{rev}')") - sys.exit(1) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py new file mode 100755 index 0000000000000..b8a473044f8b8 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Module to update db migration information in Airflow +""" +import re +from pathlib import Path +from textwrap import wrap +from typing import TYPE_CHECKING, Iterable + +from alembic.script import ScriptDirectory +from tabulate import tabulate + +from airflow.utils.db import _get_alembic_config +from airflow.version import version as _airflow_version + +if TYPE_CHECKING: + from alembic.script import Script + +airflow_version = re.match(r'(\d+\.\d+\.\d+).*', _airflow_version).group(1) # type: ignore +project_root = Path(__file__).parent.parent.parent.parent + + +def replace_text_between(file: Path, start: str, end: str, replacement_text: str): + original_text = file.read_text() + leading_text = original_text.split(start)[0] + trailing_text = original_text.split(end)[1] + file.write_text(leading_text + start + replacement_text + end + trailing_text) + + +def wrap_backticks(val): + def _wrap_backticks(x): + return f"``{x}``" + + return ',\n'.join(map(_wrap_backticks, val)) if isinstance(val, (tuple, list)) else _wrap_backticks(val) + + +def update_doc(file, data): + replace_text_between( + file=file, + start=" .. Beginning of auto-generated table\n", + end=" .. End of auto-generated table\n", + replacement_text="\n" + + tabulate( + tabular_data=data, + tablefmt='grid', + stralign="left", + disable_numparse=True, + ) + + "\n\n", + ) + + +def has_version(content): + return re.search(r'^airflow_version\s*=.*', content, flags=re.MULTILINE) is not None + + +def insert_version(old_content, file): + new_content = re.sub( + r'(^depends_on.*)', + lambda x: f"{x.group(1)}\nairflow_version = '{airflow_version}'", + old_content, + flags=re.MULTILINE, + ) + file.write_text(new_content) + + +def revision_suffix(rev: "Script"): + if rev.is_head: + return ' (head)' + if rev.is_base: + return ' (base)' + if rev.is_merge_point: + return ' (merge_point)' + if rev.is_branch_point: + return ' (branch_point)' + return '' + + +def ensure_airflow_version(revisions: Iterable["Script"]): + for rev in revisions: + file = Path(rev.module.__file__) + content = file.read_text() + if not has_version(content): + insert_version(content, file) + + +def get_revisions() -> Iterable["Script"]: + config = _get_alembic_config() + script = ScriptDirectory.from_config(config) + yield from script.walk_revisions() + + +def update_docs(revisions: Iterable["Script"]): + doc_data = [] + for rev in revisions: + doc_data.append( + dict( + revision=wrap_backticks(rev.revision) + revision_suffix(rev), + down_revision=wrap_backticks(rev.down_revision), + version=wrap_backticks(rev.module.airflow_version), # type: ignore + description='\n'.join(wrap(rev.doc, width=60)), + ) + ) + + update_doc( + file=project_root / "docs" / "apache-airflow" / "migrations-ref.rst", + data=doc_data, + ) + + +if __name__ == '__main__': + ensure_airflow_version(revisions=get_revisions()) + + # if `ensure_airflow_version` modified any migrations, we'll need to reload + revisions = list(get_revisions()) + + update_docs(revisions) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.sh b/scripts/ci/pre_commit/pre_commit_migration_reference.sh new file mode 100755 index 0000000000000..59a7989218aa7 --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +export PYTHON_MAJOR_MINOR_VERSION="3.7" +export PRINT_INFO_FROM_SCRIPTS="false" + +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +function migration_reference() { + if [[ "${#@}" == "0" ]]; then + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/usr/local/bin/dumb-init" \ + "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/in_container/run_migration_reference.sh" + else + docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/usr/local/bin/dumb-init" \ + "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/in_container/run_migration_reference.sh" "${@}" + fi +} + +build_images::prepare_ci_build + +build_images::rebuild_ci_image_if_confirmed_for_pre_commit + +migration_reference "$@" diff --git a/scripts/in_container/run_migration_reference.sh b/scripts/in_container/run_migration_reference.sh new file mode 100755 index 0000000000000..97b188cdf87b3 --- /dev/null +++ b/scripts/in_container/run_migration_reference.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/in_container/_in_container_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" +python scripts/ci/pre_commit/pre_commit_migration_reference.py "$@"