Enhance pre-commit hook to automate migrations ref doc and revision metadata

.pre-commit-config.yaml

@@ -488,13 +488,6 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['pyyaml']
-      - id: verify-db-migrations-documented
-        name: Verify that DB Migrations have been documented
-        language: python
-        entry: ./scripts/ci/pre_commit/pre_commit_migration_documented.py
-        files: ^airflow/migrations/versions/.*\.(py)$
-        pass_filenames: false
-        require_serial: true
       - id: boring-cyborg
         name: Checks for Boring Cyborg configuration consistency
         language: python
@@ -738,4 +731,11 @@ repos:
         files: \.py$
         pass_filenames: true
         exclude: ^airflow/_vendor/
+      - id: migration-reference
+        name: Update migration ref doc
+        language: system
+        entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.sh
+        files: \.py$
+        pass_filenames: false
+        files: ^airflow/migrations/versions/.*\.(py)$|^docs/apache-airflow/migrations-ref.rst$
         ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE

BREEZE.rst

@@ -2296,16 +2296,17 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  debug-statements detect-private-key docstring-params doctoc dont-use-safe-filter
                  end-of-file-fixer fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint
                  identity incorrect-use-of-LoggingMixin insert-license isort json-schema
-                 language-matters lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending
-                 mypy mypy-helm no-providers-in-core-examples no-relative-imports
-                 persist-credentials-disabled pre-commit-descriptions pre-commit-hook-names
-                 pretty-format-json provide-create-sessions providers-changelogs providers-init-file
-                 providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn
-                 pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages
-                 shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace
-                 ui-lint update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
-                 update-supported-versions update-versions vendor-k8s-json-schema
-                 verify-db-migrations-documented version-sync www-lint yamllint yesqa
+                 language-matters lint-dockerfile lint-openapi markdownlint mermaid
+                 migration-reference mixed-line-ending mypy mypy-helm no-providers-in-core-examples
+                 no-relative-imports persist-credentials-disabled pre-commit-descriptions
+                 pre-commit-hook-names pretty-format-json provide-create-sessions
+                 providers-changelogs providers-init-file providers-subpackages-init-file
+                 provider-yamls pydevd pydocstyle python-no-log-warn pyupgrade restrict-start_date
+                 rst-backticks setup-order setup-extra-packages shellcheck sort-in-the-wild
+                 sort-spelling-wordlist stylelint trailing-whitespace ui-lint update-breeze-file
+                 update-extras update-local-yml-file update-setup-cfg-file update-supported-versions
+                 update-versions vendor-k8s-json-schema verify-db-migrations-documented version-sync
+                 www-lint yamllint yesqa
 
         You can pass extra arguments including options to the pre-commit framework as
         <EXTRA_ARGS> passed after --. For example:

STATIC_CODE_CHECKS.rst

@@ -216,6 +216,8 @@ require Breeze Docker images to be installed locally.
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``markdownlint``                       Lints Markdown files
 ------------------------------------ ---------------------------------------------------------------- ------------
+``migration-reference``                Update migration reference doc and revision metadata
+------------------------------------ ---------------------------------------------------------------- ------------
 ``mixed-line-ending``                  Detects if mixed line ending is used (\r vs. \r\n)
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``mypy``                               Runs mypy                                                            *

airflow/migrations/versions/004c1210f153_increase_queue_name_size_limit.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""increase queue name size limit
+"""Increase queue name size limit
 
 Revision ID: 004c1210f153
 Revises: 939bb1e647c8
@@ -32,6 +32,7 @@
 down_revision = '939bb1e647c8'
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.4'
 
 
 def upgrade():

airflow/migrations/versions/03afc6b6f902_increase_length_of_fab_ab_view_menu_.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Increase length of FAB ab_view_menu.name column
+"""Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column
 
 Revision ID: 03afc6b6f902
 Revises: 92c57b58940d
@@ -35,10 +35,11 @@
 down_revision = '92c57b58940d'
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.13'
 
 
 def upgrade():
-    """Apply Increase length of FAB ab_view_menu.name column"""
+    """Apply Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column"""
     conn = op.get_bind()
     inspector = Inspector.from_engine(conn)
     tables = inspector.get_table_names()
@@ -69,7 +70,7 @@ def upgrade():
 
 
 def downgrade():
-    """Unapply Increase length of FAB ab_view_menu.name column"""
+    """Unapply Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column"""
     conn = op.get_bind()
     inspector = Inspector.from_engine(conn)
     tables = inspector.get_table_names()

airflow/migrations/versions/03bc53e68815_add_sm_dag_index.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""merge_heads_2
+"""Merge migrations Heads
 
 Revision ID: 03bc53e68815
 Revises: 0a2a5b66e19d, bf00311e1990
@@ -30,6 +30,7 @@
 down_revision = ('0a2a5b66e19d', 'bf00311e1990')
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.2'
 
 
 def upgrade():

airflow/migrations/versions/05f30312d566_merge_heads.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""merge heads
+"""Merge migrations Heads
 
 Revision ID: 05f30312d566
 Revises: 86770d1215c0, 0e2a74e0fc9f
@@ -28,6 +28,7 @@
 down_revision = ('86770d1215c0', '0e2a74e0fc9f')
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.0'
 
 
 def upgrade():

airflow/migrations/versions/08364691d074_merge_heads.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Merge the four heads back together
+"""Straighten out the migrations
 
 Revision ID: 08364691d074
 Revises: a56c9515abdc, 004c1210f153, 74effc47d867, b3b105409875
@@ -29,6 +29,7 @@
 down_revision = ('a56c9515abdc', '004c1210f153', '74effc47d867', 'b3b105409875')
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.7'
 
 
 def upgrade():

airflow/migrations/versions/092435bf5d12_add_max_active_runs_column_to_dagmodel_.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Add max_active_runs column to dagmodel table
+"""Add ``max_active_runs`` column to ``dag_model`` table
 
 Revision ID: 092435bf5d12
 Revises: 97cdd93827b8
@@ -33,10 +33,11 @@
 down_revision = '97cdd93827b8'
 branch_labels = None
 depends_on = None
+airflow_version = '2.1.4'
 
 
 def upgrade():
-    """Apply Add max_active_runs column to dagmodel table"""
+    """Apply Add ``max_active_runs`` column to ``dag_model`` table"""
     op.add_column('dag', sa.Column('max_active_runs', sa.Integer(), nullable=True))
     with op.batch_alter_table('dag_run', schema=None) as batch_op:
         # Add index to dag_run.dag_id and also add index to dag_run.state where state==running
@@ -51,7 +52,7 @@ def upgrade():
 
 
 def downgrade():
-    """Unapply Add max_active_runs column to dagmodel table"""
+    """Unapply Add ``max_active_runs`` column to ``dag_model`` table"""
     op.drop_column('dag', 'max_active_runs')
     with op.batch_alter_table('dag_run', schema=None) as batch_op:
         # Drop index to dag_run.dag_id and also drop index to dag_run.state where state==running

airflow/migrations/versions/0a2a5b66e19d_add_task_reschedule_table.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""add task_reschedule table
+"""Add ``task_reschedule`` table
 
 Revision ID: 0a2a5b66e19d
 Revises: 9635ae0956e7
@@ -32,6 +32,7 @@
 down_revision = '9635ae0956e7'
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.2'
 
 TABLE_NAME = 'task_reschedule'
 INDEX_NAME = 'idx_' + TABLE_NAME + '_dag_task_date'

airflow/migrations/versions/0e2a74e0fc9f_add_time_zone_awareness.py

@@ -32,6 +32,7 @@
 down_revision = "d2ae31099d61"
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.0'
 
 
 def upgrade():

airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""Add dag_id/state index on dag_run table
+"""Add ``dag_id``/``state`` index on ``dag_run`` table
 
 Revision ID: 127d2bf2dfa7
 Revises: 5e7d17757c7a
@@ -30,6 +30,7 @@
 down_revision = '5e7d17757c7a'
 branch_labels = None
 depends_on = None
+airflow_version = '1.7.1.3'
 
 
 def upgrade():

airflow/migrations/versions/13eb55f81627_for_compatibility.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""maintain history for compatibility with earlier migrations
+"""Maintain history for compatibility with earlier migrations
 
 Revision ID: 13eb55f81627
 Revises: 1507a7289a2f
@@ -29,6 +29,7 @@
 down_revision = '1507a7289a2f'
 branch_labels = None
 depends_on = None
+airflow_version = '1.5.0'
 
 
 def upgrade():

airflow/migrations/versions/142555e44c17_add_data_interval_start_end_to_dagmodel_and_dagrun.py

@@ -34,6 +34,7 @@
 down_revision = "54bebd308c5f"
 branch_labels = None
 depends_on = None
+airflow_version = '2.2.0'
 
 
 def upgrade():

airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""create is_encrypted
+"""Add ``is_encrypted`` column in ``connection`` table
 
 Revision ID: 1507a7289a2f
 Revises: e3a246e0dc1
@@ -32,6 +32,7 @@
 down_revision = 'e3a246e0dc1'
 branch_labels = None
 depends_on = None
+airflow_version = '1.5.0'
 
 connectionhelper = sa.Table(
     'connection', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True), sa.Column('is_encrypted')

airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""add is_encrypted column to variable table
+"""Add ``is_encrypted`` column to variable table
 
 Revision ID: 1968acfc09e3
 Revises: bba5a7cfc896
@@ -31,6 +31,7 @@
 down_revision = 'bba5a7cfc896'
 branch_labels = None
 depends_on = None
+airflow_version = '1.7.0'
 
 
 def upgrade():

airflow/migrations/versions/1b38cef5b76e_add_dagrun.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""add dagrun
+"""Add ``dag_run`` table
 
 Revision ID: 1b38cef5b76e
 Revises: 52d714495f0
@@ -34,6 +34,7 @@
 down_revision = '502898887f84'
 branch_labels = None
 depends_on = None
+airflow_version = '1.6.0'
 
 
 def upgrade():

airflow/migrations/versions/211e584da130_add_ti_state_index.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""add TI state index
+"""Add TI state index
 
 Revision ID: 211e584da130
 Revises: 2e82aab8ef20
@@ -30,6 +30,7 @@
 down_revision = '2e82aab8ef20'
 branch_labels = None
 depends_on = None
+airflow_version = '1.7.1.3'
 
 
 def upgrade():

airflow/migrations/versions/27c6a30d7c24_add_executor_config_to_task_instance.py

@@ -16,7 +16,7 @@
 # under the License.
 
 
-"""kubernetes_resource_checkpointing
+"""Add ``executor_config`` column to ``task_instance`` table
 
 Revision ID: 33ae817a1ff4
 Revises: 947454bf1dff
@@ -33,6 +33,7 @@
 down_revision = '33ae817a1ff4'
 branch_labels = None
 depends_on = None
+airflow_version = '1.10.0'
 
 TASK_INSTANCE_TABLE = "task_instance"
 NEW_COLUMN = "executor_config"

airflow/migrations/versions/2c6edca13270_resource_based_permissions.py

@@ -33,6 +33,7 @@
 down_revision = '849da589634d'
 branch_labels = None
 depends_on = None
+airflow_version = '2.0.0'
 
 
 mapping = {

airflow/migrations/versions/2e42bb497a22_rename_last_scheduler_run_column.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""rename last_scheduler_run column
+"""Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time``
 
 Revision ID: 2e42bb497a22
 Revises: 8646922c8a04
@@ -33,10 +33,11 @@
 down_revision = '8646922c8a04'
 branch_labels = None
 depends_on = None
+airflow_version = '2.0.2'
 
 
 def upgrade():
-    """Apply rename last_scheduler_run column"""
+    """Apply Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time``"""
     conn = op.get_bind()
     if conn.dialect.name == "mssql":
         with op.batch_alter_table('dag') as batch_op:
@@ -51,7 +52,7 @@ def upgrade():
 
 
 def downgrade():
-    """Unapply rename last_scheduler_run column"""
+    """Unapply Rename ``last_scheduler_run`` column in ``DAG`` table to ``last_parsed_time``"""
     conn = op.get_bind()
     if conn.dialect.name == "mssql":
         with op.batch_alter_table('dag') as batch_op: