From daa62db77036704d9b72cf5906bacbca56a63f28 Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Sat, 30 Dec 2023 10:55:14 -0500
Subject: [PATCH 1/6] address conflicts

---
 .../celery_executor.rst}                      |   9 +-
 .../celery_kubernetes_executor.rst}           |   4 +-
 .../img/run_task_on_celery_executor.png       | Bin
 .../apache-airflow-providers-celery/index.rst |   8 ++
 .../img/arch-diag-kubernetes.png              | Bin
 .../img/arch-diag-kubernetes2.png             | Bin
 .../img/k8s-failed-pod.png                    | Bin
 .../img/k8s-happy-path.png                    | Bin
 .../index.rst                                 |   8 ++
 .../kubernetes_executor.rst}                  |  10 +--
 .../local_kubernetes_executor.rst}            |   2 +-
 .../operators.rst                             |   2 +-
 .../dask_executor.rst                         |  53 ++++++++++++
 .../production-deployment.rst                 |   4 +-
 docs/apache-airflow/best-practices.rst        |   2 +-
 .../core-concepts/executor/debug.rst          |  81 ++++++++++--------
 .../core-concepts/executor/index.rst          |  16 ++--
 docs/apache-airflow/core-concepts/index.rst   |   1 +
 .../howto/docker-compose/index.rst            |   4 +-
 docs/apache-airflow/img/k8s-0-worker.jpeg     | Bin 90008 -> 0 bytes
 docs/apache-airflow/img/k8s-3-worker.jpeg     | Bin 129900 -> 0 bytes
 docs/apache-airflow/img/k8s-5-worker.jpeg     | Bin 162253 -> 0 bytes
 22 files changed, 137 insertions(+), 67 deletions(-)
 rename docs/{apache-airflow/core-concepts/executor/celery.rst => apache-airflow-providers-celery/celery_executor.rst} (97%)
 rename docs/{apache-airflow/core-concepts/executor/celery_kubernetes.rst => apache-airflow-providers-celery/celery_kubernetes_executor.rst} (95%)
 rename docs/{apache-airflow => apache-airflow-providers-celery}/img/run_task_on_celery_executor.png (100%)
 rename docs/{apache-airflow => apache-airflow-providers-cncf-kubernetes}/img/arch-diag-kubernetes.png (100%)
 rename docs/{apache-airflow => apache-airflow-providers-cncf-kubernetes}/img/arch-diag-kubernetes2.png (100%)
 rename docs/{apache-airflow => apache-airflow-providers-cncf-kubernetes}/img/k8s-failed-pod.png (100%)
 rename docs/{apache-airflow => apache-airflow-providers-cncf-kubernetes}/img/k8s-happy-path.png (100%)
 rename docs/{apache-airflow/core-concepts/executor/kubernetes.rst => apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst} (98%)
 rename docs/{apache-airflow/core-concepts/executor/local_kubernetes.rst => apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst} (97%)
 create mode 100644 docs/apache-airflow-providers-openai/dask_executor.rst
 delete mode 100644 docs/apache-airflow/img/k8s-0-worker.jpeg
 delete mode 100644 docs/apache-airflow/img/k8s-3-worker.jpeg
 delete mode 100644 docs/apache-airflow/img/k8s-5-worker.jpeg

diff --git a/docs/apache-airflow/core-concepts/executor/celery.rst b/docs/apache-airflow-providers-celery/celery_executor.rst
similarity index 97%
rename from docs/apache-airflow/core-concepts/executor/celery.rst
rename to docs/apache-airflow-providers-celery/celery_executor.rst
index 1113bda53068a..855746202202b 100644
--- a/docs/apache-airflow/core-concepts/executor/celery.rst
+++ b/docs/apache-airflow-providers-celery/celery_executor.rst
@@ -15,9 +15,6 @@
     specific language governing permissions and limitations
     under the License.
 
-
-.. _executor:CeleryExecutor:
-
 Celery Executor
 ===============
 
@@ -36,7 +33,7 @@ change your ``airflow.cfg`` to point the executor parameter to
 For more information about setting up a Celery broker, refer to the
 exhaustive `Celery documentation on the topic <https://docs.celeryq.dev/en/latest/getting-started/>`_.
 
-The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`.
+The configuration parameters of the Celery Executor can be found in :doc:`configurations-ref`.
 
 Here are a few imperative requirements for your workers:
 
@@ -97,7 +94,7 @@ Some caveats:
 - Tasks can consume resources. Make sure your worker has enough resources to run ``worker_concurrency`` tasks
 - Queue names are limited to 256 characters, but each broker backend might have its own restrictions
 
-See :doc:`/administration-and-deployment/modules_management` for details on how Python and Airflow manage modules.
+See :doc:`apache-airflow:administration-and-deployment/modules_management` for details on how Python and Airflow manage modules.
 
 Architecture
 ------------
@@ -173,7 +170,7 @@ The components communicate with each other in many places
 Task execution process
 ----------------------
 
-.. figure:: ../../img/run_task_on_celery_executor.png
+.. figure:: img/run_task_on_celery_executor.png
     :scale: 50 %
 
     Sequence diagram - task execution process
diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
similarity index 95%
rename from docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
rename to docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
index 6416f5d2fa58a..2a72bd2e83272 100644
--- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
+++ b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
@@ -16,8 +16,6 @@
     under the License.
 
 
-.. _executor:CeleryKubernetesExecutor:
-
 CeleryKubernetes Executor
 =========================
 
@@ -36,7 +34,7 @@ An executor is chosen to run a task based on the task's queue.
 ``CeleryKubernetesExecutor`` inherits the scalability of the ``CeleryExecutor`` to
 handle the high load at the peak time and runtime isolation of the ``KubernetesExecutor``.
 
-The configuration parameters of the Celery Executor can be found in :doc:`apache-airflow-providers-celery:configurations-ref`.
+The configuration parameters of the Celery Executor can be found in :doc:`configurations-ref`.
 
 
 When to use CeleryKubernetesExecutor
diff --git a/docs/apache-airflow/img/run_task_on_celery_executor.png b/docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png
similarity index 100%
rename from docs/apache-airflow/img/run_task_on_celery_executor.png
rename to docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png
diff --git a/docs/apache-airflow-providers-celery/index.rst b/docs/apache-airflow-providers-celery/index.rst
index 0205ce7765511..220204c1d9415 100644
--- a/docs/apache-airflow-providers-celery/index.rst
+++ b/docs/apache-airflow-providers-celery/index.rst
@@ -29,6 +29,14 @@
     Changelog <changelog>
     Security <security>
 
+.. toctree::
+    :hidden:
+    :maxdepth: 1
+    :caption: Executors
+
+    CeleryExecutor details <celery_executor>
+    CeleryKubernetesExecutor details <celery_kubernetes_executor>
+
 .. toctree::
     :hidden:
     :maxdepth: 1
diff --git a/docs/apache-airflow/img/arch-diag-kubernetes.png b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png
similarity index 100%
rename from docs/apache-airflow/img/arch-diag-kubernetes.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png
diff --git a/docs/apache-airflow/img/arch-diag-kubernetes2.png b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png
similarity index 100%
rename from docs/apache-airflow/img/arch-diag-kubernetes2.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png
diff --git a/docs/apache-airflow/img/k8s-failed-pod.png b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png
similarity index 100%
rename from docs/apache-airflow/img/k8s-failed-pod.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png
diff --git a/docs/apache-airflow/img/k8s-happy-path.png b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png
similarity index 100%
rename from docs/apache-airflow/img/k8s-happy-path.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png
diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/docs/apache-airflow-providers-cncf-kubernetes/index.rst
index 05ff7251551d7..a3e8063651769 100644
--- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst
@@ -29,6 +29,14 @@
     Changelog <changelog>
     Security <security>
 
+.. toctree::
+    :hidden:
+    :maxdepth: 1
+    :caption: Executors
+
+    KubernetesExecutor details <kubernetes_executor>
+    LocalKubernetesExecutor details <local_kubernetes_executor>
+
 .. toctree::
     :hidden:
     :maxdepth: 1
diff --git a/docs/apache-airflow/core-concepts/executor/kubernetes.rst b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
similarity index 98%
rename from docs/apache-airflow/core-concepts/executor/kubernetes.rst
rename to docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
index e19264c96e5e5..5662d7a0e1f26 100644
--- a/docs/apache-airflow/core-concepts/executor/kubernetes.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
@@ -16,7 +16,7 @@
     under the License.
 
 
-.. _executor:KubernetesExecutor:
+.. _KubernetesExecutor:
 
 Kubernetes Executor
 ===================
@@ -38,12 +38,12 @@ KubernetesExecutor requires a non-sqlite database in the backend.
 
 When a DAG submits a task, the KubernetesExecutor requests a worker pod from the Kubernetes API. The worker pod then runs the task, reports the result, and terminates.
 
-.. image:: ../../img/arch-diag-kubernetes.png
+.. image:: img/arch-diag-kubernetes.png
 
 
 One example of an Airflow deployment running on a distributed set of five nodes in a Kubernetes cluster is shown below.
 
-.. image:: ../../img/arch-diag-kubernetes2.png
+.. image:: img/arch-diag-kubernetes2.png
 
 Consistent with the regular Airflow architecture, the Workers need access to the DAG files to execute the tasks within those DAGs and interact with the Metadata repository. Also, configuration information specific to the Kubernetes Executor, such as the worker namespace and image information, needs to be specified in the Airflow Configuration file.
 
@@ -56,7 +56,7 @@ Additionally, the Kubernetes Executor enables specification of additional featur
 .. Airflow_Worker -> Kubernetes: Pod completes with state "Succeeded" and k8s records in ETCD
 .. Kubernetes -> Airflow_Scheduler: Airflow scheduler reads "Succeeded" from k8s watcher thread
 .. @enduml
-.. image:: ../../img/k8s-happy-path.png
+.. image:: img/k8s-happy-path.png
 
 Configuration
 -------------
@@ -272,7 +272,7 @@ In the case where a worker dies before it can report its status to the backend D
 ..
 .. @enduml
 
-.. image:: ../../img/k8s-failed-pod.png
+.. image:: img/k8s-failed-pod.png
 
 
 A Kubernetes watcher is a thread that can subscribe to every change that occurs in Kubernetes' database. It is alerted when pods start, run, end, and fail.
diff --git a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
similarity index 97%
rename from docs/apache-airflow/core-concepts/executor/local_kubernetes.rst
rename to docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
index d50c7fc8780bd..daed3d4d1f270 100644
--- a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
@@ -16,7 +16,7 @@
     under the License.
 
 
-.. _executor:LocalKubernetesExecutor:
+.. _LocalKubernetesExecutor:
 
 LocalKubernetes Executor
 =========================
diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
index f24fd61602efa..95c07c50ab98d 100644
--- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
@@ -35,7 +35,7 @@ you to create and run Pods on a Kubernetes cluster.
   - :ref:`EksPodOperator <howto/operator:EksPodOperator>` operator for `AWS Elastic Kubernetes Engine <https://aws.amazon.com/eks/>`__.
 
 .. note::
-  The :doc:`Kubernetes executor <apache-airflow:core-concepts/executor/kubernetes>` is **not** required to use this operator.
+  The :doc:`Kubernetes executor <kubernetes_executor>` is **not** required to use this operator.
 
 How does this operator work?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/docs/apache-airflow-providers-openai/dask_executor.rst b/docs/apache-airflow-providers-openai/dask_executor.rst
new file mode 100644
index 0000000000000..34aca5a80d223
--- /dev/null
+++ b/docs/apache-airflow-providers-openai/dask_executor.rst
@@ -0,0 +1,53 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Dask Executor
+=============
+
+:class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` allows you to run Airflow tasks in a Dask Distributed cluster.
+
+Dask clusters can be run on a single machine or on remote networks. For complete
+details, consult the `Distributed documentation <https://distributed.readthedocs.io/>`_.
+
+To create a cluster, first start a Scheduler:
+
+.. code-block:: bash
+
+    # default settings for a local cluster
+    DASK_HOST=127.0.0.1
+    DASK_PORT=8786
+
+    dask-scheduler --host $DASK_HOST --port $DASK_PORT
+
+Next start at least one Worker on any machine that can connect to the host:
+
+.. code-block:: bash
+
+    dask-worker $DASK_HOST:$DASK_PORT
+
+Edit your ``airflow.cfg`` to set your executor to :class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor` and provide
+the Dask Scheduler address in the ``[dask]`` section. For more information on setting the configuration,
+see :doc:`apache-airflow:howto/set-config`.
+
+Please note:
+
+- Each Dask worker must be able to import Airflow and any dependencies you
+  require.
+- The DaskExecutor implements queues using
+  `Dask Worker Resources <https://distributed.dask.org/en/latest/resources.html>`_ functionality. To enable the use of
+  queues, start your Dask workers with resources of the same name as the desired queues and a limit of ``inf``.
+  E.g. ``dask-worker <scheduler_address> --resources="QUEUE1=inf,QUEUE2=inf"``.
diff --git a/docs/apache-airflow/administration-and-deployment/production-deployment.rst b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
index 281dc7f658cb2..7b126117ed5e1 100644
--- a/docs/apache-airflow/administration-and-deployment/production-deployment.rst
+++ b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
@@ -60,8 +60,8 @@ Airflow uses :class:`~airflow.executors.sequential_executor.SequentialExecutor`
 nature, the user is limited to executing at most one task at a time. ``Sequential Executor`` also pauses
 the scheduler when it runs a task, hence it is not recommended in a production setup. You should use the
 :class:`~airflow.executors.local_executor.LocalExecutor` for a single machine.
-For a multi-node setup, you should use the :doc:`Kubernetes executor <../core-concepts/executor/kubernetes>` or
-the :doc:`Celery executor <../core-concepts/executor/celery>`.
+For a multi-node setup, you should use the :doc:`Kubernetes executor <.apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or
+the :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
 
 
 Once you have configured the executor, it is necessary to make sure that every node in the cluster contains
diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst
index e166c1024797c..0127bc266cce8 100644
--- a/docs/apache-airflow/best-practices.rst
+++ b/docs/apache-airflow/best-practices.rst
@@ -82,7 +82,7 @@ it difficult to check the logs of that Task from the Webserver. If that is not d
 Communication
 --------------
 
-Airflow executes tasks of a DAG on different servers in case you are using :doc:`Kubernetes executor </core-concepts/executor/kubernetes>` or :doc:`Celery executor </core-concepts/executor/celery>`.
+Airflow executes tasks of a DAG on different servers in case you are using :doc:`Kubernetes executor <apache-airflow-providers-cncf-kubernetes/kubernetes_executor>` or :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
 Therefore, you should not store any file or config in the local filesystem as the next task is likely to run on a different server without access to it — for example, a task that downloads the data file that the next task processes.
 In the case of :class:`Local executor <airflow.executors.local_executor.LocalExecutor>`,
 storing a file on disk can make retries harder e.g., your task requires a config file that is deleted by another task in DAG.
diff --git a/docs/apache-airflow/core-concepts/executor/debug.rst b/docs/apache-airflow/core-concepts/executor/debug.rst
index 896f79ce1ca76..6dcdfa51a40d2 100644
--- a/docs/apache-airflow/core-concepts/executor/debug.rst
+++ b/docs/apache-airflow/core-concepts/executor/debug.rst
@@ -15,44 +15,10 @@
     specific language governing permissions and limitations
     under the License.
 
-.. _executor:DebugExecutor:
-
-Debug Executor (deprecated)
-===========================
+.. _concepts:debugging:
 
-The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as
-a debug tool and can be used from IDE. It is a single process executor that
-queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by running
-``_run_raw_task`` method.
-
-Due to its nature the executor can be used with SQLite database. When used
-with sensors the executor will change sensor mode to ``reschedule`` to avoid
-blocking the execution of DAG.
-
-Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make
-all other running or scheduled tasks fail immediately. To enable this option set
-``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your ``airflow.cfg``.
-For more information on setting the configuration, see :doc:`../../howto/set-config`.
-
-**IDE setup steps:**
-
-1. Add ``main`` block at the end of your DAG file to make it runnable.
-
-It will run a backfill job:
-
-.. code-block:: python
-
-  if __name__ == "__main__":
-      from airflow.utils.state import State
-
-      dag.clear()
-      dag.run()
-
-
-2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of your IDE. In
-   this step you should also setup all environment variables required by your DAG.
-
-3. Run / debug the DAG file.
+Debugging Airflow DAGs
+======================
 
 Testing DAGs with dag.test()
 *****************************
@@ -74,7 +40,7 @@ and that's it! You can add argument such as ``execution_date`` if you want to te
 you can run or debug DAGs as needed.
 
 Comparison with DebugExecutor
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+-----------------------------
 
 The ``dag.test`` command has the following benefits over the :class:`~airflow.executors.debug_executor.DebugExecutor`
 class, which is now deprecated:
@@ -102,3 +68,42 @@ Run ``python -m pdb <path to dag file>.py`` for an interactive debugging experie
   -> bash_command='echo 1',
   (Pdb) run_this_last
   <Task(EmptyOperator): run_this_last>
+
+.. _executor:DebugExecutor:
+
+Debug Executor (deprecated)
+***************************
+
+The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as
+a debug tool and can be used from IDE. It is a single process executor that
+queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by running
+``_run_raw_task`` method.
+
+Due to its nature the executor can be used with SQLite database. When used
+with sensors the executor will change sensor mode to ``reschedule`` to avoid
+blocking the execution of DAG.
+
+Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make
+all other running or scheduled tasks fail immediately. To enable this option set
+``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your ``airflow.cfg``.
+For more information on setting the configuration, see :doc:`../../howto/set-config`.
+
+**IDE setup steps:**
+
+1. Add ``main`` block at the end of your DAG file to make it runnable.
+
+It will run a backfill job:
+
+.. code-block:: python
+
+  if __name__ == "__main__":
+      from airflow.utils.state import State
+
+      dag.clear()
+      dag.run()
+
+
+2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of your IDE. In
+   this step you should also setup all environment variables required by your DAG.
+
+3. Run / debug the DAG file.
diff --git a/docs/apache-airflow/core-concepts/executor/index.rst b/docs/apache-airflow/core-concepts/executor/index.rst
index e1cc52cf6f07c..d463cde3c83a4 100644
--- a/docs/apache-airflow/core-concepts/executor/index.rst
+++ b/docs/apache-airflow/core-concepts/executor/index.rst
@@ -52,19 +52,19 @@ There are two types of executors - those that run tasks *locally* (inside the ``
 .. toctree::
     :maxdepth: 1
 
-    debug
     local
     sequential
 
-**Remote Executors**
+The DebugExecutor also exists but has been deprecated in favor of using the ``dag.test()`` command for :doc:`debugging <../debug>`.
 
-.. toctree::
-    :maxdepth: 1
 
-    celery
-    celery_kubernetes
-    kubernetes
-    local_kubernetes
+**Remote Executors**
+
+* :doc:`CeleryExecutor apache-airflow-providers-celery:celery_executor`
+* :doc:`CeleryKubernetesExecutor apache-airflow-providers-celery:celery_kubernetes_executor`
+* :doc:`DaskExecutor apache-airflow-providers-daskexecutor:dask_executor`
+* :doc:`KubernetesExecutor <apache-airflow-providers-cncf-kubernetes:kubernetes_executor>`
+* :doc:`KubernetesLocalExecutor <apache-airflow-providers-cncf-kubernetes:local_kubernetes_executor>`
 
 
 .. note::
diff --git a/docs/apache-airflow/core-concepts/index.rst b/docs/apache-airflow/core-concepts/index.rst
index 9b24d4ea3f26d..5659f63b151d8 100644
--- a/docs/apache-airflow/core-concepts/index.rst
+++ b/docs/apache-airflow/core-concepts/index.rst
@@ -35,6 +35,7 @@ Here you can find detailed documentation about each one of the core concepts of
 
     dags
     dag-run
+    debug
     tasks
     operators
     sensors
diff --git a/docs/apache-airflow/howto/docker-compose/index.rst b/docs/apache-airflow/howto/docker-compose/index.rst
index 383a0f4c559df..c15a4cf09a05d 100644
--- a/docs/apache-airflow/howto/docker-compose/index.rst
+++ b/docs/apache-airflow/howto/docker-compose/index.rst
@@ -20,7 +20,7 @@
 Running Airflow in Docker
 #########################
 
-This quick-start guide will allow you to quickly get Airflow up and running with :doc:`CeleryExecutor </core-concepts/executor/celery>` in Docker.
+This quick-start guide will allow you to quickly get Airflow up and running with the :doc:`CeleryExecutor <apache-airflow-providers-celery:celery_executor>` in Docker.
 
 .. caution::
     This procedure can be useful for learning and exploration. However, adapting it for use in real-world situations can be complicated. Making changes to this procedure will require specialized expertise in Docker & Docker Compose, and the Airflow community may not be able to help you.
@@ -99,7 +99,7 @@ Optionally, you can enable flower by adding ``--profile flower`` option, e.g. ``
 
 - ``flower`` - `The flower app <https://flower.readthedocs.io/en/latest/>`__ for monitoring the environment. It is available at ``http://localhost:5555``.
 
-All these services allow you to run Airflow with :doc:`CeleryExecutor </core-concepts/executor/celery>`. For more information, see :doc:`/core-concepts/overview`.
+All these services allow you to run Airflow with the :doc:`CeleryExecutor <apache-airflow-providers-celery:celery_executor>`. For more information, see :doc:`/core-concepts/overview`.
 
 Some directories in the container are mounted, which means that their contents are synchronized between your computer and the container.
 
diff --git a/docs/apache-airflow/img/k8s-0-worker.jpeg b/docs/apache-airflow/img/k8s-0-worker.jpeg
deleted file mode 100644
index 71b1aedf612c6fbf8c9d0a5477c903958e279a08..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/apache-airflow/img/k8s-3-worker.jpeg b/docs/apache-airflow/img/k8s-3-worker.jpeg
deleted file mode 100644
index 0d6f5ca416165254b86d7b8b4b1fbfeff1ee4d2a..0000000000000000000000000000000000000000
GIT binary patch
diff --git a/docs/apache-airflow/img/k8s-5-worker.jpeg b/docs/apache-airflow/img/k8s-5-worker.jpeg
deleted file mode 100644
index 30227840cf20ae5afd0b32dae1fe0a5a182d7c07..0000000000000000000000000000000000000000
GIT binary patch

From 6c0693aa5e022ffba8099dc44afee7446bc09472 Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Mon, 16 Oct 2023 21:53:17 -0400
Subject: [PATCH 2/6] fix broken links

---
 .github/boring-cyborg.yml                                   | 3 ---
 .../administration-and-deployment/kubernetes.rst            | 2 +-
 .../logging-monitoring/check-health.rst                     | 2 +-
 .../administration-and-deployment/production-deployment.rst | 2 +-
 docs/apache-airflow/best-practices.rst                      | 2 +-
 docs/apache-airflow/core-concepts/executor/index.rst        | 6 +++---
 6 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index f6798b54dbd08..27170e1551727 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-
 # Details: https://github.com/kaxil/boring-cyborg
 
 labelPRBasedOnFilePath:
@@ -154,8 +153,6 @@ labelPRBasedOnFilePath:
     - airflow/example_dags/example_kubernetes_executor.py
     - airflow/providers/cncf/kubernetes/**/*
     - airflow/providers/celery/executors/celery_kubernetes_executor.py
-    - docs/apache-airflow/core-concepts/executor/kubernetes.rst
-    - docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
     - docs/apache-airflow-providers-cncf-kubernetes/**/*
     - kubernetes_tests/**/*
     - tests/providers/cncf/kubernetes/**/*
diff --git a/docs/apache-airflow/administration-and-deployment/kubernetes.rst b/docs/apache-airflow/administration-and-deployment/kubernetes.rst
index 160694fa11466..77bf719e27969 100644
--- a/docs/apache-airflow/administration-and-deployment/kubernetes.rst
+++ b/docs/apache-airflow/administration-and-deployment/kubernetes.rst
@@ -32,7 +32,7 @@ We maintain an :doc:`official Helm chart <helm-chart:index>` for Airflow that he
 Kubernetes Executor
 ^^^^^^^^^^^^^^^^^^^
 
-The :doc:`Kubernetes Executor </core-concepts/executor/kubernetes>` allows you to run all the Airflow tasks on
+The :doc:`Kubernetes Executor <apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` allows you to run all the Airflow tasks on
 Kubernetes as separate Pods.
 
 KubernetesPodOperator
diff --git a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
index 78cf0f37254c8..773c65316d316 100644
--- a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
+++ b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
@@ -135,7 +135,7 @@ HTTP monitoring for Celery Cluster
 
 You can optionally use Flower to monitor the health of the Celery cluster. It also provides an HTTP API that you can use to build a health check for your environment.
 
-For details about installation, see: :ref:`executor:CeleryExecutor`. For details about usage, see: `The Flower project documentation <https://flower.readthedocs.io/>`__.
+For details about installation, see: :doc:`CeleryExecutor <apache-airflow-providers-celery:celery_executor>`. For details about usage, see: `The Flower project documentation <https://flower.readthedocs.io/>`__.
 
 CLI Check for Celery Workers
 ----------------------------
diff --git a/docs/apache-airflow/administration-and-deployment/production-deployment.rst b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
index 7b126117ed5e1..fdbe63a048cc5 100644
--- a/docs/apache-airflow/administration-and-deployment/production-deployment.rst
+++ b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
@@ -60,7 +60,7 @@ Airflow uses :class:`~airflow.executors.sequential_executor.SequentialExecutor`
 nature, the user is limited to executing at most one task at a time. ``Sequential Executor`` also pauses
 the scheduler when it runs a task, hence it is not recommended in a production setup. You should use the
 :class:`~airflow.executors.local_executor.LocalExecutor` for a single machine.
-For a multi-node setup, you should use the :doc:`Kubernetes executor <.apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or
+For a multi-node setup, you should use the :doc:`Kubernetes executor <apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or
 the :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
 
 
diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst
index 0127bc266cce8..3fa90120eeb11 100644
--- a/docs/apache-airflow/best-practices.rst
+++ b/docs/apache-airflow/best-practices.rst
@@ -82,7 +82,7 @@ it difficult to check the logs of that Task from the Webserver. If that is not d
 Communication
 --------------
 
-Airflow executes tasks of a DAG on different servers in case you are using :doc:`Kubernetes executor <apache-airflow-providers-cncf-kubernetes/kubernetes_executor>` or :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
+Airflow executes tasks of a DAG on different servers in case you are using :doc:`KubernetesExecutor <apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
 Therefore, you should not store any file or config in the local filesystem as the next task is likely to run on a different server without access to it — for example, a task that downloads the data file that the next task processes.
 In the case of :class:`Local executor <airflow.executors.local_executor.LocalExecutor>`,
 storing a file on disk can make retries harder e.g., your task requires a config file that is deleted by another task in DAG.
diff --git a/docs/apache-airflow/core-concepts/executor/index.rst b/docs/apache-airflow/core-concepts/executor/index.rst
index d463cde3c83a4..b74fd7a7d0fa0 100644
--- a/docs/apache-airflow/core-concepts/executor/index.rst
+++ b/docs/apache-airflow/core-concepts/executor/index.rst
@@ -60,9 +60,9 @@ The DebugExecutor also exists but has been deprecated in favor of using the ``da
 
 **Remote Executors**
 
-* :doc:`CeleryExecutor apache-airflow-providers-celery:celery_executor`
-* :doc:`CeleryKubernetesExecutor apache-airflow-providers-celery:celery_kubernetes_executor`
-* :doc:`DaskExecutor apache-airflow-providers-daskexecutor:dask_executor`
+* :doc:`CeleryExecutor <apache-airflow-providers-celery:celery_executor>`
+* :doc:`CeleryKubernetesExecutor <apache-airflow-providers-celery:celery_kubernetes_executor>`
+* :doc:`DaskExecutor <apache-airflow-providers-daskexecutor:dask_executor>`
 * :doc:`KubernetesExecutor <apache-airflow-providers-cncf-kubernetes:kubernetes_executor>`
 * :doc:`KubernetesLocalExecutor <apache-airflow-providers-cncf-kubernetes:local_kubernetes_executor>`
 

From c22685feabf475e07a49a3fdb354a23800488a74 Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Tue, 17 Oct 2023 20:49:14 -0400
Subject: [PATCH 3/6] redirects

---
 docs/apache-airflow/core-concepts/index.rst | 10 +++++++++-
 docs/apache-airflow/redirects.txt           | 16 ++++++++++++++++
 docs/exts/redirects.py                      | 11 ++++++++++-
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/docs/apache-airflow/core-concepts/index.rst b/docs/apache-airflow/core-concepts/index.rst
index 5659f63b151d8..62d562a809ac4 100644
--- a/docs/apache-airflow/core-concepts/index.rst
+++ b/docs/apache-airflow/core-concepts/index.rst
@@ -35,7 +35,6 @@ Here you can find detailed documentation about each one of the core concepts of
 
     dags
     dag-run
-    debug
     tasks
     operators
     sensors
@@ -52,3 +51,12 @@ Here you can find detailed documentation about each one of the core concepts of
     xcoms
     variables
     params
+
+**Debugging**
+
+.. toctree::
+    :titlesonly:
+    :maxdepth: 1
+
+    debug
+
diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt
index 2eb3e97a38174..d2b867ee6f229 100644
--- a/docs/apache-airflow/redirects.txt
+++ b/docs/apache-airflow/redirects.txt
@@ -139,6 +139,22 @@ executor/local.rst core-concepts/executor/local.rst
 executor/sequential.rst core-concepts/executor/sequential.rst
 upgrading-from-1-10/upgrade-check.rst howto/upgrading-from-1-10/upgrade-check.rst
 
+# Moving provider executor docs to providers
+## The `../` indicates that it will move to the root of the docs directory, unlike the rest of 
+## the entries here which move to docs/apache-airflow.
+## It's okay to include `/stable/`, because there's no relationship between a version of
+## Airflow and the version of any provider package.
+### Kubernetes Executors
+core-concepts/executor/kubernetes.rst ../apache-airflow-providers-cncf-kubernetes/stable/kubernetes_executor.html
+core-concepts/executor/local_kubernetes.rst ../apache-airflow-providers-cncf-kubernetes/stable/local_kubernetes_executor.html
+
+### Celery Executors
+core-concepts/executor/celery_kubernetes.rst ../apache-airflow-providers-celery/stable/celery_kubernetes_executor.html
+core-concepts/executor/celery.rst ../apache-airflow-providers-celery/stable/celery_executor.html
+
+### Dask Executor
+core-concepts/executor/dask.rst ../apache-airflow-providers-daskexecutor/stable/dask_executor.html
+
 # Python API
 python-api-ref.rst public-airflow-interface.rst
 
diff --git a/docs/exts/redirects.py b/docs/exts/redirects.py
index b186da9994416..3f8d9dcb6275e 100644
--- a/docs/exts/redirects.py
+++ b/docs/exts/redirects.py
@@ -49,18 +49,27 @@ def generate_redirects(app):
             if line.startswith("#"):
                 continue
 
+            # Split line into the original path `from_path` and where the URL should redirect to `to_path`
             from_path, _, to_path = line.rstrip().partition(" ")
 
             log.debug("Redirecting '%s' to '%s'", from_path, to_path)
 
+            # in_suffix is often ".rst"
             from_path = from_path.replace(in_suffix, ".html")
             to_path = to_path.replace(in_suffix, ".html")
 
-            to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)) - 1)
+            # The redirect path needs to move back to the root of the apache-airflow docs directory
+            # or the root of the docs directory altogether for provider packages.
+            if "../" and "providers" in to_path:
+                to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)))
+            else:
+                to_path_prefix = f"..{os.path.sep}" * (len(from_path.split(os.path.sep)) - 1)
+
             to_path = to_path_prefix + to_path
 
             log.debug("Resolved redirect '%s' to '%s'", from_path, to_path)
 
+            # This will be used to save an HTML file with `TEMPLATE` formatted
             redirected_filename = os.path.join(app.builder.outdir, from_path)
             redirected_directory = os.path.dirname(redirected_filename)
 

From 79d6c0f92daf91079effce9e358320b8024c5031 Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Wed, 18 Oct 2023 17:12:17 -0400
Subject: [PATCH 4/6] fixing conflicts

---
 .../core-concepts/{executor => }/debug.rst    | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 rename docs/apache-airflow/core-concepts/{executor => }/debug.rst (67%)

diff --git a/docs/apache-airflow/core-concepts/executor/debug.rst b/docs/apache-airflow/core-concepts/debug.rst
similarity index 67%
rename from docs/apache-airflow/core-concepts/executor/debug.rst
rename to docs/apache-airflow/core-concepts/debug.rst
index 6dcdfa51a40d2..42385073c0204 100644
--- a/docs/apache-airflow/core-concepts/executor/debug.rst
+++ b/docs/apache-airflow/core-concepts/debug.rst
@@ -107,3 +107,52 @@ It will run a backfill job:
    this step you should also setup all environment variables required by your DAG.
 
 3. Run / debug the DAG file.
+
+Testing DAGs with dag.test()
+*****************************
+
+To debug DAGs in an IDE, you can set up the ``dag.test`` command in your dag file and run through your DAG in a single
+serialized python process.
+
+This approach can be used with any supported database (including a local SQLite database) and will
+*fail fast* as all tasks run in a single process.
+
+To set up ``dag.test``, add these two lines to the bottom of your dag file:
+
+.. code-block:: python
+
+  if __name__ == "__main__":
+      dag.test()
+
+and that's it! You can add argument such as ``execution_date`` if you want to test argument-specific DAG runs, but otherwise
+you can run or debug DAGs as needed.
+
+Comparison with DebugExecutor
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``dag.test`` command has the following benefits over the :class:`~airflow.executors.debug_executor.DebugExecutor`
+class, which is now deprecated:
+
+1. It does not require running an executor at all. Tasks are run one at a time with no executor or scheduler logs.
+2. It is significantly faster than running code with a DebugExecutor as it does not need to go through a scheduler loop.
+3. It does not perform a backfill.
+
+
+Debugging Airflow DAGs on the command line
+******************************************
+
+With the same two line addition as mentioned in the above section, you can now easily debug a DAG using pdb as well.
+Run ``python -m pdb <path to dag file>.py`` for an interactive debugging experience on the command line.
+
+.. code-block:: bash
+
+  root@ef2c84ad4856:/opt/airflow# python -m pdb airflow/example_dags/example_bash_operator.py
+  > /opt/airflow/airflow/example_dags/example_bash_operator.py(18)<module>()
+  -> """Example DAG demonstrating the usage of the BashOperator."""
+  (Pdb) b 45
+  Breakpoint 1 at /opt/airflow/airflow/example_dags/example_bash_operator.py:45
+  (Pdb) c
+  > /opt/airflow/airflow/example_dags/example_bash_operator.py(45)<module>()
+  -> bash_command='echo 1',
+  (Pdb) run_this_last
+  <Task(EmptyOperator): run_this_last>

From fe5bfeeaa87e344d6557f78bd261f3c140adc320 Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Sat, 21 Oct 2023 09:57:55 -0400
Subject: [PATCH 5/6] resolve conflicts

---
 docs/apache-airflow/core-concepts/debug.rst | 49 ---------------------
 docs/apache-airflow/core-concepts/index.rst |  2 -
 docs/apache-airflow/redirects.txt           |  4 +-
 3 files changed, 2 insertions(+), 53 deletions(-)

diff --git a/docs/apache-airflow/core-concepts/debug.rst b/docs/apache-airflow/core-concepts/debug.rst
index 42385073c0204..6dcdfa51a40d2 100644
--- a/docs/apache-airflow/core-concepts/debug.rst
+++ b/docs/apache-airflow/core-concepts/debug.rst
@@ -107,52 +107,3 @@ It will run a backfill job:
    this step you should also setup all environment variables required by your DAG.
 
 3. Run / debug the DAG file.
-
-Testing DAGs with dag.test()
-*****************************
-
-To debug DAGs in an IDE, you can set up the ``dag.test`` command in your dag file and run through your DAG in a single
-serialized python process.
-
-This approach can be used with any supported database (including a local SQLite database) and will
-*fail fast* as all tasks run in a single process.
-
-To set up ``dag.test``, add these two lines to the bottom of your dag file:
-
-.. code-block:: python
-
-  if __name__ == "__main__":
-      dag.test()
-
-and that's it! You can add argument such as ``execution_date`` if you want to test argument-specific DAG runs, but otherwise
-you can run or debug DAGs as needed.
-
-Comparison with DebugExecutor
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The ``dag.test`` command has the following benefits over the :class:`~airflow.executors.debug_executor.DebugExecutor`
-class, which is now deprecated:
-
-1. It does not require running an executor at all. Tasks are run one at a time with no executor or scheduler logs.
-2. It is significantly faster than running code with a DebugExecutor as it does not need to go through a scheduler loop.
-3. It does not perform a backfill.
-
-
-Debugging Airflow DAGs on the command line
-******************************************
-
-With the same two line addition as mentioned in the above section, you can now easily debug a DAG using pdb as well.
-Run ``python -m pdb <path to dag file>.py`` for an interactive debugging experience on the command line.
-
-.. code-block:: bash
-
-  root@ef2c84ad4856:/opt/airflow# python -m pdb airflow/example_dags/example_bash_operator.py
-  > /opt/airflow/airflow/example_dags/example_bash_operator.py(18)<module>()
-  -> """Example DAG demonstrating the usage of the BashOperator."""
-  (Pdb) b 45
-  Breakpoint 1 at /opt/airflow/airflow/example_dags/example_bash_operator.py:45
-  (Pdb) c
-  > /opt/airflow/airflow/example_dags/example_bash_operator.py(45)<module>()
-  -> bash_command='echo 1',
-  (Pdb) run_this_last
-  <Task(EmptyOperator): run_this_last>
diff --git a/docs/apache-airflow/core-concepts/index.rst b/docs/apache-airflow/core-concepts/index.rst
index 62d562a809ac4..dab97ea3ff8ad 100644
--- a/docs/apache-airflow/core-concepts/index.rst
+++ b/docs/apache-airflow/core-concepts/index.rst
@@ -55,8 +55,6 @@ Here you can find detailed documentation about each one of the core concepts of
 **Debugging**
 
 .. toctree::
-    :titlesonly:
     :maxdepth: 1
 
     debug
-
diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt
index d2b867ee6f229..d7c6263c941af 100644
--- a/docs/apache-airflow/redirects.txt
+++ b/docs/apache-airflow/redirects.txt
@@ -140,9 +140,9 @@ executor/sequential.rst core-concepts/executor/sequential.rst
 upgrading-from-1-10/upgrade-check.rst howto/upgrading-from-1-10/upgrade-check.rst
 
 # Moving provider executor docs to providers
-## The `../` indicates that it will move to the root of the docs directory, unlike the rest of 
+## The ``../`` indicates that it will move to the root of the docs directory, unlike the rest of
 ## the entries here which move to docs/apache-airflow.
-## It's okay to include `/stable/`, because there's no relationship between a version of
+## It's okay to include ``/stable/``, because there's no relationship between a version of
 ## Airflow and the version of any provider package.
 ### Kubernetes Executors
 core-concepts/executor/kubernetes.rst ../apache-airflow-providers-cncf-kubernetes/stable/kubernetes_executor.html

From 0546c48ab8f17125214f3298e717d75090a1682c Mon Sep 17 00:00:00 2001
From: Ryan Hatter <ryanhatter10@gmail.com>
Date: Sun, 22 Oct 2023 21:29:31 -0400
Subject: [PATCH 6/6] skip clean on doc build retries

---
 docs/build_docs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/build_docs.py b/docs/build_docs.py
index 552f94e25499b..4ddbf5c59de2d 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -224,6 +224,7 @@ def build_docs_for_packages(
     spellcheck_only: bool,
     jobs: int,
     verbose: bool,
+    retry: bool = False,
 ) -> tuple[dict[str, list[DocBuildError]], dict[str, list[SpellingError]]]:
     """Builds documentation for all packages and combines errors."""
     all_build_errors: dict[str, list[DocBuildError]] = defaultdict(list)
