From 883b56f702339a5884ab77ee7ef299dd1e1445ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= <kamil.bregula@polidea.com>
Date: Tue, 14 Jul 2020 02:28:52 +0200
Subject: [PATCH] Add redoc docs to sphinx

---
 docs/conf.py                            |  23 +++++
 docs/exts/sphinx_script_update.py       | 113 ++++++++++++++++++++++++
 docs/index.rst                          |   1 +
 docs/rest-api-ref.rst                   |   6 +-
 docs/stable-rest-api/redoc.rst          |  23 +++++
 requirements/requirements-python3.6.txt |   1 +
 requirements/requirements-python3.7.txt |   4 +-
 requirements/requirements-python3.8.txt |   4 +-
 requirements/setup-3.6.md5              |   2 +-
 requirements/setup-3.7.md5              |   2 +-
 requirements/setup-3.8.md5              |   2 +-
 setup.py                                |   1 +
 12 files changed, 172 insertions(+), 10 deletions(-)
 create mode 100644 docs/exts/sphinx_script_update.py
 create mode 100644 docs/stable-rest-api/redoc.rst

diff --git a/docs/conf.py b/docs/conf.py
index b0033bd61e1ce..2fefdac9bf24d 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -136,6 +136,10 @@
     'removemarktransform',
     'sphinx_copybutton',
     'redirects',
+    # First, generate redoc
+    'sphinxcontrib.redoc',
+    # Second, update redoc script
+    "sphinx_script_update",
 ]
 
 autodoc_default_options = {
@@ -519,6 +523,25 @@
 # -- Options for sphinxcontrib-redirects ----------------------------------
 redirects_file = 'redirects.txt'
 
+# -- Options for redoc docs ----------------------------------
+OPENAPI_FILE = os.path.join(
+    os.path.dirname(__file__),
+    "..", "airflow", "api_connexion", "openapi", "v1.yaml"
+)
+redoc = [
+    {
+        'name': 'Airflow REST API',
+        'page': 'stable-rest-api/redoc',
+        'spec': OPENAPI_FILE,
+        'opts': {
+            'hide-hostname': True,
+        }
+    },
+]
+
+# Options for script updater
+redoc_script_url = "https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.30/bundles/redoc.standalone.js"
+
 # -- Additional HTML Context variable
 html_context = {
     # Google Analytics ID.
diff --git a/docs/exts/sphinx_script_update.py b/docs/exts/sphinx_script_update.py
new file mode 100644
index 0000000000000..0210aa7617c92
--- /dev/null
+++ b/docs/exts/sphinx_script_update.py
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import hashlib
+import json
+import os
+import sys
+import tempfile
+from distutils.file_util import copy_file
+from typing import Dict
+
+import requests
+from sphinx.builders import html as builders
+from sphinx.util import logging
+
+log = logging.getLogger(__name__)
+
+
+def _gethash(string: str):
+    hash_object = hashlib.sha256(string.encode())
+    return hash_object.hexdigest()
+
+
+def _user_cache_dir(appname=None):
+    """
+    Return full path to the user-specific cache dir for this application
+    """
+    if sys.platform == "win32":
+        # Windows has a complex procedure to download the App Dir directory because this directory can be
+        # changed in window registry, so i use temporary directory for cache
+        path = os.path.join(tempfile.gettempdir(), appname)
+    elif sys.platform == 'darwin':
+        path = os.path.expanduser('~/Library/Caches')
+    else:
+        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+    path = os.path.join(path, appname)
+    return path
+
+
+def fetch_and_cache(script_url: str, output_filename: str):
+    """Fetch URL to local cache and returns path."""
+    cache_key = _gethash(script_url)
+    cache_dir = _user_cache_dir("redoc-doc")
+    cache_metadata_filepath = os.path.join(cache_dir, "cache-metadata.json")
+    cache_filepath = os.path.join(cache_dir, f"{cache_key}-{output_filename}")
+    # Create cache directory
+    os.makedirs(cache_dir, exist_ok=True)
+    # Load cache metadata
+    cache_metadata: Dict[str, str] = {}
+    if os.path.exists(cache_metadata_filepath):
+        try:
+            with open(cache_metadata_filepath, "r") as cache_file:
+                cache_metadata = json.load(cache_file)
+        except json.JSONDecodeError:
+            os.remove(cache_metadata_filepath)
+    etag = cache_metadata.get(cache_key)
+
+    # If we have a file and etag, check the fast path
+    if os.path.exists(cache_filepath) and etag:
+        res = requests.get(script_url, headers={"If-None-Match": etag})
+        if res.status_code == 304:
+            return cache_filepath
+
+    # Slow patch
+    res = requests.get(script_url)
+    res.raise_for_status()
+
+    with open(cache_filepath, "wb") as output_file:
+        output_file.write(res.content)
+
+    # Save cache metadata, if needed
+    etag = res.headers.get('etag', None)
+    if etag:
+        cache_metadata[cache_key] = etag
+        with open(cache_metadata_filepath, 'w') as cache_file:
+            json.dump(cache_metadata, cache_file)
+
+    return cache_filepath
+
+
+def build_finished(app, exception):
+    """Sphinx "build_finished" event handler."""
+    if exception:
+        return
+    if not isinstance(app.builder, builders.StandaloneHTMLBuilder):
+        log.warning(
+            F"The plugin is support only 'html' builder, but you are using '{type(app.builder)}'. Skipping..."
+        )
+        return
+    script_url = app.config.redoc_script_url
+    output_filename = "script.js"
+
+    cache_filepath = fetch_and_cache(script_url, output_filename)
+    copy_file(cache_filepath, os.path.join(app.builder.outdir, '_static', "redoc.js"))
+
+
+def setup(app):
+    """Setup plugin"""
+    app.add_config_value("redoc_script_url", None, "env")
+    app.connect("build-finished", build_finished)
diff --git a/docs/index.rst b/docs/index.rst
index c7ed4236e0904..4c7494275a7c4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -111,4 +111,5 @@ Content
     Macros <macros-ref>
     Python API <_api/index>
     Experimental REST API <rest-api-ref>
+    Stable REST API <stable-rest-api/redoc>
     Configurations <configurations-ref>
diff --git a/docs/rest-api-ref.rst b/docs/rest-api-ref.rst
index e9efe805ba5d0..4456e49932fd2 100644
--- a/docs/rest-api-ref.rst
+++ b/docs/rest-api-ref.rst
@@ -15,10 +15,8 @@
     specific language governing permissions and limitations
     under the License.
 
-
-
-REST API Reference
-==================
+Experimental REST API Reference
+===============================
 
 Airflow exposes an REST API. It is available through the webserver. Endpoints are
 available at ``/api/experimental/``.
diff --git a/docs/stable-rest-api/redoc.rst b/docs/stable-rest-api/redoc.rst
new file mode 100644
index 0000000000000..60a80c16f26b7
--- /dev/null
+++ b/docs/stable-rest-api/redoc.rst
@@ -0,0 +1,23 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+REST API Reference
+==================
+
+It's a stub file. It will be converted automatically during the build process
+to the valid documentation by the shpinx plugin. See: /docs/exts/sphinx_redoc.py
diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt
index 3ecde7f7b026f..91051f5330482 100644
--- a/requirements/requirements-python3.6.txt
+++ b/requirements/requirements-python3.6.txt
@@ -354,6 +354,7 @@ sphinxcontrib-htmlhelp==1.0.3
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-redoc==1.6.0
 sphinxcontrib-serializinghtml==1.1.4
 spython==0.0.84
 sshpubkeys==3.1.0
diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt
index b7a0145949c80..5bbc3b08cce43 100644
--- a/requirements/requirements-python3.7.txt
+++ b/requirements/requirements-python3.7.txt
@@ -281,7 +281,6 @@ pyflakes==2.2.0
 pykerberos==1.2.1
 pylint==2.5.3
 pymongo==3.10.1
-pymssql==2.1.4
 pyodbc==4.0.30
 pyparsing==2.4.7
 pypd==1.1.0
@@ -310,7 +309,7 @@ pywinrm==0.4.1
 pyzmq==19.0.1
 qds-sdk==1.16.0
 redis==3.5.3
-regex==2020.6.8
+regex==2020.7.14
 requests-kerberos==0.12.0
 requests-mock==1.8.0
 requests-ntlm==1.1.0
@@ -349,6 +348,7 @@ sphinxcontrib-htmlhelp==1.0.3
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-redoc==1.6.0
 sphinxcontrib-serializinghtml==1.1.4
 spython==0.0.84
 sshpubkeys==3.1.0
diff --git a/requirements/requirements-python3.8.txt b/requirements/requirements-python3.8.txt
index b0d277346c166..3eeebe0039690 100644
--- a/requirements/requirements-python3.8.txt
+++ b/requirements/requirements-python3.8.txt
@@ -281,6 +281,7 @@ pyflakes==2.2.0
 pykerberos==1.2.1
 pylint==2.5.3
 pymongo==3.10.1
+pymssql==2.1.4
 pyodbc==4.0.30
 pyparsing==2.4.7
 pypd==1.1.0
@@ -309,7 +310,7 @@ pywinrm==0.4.1
 pyzmq==19.0.1
 qds-sdk==1.16.0
 redis==3.5.3
-regex==2020.6.8
+regex==2020.7.14
 requests-kerberos==0.12.0
 requests-mock==1.8.0
 requests-ntlm==1.1.0
@@ -348,6 +349,7 @@ sphinxcontrib-htmlhelp==1.0.3
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-redoc==1.6.0
 sphinxcontrib-serializinghtml==1.1.4
 spython==0.0.84
 sshpubkeys==3.1.0
diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5
index 3956d1fc00634..393768dfd67b0 100644
--- a/requirements/setup-3.6.md5
+++ b/requirements/setup-3.6.md5
@@ -1 +1 @@
-4b8e4aec9535614784887c046bed986b  /opt/airflow/setup.py
+ef23b0e1ef1c979fd0a1c04a570ca679  /opt/airflow/setup.py
diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5
index 3956d1fc00634..393768dfd67b0 100644
--- a/requirements/setup-3.7.md5
+++ b/requirements/setup-3.7.md5
@@ -1 +1 @@
-4b8e4aec9535614784887c046bed986b  /opt/airflow/setup.py
+ef23b0e1ef1c979fd0a1c04a570ca679  /opt/airflow/setup.py
diff --git a/requirements/setup-3.8.md5 b/requirements/setup-3.8.md5
index 3956d1fc00634..393768dfd67b0 100644
--- a/requirements/setup-3.8.md5
+++ b/requirements/setup-3.8.md5
@@ -1 +1 @@
-4b8e4aec9535614784887c046bed986b  /opt/airflow/setup.py
+ef23b0e1ef1c979fd0a1c04a570ca679  /opt/airflow/setup.py
diff --git a/setup.py b/setup.py
index c70519c9ff494..121bf863e8953 100644
--- a/setup.py
+++ b/setup.py
@@ -232,6 +232,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'sphinx-jinja~=1.1',
     'sphinx-rtd-theme>=0.1.6',
     'sphinxcontrib-httpdomain>=1.7.0',
+    "sphinxcontrib-redoc>=1.6.0"
 ]
 docker = [
     'docker~=3.0',
