diff --git a/docs/conf.py b/docs/conf.py index b0033bd61e1ce..2fefdac9bf24d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -136,6 +136,10 @@ 'removemarktransform', 'sphinx_copybutton', 'redirects', + # First, generate redoc + 'sphinxcontrib.redoc', + # Second, update redoc script + "sphinx_script_update", ] autodoc_default_options = { @@ -519,6 +523,25 @@ # -- Options for sphinxcontrib-redirects ---------------------------------- redirects_file = 'redirects.txt' +# -- Options for redoc docs ---------------------------------- +OPENAPI_FILE = os.path.join( + os.path.dirname(__file__), + "..", "airflow", "api_connexion", "openapi", "v1.yaml" +) +redoc = [ + { + 'name': 'Airflow REST API', + 'page': 'stable-rest-api/redoc', + 'spec': OPENAPI_FILE, + 'opts': { + 'hide-hostname': True, + } + }, +] + +# Options for script updater +redoc_script_url = "https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.30/bundles/redoc.standalone.js" + # -- Additional HTML Context variable html_context = { # Google Analytics ID. diff --git a/docs/exts/sphinx_script_update.py b/docs/exts/sphinx_script_update.py new file mode 100644 index 0000000000000..0210aa7617c92 --- /dev/null +++ b/docs/exts/sphinx_script_update.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import hashlib +import json +import os +import sys +import tempfile +from distutils.file_util import copy_file +from typing import Dict + +import requests +from sphinx.builders import html as builders +from sphinx.util import logging + +log = logging.getLogger(__name__) + + +def _gethash(string: str): + hash_object = hashlib.sha256(string.encode()) + return hash_object.hexdigest() + + +def _user_cache_dir(appname=None): + """ + Return full path to the user-specific cache dir for this application + """ + if sys.platform == "win32": + # Windows has a complex procedure to download the App Dir directory because this directory can be + # changed in window registry, so i use temporary directory for cache + path = os.path.join(tempfile.gettempdir(), appname) + elif sys.platform == 'darwin': + path = os.path.expanduser('~/Library/Caches') + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + path = os.path.join(path, appname) + return path + + +def fetch_and_cache(script_url: str, output_filename: str): + """Fetch URL to local cache and returns path.""" + cache_key = _gethash(script_url) + cache_dir = _user_cache_dir("redoc-doc") + cache_metadata_filepath = os.path.join(cache_dir, "cache-metadata.json") + cache_filepath = os.path.join(cache_dir, f"{cache_key}-{output_filename}") + # Create cache directory + os.makedirs(cache_dir, exist_ok=True) + # Load cache metadata + cache_metadata: Dict[str, str] = {} + if os.path.exists(cache_metadata_filepath): + try: + with open(cache_metadata_filepath, "r") as cache_file: + cache_metadata = json.load(cache_file) + except json.JSONDecodeError: + os.remove(cache_metadata_filepath) + etag = cache_metadata.get(cache_key) + + # If we have a file and etag, check the fast path + if os.path.exists(cache_filepath) and etag: + res = requests.get(script_url, headers={"If-None-Match": etag}) + if res.status_code == 304: + return cache_filepath + + # Slow patch + res = requests.get(script_url) + res.raise_for_status() + + with open(cache_filepath, "wb") as output_file: + output_file.write(res.content) + + # Save cache metadata, if needed + etag = res.headers.get('etag', None) + if etag: + cache_metadata[cache_key] = etag + with open(cache_metadata_filepath, 'w') as cache_file: + json.dump(cache_metadata, cache_file) + + return cache_filepath + + +def build_finished(app, exception): + """Sphinx "build_finished" event handler.""" + if exception: + return + if not isinstance(app.builder, builders.StandaloneHTMLBuilder): + log.warning( + F"The plugin is support only 'html' builder, but you are using '{type(app.builder)}'. Skipping..." + ) + return + script_url = app.config.redoc_script_url + output_filename = "script.js" + + cache_filepath = fetch_and_cache(script_url, output_filename) + copy_file(cache_filepath, os.path.join(app.builder.outdir, '_static', "redoc.js")) + + +def setup(app): + """Setup plugin""" + app.add_config_value("redoc_script_url", None, "env") + app.connect("build-finished", build_finished) diff --git a/docs/index.rst b/docs/index.rst index c7ed4236e0904..4c7494275a7c4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -111,4 +111,5 @@ Content Macros Python API <_api/index> Experimental REST API + Stable REST API Configurations diff --git a/docs/rest-api-ref.rst b/docs/rest-api-ref.rst index e9efe805ba5d0..4456e49932fd2 100644 --- a/docs/rest-api-ref.rst +++ b/docs/rest-api-ref.rst @@ -15,10 +15,8 @@ specific language governing permissions and limitations under the License. - - -REST API Reference -================== +Experimental REST API Reference +=============================== Airflow exposes an REST API. It is available through the webserver. Endpoints are available at ``/api/experimental/``. diff --git a/docs/stable-rest-api/redoc.rst b/docs/stable-rest-api/redoc.rst new file mode 100644 index 0000000000000..60a80c16f26b7 --- /dev/null +++ b/docs/stable-rest-api/redoc.rst @@ -0,0 +1,23 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +REST API Reference +================== + +It's a stub file. It will be converted automatically during the build process +to the valid documentation by the shpinx plugin. See: /docs/exts/sphinx_redoc.py diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt index 3ecde7f7b026f..91051f5330482 100644 --- a/requirements/requirements-python3.6.txt +++ b/requirements/requirements-python3.6.txt @@ -354,6 +354,7 @@ sphinxcontrib-htmlhelp==1.0.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.4 spython==0.0.84 sshpubkeys==3.1.0 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt index b7a0145949c80..5bbc3b08cce43 100644 --- a/requirements/requirements-python3.7.txt +++ b/requirements/requirements-python3.7.txt @@ -281,7 +281,6 @@ pyflakes==2.2.0 pykerberos==1.2.1 pylint==2.5.3 pymongo==3.10.1 -pymssql==2.1.4 pyodbc==4.0.30 pyparsing==2.4.7 pypd==1.1.0 @@ -310,7 +309,7 @@ pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 redis==3.5.3 -regex==2020.6.8 +regex==2020.7.14 requests-kerberos==0.12.0 requests-mock==1.8.0 requests-ntlm==1.1.0 @@ -349,6 +348,7 @@ sphinxcontrib-htmlhelp==1.0.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.4 spython==0.0.84 sshpubkeys==3.1.0 diff --git a/requirements/requirements-python3.8.txt b/requirements/requirements-python3.8.txt index b0d277346c166..3eeebe0039690 100644 --- a/requirements/requirements-python3.8.txt +++ b/requirements/requirements-python3.8.txt @@ -281,6 +281,7 @@ pyflakes==2.2.0 pykerberos==1.2.1 pylint==2.5.3 pymongo==3.10.1 +pymssql==2.1.4 pyodbc==4.0.30 pyparsing==2.4.7 pypd==1.1.0 @@ -309,7 +310,7 @@ pywinrm==0.4.1 pyzmq==19.0.1 qds-sdk==1.16.0 redis==3.5.3 -regex==2020.6.8 +regex==2020.7.14 requests-kerberos==0.12.0 requests-mock==1.8.0 requests-ntlm==1.1.0 @@ -348,6 +349,7 @@ sphinxcontrib-htmlhelp==1.0.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.4 spython==0.0.84 sshpubkeys==3.1.0 diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5 index 3956d1fc00634..393768dfd67b0 100644 --- a/requirements/setup-3.6.md5 +++ b/requirements/setup-3.6.md5 @@ -1 +1 @@ -4b8e4aec9535614784887c046bed986b /opt/airflow/setup.py +ef23b0e1ef1c979fd0a1c04a570ca679 /opt/airflow/setup.py diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5 index 3956d1fc00634..393768dfd67b0 100644 --- a/requirements/setup-3.7.md5 +++ b/requirements/setup-3.7.md5 @@ -1 +1 @@ -4b8e4aec9535614784887c046bed986b /opt/airflow/setup.py +ef23b0e1ef1c979fd0a1c04a570ca679 /opt/airflow/setup.py diff --git a/requirements/setup-3.8.md5 b/requirements/setup-3.8.md5 index 3956d1fc00634..393768dfd67b0 100644 --- a/requirements/setup-3.8.md5 +++ b/requirements/setup-3.8.md5 @@ -1 +1 @@ -4b8e4aec9535614784887c046bed986b /opt/airflow/setup.py +ef23b0e1ef1c979fd0a1c04a570ca679 /opt/airflow/setup.py diff --git a/setup.py b/setup.py index c70519c9ff494..121bf863e8953 100644 --- a/setup.py +++ b/setup.py @@ -232,6 +232,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'sphinx-jinja~=1.1', 'sphinx-rtd-theme>=0.1.6', 'sphinxcontrib-httpdomain>=1.7.0', + "sphinxcontrib-redoc>=1.6.0" ] docker = [ 'docker~=3.0',