From ad35f73bdead3a04f67800d99a0b55116e4dea0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Wed, 17 Nov 2021 09:56:24 +0100 Subject: [PATCH 01/11] Update INTHEWILD.md --- INTHEWILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INTHEWILD.md b/INTHEWILD.md index 5a674333f837e..3cf5c69cabb6e 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -425,6 +425,7 @@ Currently, **officially** using Airflow: 1. [Ubisoft](https://www.ubisoft.com/) [[@Walkoss](https://github.com/Walkoss)] 1. [Udacity](https://www.udacity.com/) [[@dandikunited](https://github.com/DandikUnited), [@simon-uc](https://github.com/simon-uc)] 1. [Umami Collective](https://umamicollective.com) [[@juanuicich](https://github.com/juanuicich)] +1. [Unacast](https://unacast.com)[[@unacast](https://github.com/unacast)] 1. [United Airlines](https://www.united.com/) [[@ilopezfr](https://github.com/ilopezfr)] 1. [Upsight](https://www.upsight.com) 1. [US Bank](https://www.usbank.com) [@BShraman] From e21f563871305fbd0eafcadd3fcc8fc0f7532a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Wed, 17 Nov 2021 13:03:04 +0100 Subject: [PATCH 02/11] Enable GFM like markdown via markdown-it-py Closes: #16435 --- airflow/www/utils.py | 5 ++- setup.py | 3 ++ tests/www/test_utils.py | 84 ++++++++++++++++++++++++++--------------- 3 files changed, 59 insertions(+), 33 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 5d7368b367237..27c7569a0e66e 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional, Union from urllib.parse import urlencode -import markdown +from markdown_it import MarkdownIt import sqlalchemy as sqla from flask import Markup, Response, request, url_for from flask.helpers import flash @@ -395,10 +395,11 @@ def json_render(obj, lexer): def wrapped_markdown(s, css_class='rich_doc'): """Convert a Markdown string to HTML.""" + md = MarkdownIt("gfm-like") if s is None: return None s = textwrap.dedent(s) - return Markup(f'
' + markdown.markdown(s, extensions=['tables']) + "
") + return Markup(f'
' + md.render(s) + "
") def get_attr_renderer(): diff --git a/setup.py b/setup.py index afbea8596ad16..c478252445ea8 100644 --- a/setup.py +++ b/setup.py @@ -531,6 +531,9 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'ipdb', 'jira', 'jsondiff', + 'markdown-it-py', + 'mdit-py-plugins', + 'linkify-it-py', 'mongomock', 'moto~=2.2, >=2.2.7', 'mypy==0.770', diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 01c49e1fdcafd..d5367602bf017 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -87,11 +87,11 @@ def test_params_none_and_zero(self): def test_params_all(self): query = utils.get_params(tags=['tag1', 'tag2'], status='active', page=3, search='bash_') assert { - 'tags': ['tag1', 'tag2'], - 'page': ['3'], - 'search': ['bash_'], - 'status': ['active'], - } == parse_qs(query) + 'tags': ['tag1', 'tag2'], + 'page': ['3'], + 'search': ['bash_'], + 'status': ['active'], + } == parse_qs(query) def test_params_escape(self): assert 'search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E' == utils.get_params( @@ -160,7 +160,7 @@ def setUp(self): self.attr_renderer = utils.get_attr_renderer() def test_python_callable(self): - def example_callable(unused_self): + def example_callable(): print("example") rendered = self.attr_renderer["python_callable"](example_callable) @@ -184,29 +184,46 @@ def test_markdown_none(self): class TestWrappedMarkdown(unittest.TestCase): def test_wrapped_markdown_with_docstring_curly_braces(self): rendered = wrapped_markdown("{braces}", css_class="a_class") - assert '

{braces}

' == rendered + self.assertEqual( + '''

{braces}

+
''', rendered) def test_wrapped_markdown_with_some_markdown(self): - rendered = wrapped_markdown("*italic*\n**bold**\n", css_class="a_class") - assert ( + rendered = wrapped_markdown("""*italic* + **bold** + """, css_class="a_class") + self.assertEqual( '''

italic -bold

''' - == rendered - ) +bold

+''', rendered) def test_wrapped_markdown_with_table(self): rendered = wrapped_markdown( - """| Job | Duration | - | ----------- | ----------- | - | ETL | 14m |""" + """ +| Job | Duration | +| ----------- | ----------- | +| ETL | 14m | +""" ) - assert ( - '
\n\n\n\n' - '\n\n\n\n\n\n\n\n\n' - '
JobDuration
ETL' - '14m
' - ) == rendered + self.assertEqual( + '''
+ + + + + + + + + + + + +
JobDuration
ETL14m
+
''' + , rendered + ) def test_wrapped_markdown_with_indented_lines(self): rendered = wrapped_markdown( @@ -217,7 +234,8 @@ def test_wrapped_markdown_with_indented_lines(self): """ ) - assert '

header

\n

1st line\n2nd line

' == rendered + self.assertEqual('''

header

\n

1st line\n2nd line

+
''', rendered) def test_wrapped_markdown_with_raw_code_block(self): rendered = wrapped_markdown( @@ -234,11 +252,10 @@ def test_wrapped_markdown_with_raw_code_block(self): """ ) - assert ( - '

Markdown code block

\n' - '

Inline code works well.

\n' - '
Code block\ndoes not\nrespect\nnewlines\n
' - ) == rendered + self.assertEqual('''

Markdown code block

+

Inline code works well.

+
Code block\ndoes not\nrespect\nnewlines\n
+
''', rendered) def test_wrapped_markdown_with_nested_list(self): rendered = wrapped_markdown( @@ -250,7 +267,12 @@ def test_wrapped_markdown_with_nested_list(self): """ ) - assert ( - '

Docstring with a code block

\n' - '
    \n
  • And
      \n
    • A nested list
      • \n
    \n
    • \n
' - ) == rendered + self.assertEqual('''

Docstring with a code block

+
    +
  • And +
      +
    • A nested list
      • +
    +
    • +
+
''', rendered) From 69fa8241ebfeef8fadcb0b26c74e1e080799f5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Wed, 17 Nov 2021 16:04:53 +0100 Subject: [PATCH 03/11] Running isort on utils.py --- airflow/www/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 27c7569a0e66e..7d74717d978a4 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -21,13 +21,13 @@ from typing import Any, Dict, List, Optional, Union from urllib.parse import urlencode -from markdown_it import MarkdownIt import sqlalchemy as sqla from flask import Markup, Response, request, url_for from flask.helpers import flash from flask_appbuilder.forms import FieldConverter from flask_appbuilder.models.sqla import filters as fab_sqlafilters from flask_appbuilder.models.sqla.interface import SQLAInterface +from markdown_it import MarkdownIt from pendulum.datetime import DateTime from pygments import highlight, lexers from pygments.formatters import HtmlFormatter From 92c5d121323495bd9ecdeac4467251cae383fc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Fri, 19 Nov 2021 11:15:04 +0100 Subject: [PATCH 04/11] Run pre-commit black to format some files --- tests/www/test_utils.py | 85 ++++++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index d5367602bf017..517ad21bc6332 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -87,11 +87,11 @@ def test_params_none_and_zero(self): def test_params_all(self): query = utils.get_params(tags=['tag1', 'tag2'], status='active', page=3, search='bash_') assert { - 'tags': ['tag1', 'tag2'], - 'page': ['3'], - 'search': ['bash_'], - 'status': ['active'], - } == parse_qs(query) + 'tags': ['tag1', 'tag2'], + 'page': ['3'], + 'search': ['bash_'], + 'status': ['active'], + } == parse_qs(query) def test_params_escape(self): assert 'search=%27%3E%22%2F%3E%3Cimg+src%3Dx+onerror%3Dalert%281%29%3E' == utils.get_params( @@ -186,16 +186,23 @@ def test_wrapped_markdown_with_docstring_curly_braces(self): rendered = wrapped_markdown("{braces}", css_class="a_class") self.assertEqual( '''

{braces}

-
''', rendered) +''', + rendered, + ) def test_wrapped_markdown_with_some_markdown(self): - rendered = wrapped_markdown("""*italic* + rendered = wrapped_markdown( + """*italic* **bold** - """, css_class="a_class") + """, + css_class="a_class", + ) self.assertEqual( '''

italic bold

-
''', rendered) +''', + rendered, + ) def test_wrapped_markdown_with_table(self): rendered = wrapped_markdown( @@ -221,8 +228,8 @@ def test_wrapped_markdown_with_table(self): -''' - , rendered +''', + rendered, ) def test_wrapped_markdown_with_indented_lines(self): @@ -234,8 +241,11 @@ def test_wrapped_markdown_with_indented_lines(self): """ ) - self.assertEqual('''

header

\n

1st line\n2nd line

-
''', rendered) + self.assertEqual( + '''

header

\n

1st line\n2nd line

+
''', + rendered, + ) def test_wrapped_markdown_with_raw_code_block(self): rendered = wrapped_markdown( @@ -252,10 +262,13 @@ def test_wrapped_markdown_with_raw_code_block(self): """ ) - self.assertEqual('''

Markdown code block

+ self.assertEqual( + '''

Markdown code block

Inline code works well.

Code block\ndoes not\nrespect\nnewlines\n
-
''', rendered) +
''', + rendered, + ) def test_wrapped_markdown_with_nested_list(self): rendered = wrapped_markdown( @@ -267,7 +280,8 @@ def test_wrapped_markdown_with_nested_list(self): """ ) - self.assertEqual('''

Docstring with a code block

+ self.assertEqual( + '''

Docstring with a code block

  • And
      @@ -275,4 +289,41 @@ def test_wrapped_markdown_with_nested_list(self):
-
''', rendered) +
''', + rendered, + ) + + def test_wrapped_markdown_with_collapsible_section(self): + rendered = wrapped_markdown( + """ +# A collapsible section with markdown +
+ Click to expand! + + ## Heading + 1. A numbered + 2. list + * With some + * Sub bullets +
+ """ + ) + + self.assertEqual( + '''

A collapsible section with markdown

+
+ Click to expand! +

Heading

+
    +
  1. A numbered
    2. +
  2. list +
      +
    • With some
      • +
    • Sub bullets
      • +
    +
    3. +
+
+
''', + rendered, + ) From 8db36ddec2cbca37f8e593d72a3c2567a92c899c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Fri, 19 Nov 2021 14:59:30 +0100 Subject: [PATCH 05/11] Switch back to plain assert --- tests/www/test_utils.py | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 517ad21bc6332..d607f193d9e8f 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -184,10 +184,10 @@ def test_markdown_none(self): class TestWrappedMarkdown(unittest.TestCase): def test_wrapped_markdown_with_docstring_curly_braces(self): rendered = wrapped_markdown("{braces}", css_class="a_class") - self.assertEqual( + assert ( '''

{braces}

-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_some_markdown(self): @@ -197,11 +197,12 @@ def test_wrapped_markdown_with_some_markdown(self): """, css_class="a_class", ) - self.assertEqual( + + assert ( '''

italic bold

-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_table(self): @@ -213,7 +214,7 @@ def test_wrapped_markdown_with_table(self): """ ) - self.assertEqual( + assert ( '''
@@ -228,8 +229,8 @@ def test_wrapped_markdown_with_table(self):
-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_indented_lines(self): @@ -241,10 +242,10 @@ def test_wrapped_markdown_with_indented_lines(self): """ ) - self.assertEqual( + assert ( '''

header

\n

1st line\n2nd line

-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_raw_code_block(self): @@ -262,12 +263,12 @@ def test_wrapped_markdown_with_raw_code_block(self): """ ) - self.assertEqual( + assert ( '''

Markdown code block

Inline code works well.

Code block\ndoes not\nrespect\nnewlines\n
-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_nested_list(self): @@ -280,7 +281,7 @@ def test_wrapped_markdown_with_nested_list(self): """ ) - self.assertEqual( + assert ( '''

Docstring with a code block

  • And @@ -289,8 +290,8 @@ def test_wrapped_markdown_with_nested_list(self):
-
''', - rendered, +''' + == rendered ) def test_wrapped_markdown_with_collapsible_section(self): @@ -309,7 +310,7 @@ def test_wrapped_markdown_with_collapsible_section(self): """ ) - self.assertEqual( + assert ( '''

A collapsible section with markdown

Click to expand! @@ -324,6 +325,6 @@ def test_wrapped_markdown_with_collapsible_section(self):
-
''', - rendered, +''' + == rendered ) From aebf28d7211ef963beacd18a71895152e285f55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Thu, 10 Mar 2022 08:32:05 +0100 Subject: [PATCH 06/11] Make within
in Markdown collapsible --- airflow/www/static/css/main.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow/www/static/css/main.css b/airflow/www/static/css/main.css index ad13ac68f85db..b904c35e53a08 100644 --- a/airflow/www/static/css/main.css +++ b/airflow/www/static/css/main.css @@ -463,3 +463,7 @@ label[for="timezone-other"], .tooltip.d3-tip { z-index: 1070; } + +details summary { + display: list-item; +} From 913007f5209313166a7f21fb48bcfc280b677632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Mon, 14 Mar 2022 10:57:51 +0100 Subject: [PATCH 07/11] Update airflow/www/utils.py Following @uranusjr 's suggestion Co-authored-by: Tzu-ping Chung --- airflow/www/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index db2715b79ad8c..ffa4c1fa3a515 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -483,7 +483,7 @@ def wrapped_markdown(s, css_class='rich_doc'): if s is None: return None s = textwrap.dedent(s) - return Markup(f'
' + md.render(s) + "
") + return Markup(f'
{md.render(s)}
') def get_attr_renderer(): From 35e0df1df70e0f43e03845a484657e1a91d8cdc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Mon, 14 Mar 2022 11:00:26 +0100 Subject: [PATCH 08/11] Reintroduce unused_self in example_callable --- tests/www/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index d607f193d9e8f..8ac06f75de010 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -160,7 +160,7 @@ def setUp(self): self.attr_renderer = utils.get_attr_renderer() def test_python_callable(self): - def example_callable(): + def example_callable(unused_self): print("example") rendered = self.attr_renderer["python_callable"](example_callable) From 6f289dd07f8a0f2f75f602bdf38121dd16aaaee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Mon, 14 Mar 2022 12:43:25 +0100 Subject: [PATCH 09/11] Sort imports in utils.py --- airflow/www/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index ffa4c1fa3a515..b094ebd90296e 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -29,8 +29,8 @@ from flask_appbuilder.models.sqla import filters as fab_sqlafilters from flask_appbuilder.models.sqla.filters import get_field_setup_query, set_value_to_type from flask_appbuilder.models.sqla.interface import SQLAInterface -from markdown_it import MarkdownIt from flask_babel import lazy_gettext +from markdown_it import MarkdownIt from markupsafe import Markup from pendulum.datetime import DateTime from pygments import highlight, lexers From 7dc729c8ef1714202df82f06a6b7dbecd700e41d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Tue, 19 Apr 2022 09:15:27 +0200 Subject: [PATCH 10/11] Update setup.py Co-authored-by: Tzu-ping Chung --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fa67fbfe812fe..da306c275641d 100644 --- a/setup.py +++ b/setup.py @@ -616,9 +616,9 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'ipdb', 'jira', 'jsondiff', + 'linkify-it-py', 'markdown-it-py', 'mdit-py-plugins', - 'linkify-it-py', 'mongomock', 'moto>=3.0.7', 'parameterized', From 536c530602008b0a9da15baead3abc67e4ca0bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= Date: Fri, 17 Jun 2022 13:38:18 +0200 Subject: [PATCH 11/11] Move the mardown deps to setup.cfg --- setup.cfg | 3 +++ setup.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index bd7310499674b..8d996337ebfb2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -135,6 +135,7 @@ install_requires = # we pin to the same upper-bound as connexion. jsonschema>=3.2.0, <5.0 lazy-object-proxy + linkify-it-py>=2.0.0 lockfile>=0.12.2 markdown>=3.0 # Markupsafe 2.1.0 breaks with error: import name 'soft_unicode' from 'markupsafe'. @@ -142,8 +143,10 @@ install_requires = # https://github.com/pallets/markupsafe/issues/284 # or when we will be able to upgrade JINJA to newer version (currently limited due to Flask and # Flask Application Builder) + markdown-it-py>=2.1.0 markupsafe>=1.1.1,<2.1.0 marshmallow-oneofschema>=2.0.1 + mdit-py-plugins>=0.3.0 packaging>=14.0 pathspec~=0.9.0 pendulum>=2.0 diff --git a/setup.py b/setup.py index 8cab11401bd1d..a07368d127732 100644 --- a/setup.py +++ b/setup.py @@ -632,9 +632,6 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'ipdb', 'jira', 'jsondiff', - 'linkify-it-py', - 'markdown-it-py', - 'mdit-py-plugins', 'mongomock', 'moto[cloudformation, glue]>=3.1.12', 'parameterized',