From 698190b83056064d38dc934f6aa1f738aaa28a51 Mon Sep 17 00:00:00 2001 From: "clemens.valiente" Date: Wed, 21 Jun 2023 13:37:34 +0800 Subject: [PATCH 1/5] rebase #22854 --- airflow/www/utils.py | 30 +++++++--- airflow/www/views.py | 6 +- tests/www/views/test_views_rendered.py | 76 +++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 25fc1a28f98f9..f840d67d6a7cb 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -546,21 +546,35 @@ def pygment_html_render(s, lexer=lexers.TextLexer): return highlight(s, lexer(), HtmlFormatter(linenos=True)) -def render(obj, lexer): +def render(obj: Any, lexer, handler=None): """Render a given Python object with a given Pygments lexer.""" - out = "" + if isinstance(obj, str): - out = Markup(pygment_html_render(obj, lexer)) - elif isinstance(obj, (tuple, list)): + return Markup(pygment_html_render(obj, lexer)) + + if isinstance(obj, (tuple, list)): + out = "" for i, text_to_render in enumerate(obj): + if lexer == lexers.PythonLexer: + text_to_render = repr(text_to_render) out += Markup("
List item #{}
").format(i) out += Markup("
" + pygment_html_render(text_to_render, lexer) + "
") - elif isinstance(obj, dict): + return out + + if isinstance(obj, dict): + out = "" for k, v in obj.items(): + if lexer == lexers.PythonLexer: + v = repr(v) out += Markup('
Dict item "{}"
').format(k) out += Markup("
" + pygment_html_render(v, lexer) + "
") - return out + return out + + if obj and handler: + return Markup(pygment_html_render(handler(obj), lexer)) + # Return empty string otherwise + return "" def json_render(obj, lexer): """Render a given Python object with json lexer.""" @@ -600,8 +614,8 @@ def get_attr_renderer(): "mysql": lambda x: render(x, lexers.MySqlLexer), "postgresql": lambda x: render(x, lexers.PostgresLexer), "powershell": lambda x: render(x, lexers.PowerShellLexer), - "py": lambda x: render(get_python_source(x), lexers.PythonLexer), - "python_callable": lambda x: render(get_python_source(x), lexers.PythonLexer), + "py": lambda x: render(x, lexers.PythonLexer, get_python_source), + "python_callable": lambda x: render(x, lexers.PythonLexer, get_python_source), "rst": lambda x: render(x, lexers.RstLexer), "sql": lambda x: render(x, lexers.SqlLexer), "tsql": lambda x: render(x, lexers.TransactSqlLexer), diff --git a/airflow/www/views.py b/airflow/www/views.py index f50df518fe95a..09314e02c99bd 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -1447,11 +1447,7 @@ def rendered_templates(self, session): content = getattr(task, template_field) renderer = task.template_fields_renderers.get(template_field, template_field) if renderer in renderers: - if isinstance(content, (dict, list)): - json_content = json.dumps(content, sort_keys=True, indent=4) - html_dict[template_field] = renderers[renderer](json_content) - else: - html_dict[template_field] = renderers[renderer](content) + html_dict[template_field] = renderers[renderer](content) else: html_dict[template_field] = Markup("
{}
").format(pformat(content)) diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 6a2cb9717f7e9..1deac393ab904 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -23,8 +23,9 @@ import pytest from markupsafe import escape -from airflow.models import DAG, RenderedTaskInstanceFields, Variable +from airflow.models import DAG, RenderedTaskInstanceFields, Variable, BaseOperator from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator from airflow.serialization.serialized_objects import SerializedDAG from airflow.utils import timezone from airflow.utils.session import create_session @@ -64,6 +65,39 @@ def task2(dag): ) +@pytest.fixture() +def task3(dag): + class TestOperator(BaseOperator): + template_fields = ('sql',) + + def __init__(self, *, sql, **kwargs): + super().__init__(**kwargs) + self.sql = sql + + def execute(self, context): + pass + + return TestOperator( + task_id='task3', + sql=['SELECT 1;', 'SELECT 2;'], + dag=dag, + ) + + +@pytest.fixture() +def task4(dag): + def func(*op_args): + pass + + return PythonOperator( + task_id='task4', + python_callable=func, + op_args=['{{ task_instance_key_str }}_args'], + op_kwargs={'0': '{{ task_instance_key_str }}_kwargs'}, + dag=dag, + ) + + @pytest.fixture() def task_secret(dag): return BashOperator( @@ -108,6 +142,10 @@ def _create_dag_run(*, execution_date, session): ti2.state = TaskInstanceState.SCHEDULED ti3 = dag_run.get_task_instance(task_secret.task_id, session=session) ti3.state = TaskInstanceState.QUEUED + ti4 = dag_run.get_task_instance(task3.task_id, session=session) + ti4.state = TaskInstanceState.SUCCESS + ti5 = dag_run.get_task_instance(task4.task_id, session=session) + ti5.state = TaskInstanceState.SUCCESS session.flush() return dag_run @@ -290,3 +328,39 @@ def test_rendered_task_detail_env_secret(patch_app, admin_client, request, env, if request.node.callspec.id.endswith("-tpld-var"): Variable.delete("plain_var") Variable.delete("secret_var") + + +@pytest.mark.usefixtures("patch_app") +def test_rendered_template_view_for_list_template_field_args(admin_client, create_dag_run, task3): + """ + Test that the Rendered View can show a list of syntax-highlighted SQL statements + """ + assert task3.sql == ['SELECT 1;', 'SELECT 2;'] + + with create_session() as session: + create_dag_run(execution_date=DEFAULT_DATE, session=session) + + url = f'rendered-templates?task_id=task3&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}' + + resp = admin_client.get(url, follow_redirects=True) + check_content_in_response("List item #0", resp) + check_content_in_response("List item #1", resp) + + +@pytest.mark.usefixtures("patch_app") +def test_rendered_template_view_for_op_args(admin_client, create_dag_run, task4): + """ + Test that the Rendered View can show rendered values in op_args and op_kwargs + """ + assert task4.op_args == ['{{ task_instance_key_str }}_args'] + assert list(task4.op_kwargs.values()) == ['{{ task_instance_key_str }}_kwargs'] + + with create_session() as session: + create_dag_run(execution_date=DEFAULT_DATE, session=session) + + url = f'rendered-templates?task_id=task4&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}' + + resp = admin_client.get(url, follow_redirects=True) + check_content_in_response('testdag__task4__20200301_args', resp) + check_content_in_response('testdag__task4__20200301_kwargs', resp) + From 5a2945c4b4d570ed195183d39021d6fe8f17cea4 Mon Sep 17 00:00:00 2001 From: "clemens.valiente" Date: Wed, 21 Jun 2023 17:48:48 +0800 Subject: [PATCH 2/5] fix annotation --- airflow/www/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index f840d67d6a7cb..34fd1060c458a 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -20,7 +20,7 @@ import json import textwrap import time -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any, Sequence, Callable from urllib.parse import urlencode from flask import request, url_for @@ -36,6 +36,7 @@ from pendulum.datetime import DateTime from pygments import highlight, lexers from pygments.formatters import HtmlFormatter +from pygments.lexer import Lexer from sqlalchemy import delete, func, types from sqlalchemy.ext.associationproxy import AssociationProxy @@ -546,7 +547,7 @@ def pygment_html_render(s, lexer=lexers.TextLexer): return highlight(s, lexer(), HtmlFormatter(linenos=True)) -def render(obj: Any, lexer, handler=None): +def render(obj: Any, lexer: Lexer, handler: Callable[[Any], str] = None): """Render a given Python object with a given Pygments lexer.""" if isinstance(obj, str): @@ -576,6 +577,7 @@ def render(obj: Any, lexer, handler=None): # Return empty string otherwise return "" + def json_render(obj, lexer): """Render a given Python object with json lexer.""" out = "" From 1d741737c8be8fe013e451926acfdee7505c3b30 Mon Sep 17 00:00:00 2001 From: "clemens.valiente" Date: Fri, 30 Jun 2023 12:31:30 +0800 Subject: [PATCH 3/5] fix failed tests --- airflow/www/utils.py | 6 ++--- tests/www/views/test_views_rendered.py | 32 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 34fd1060c458a..709e960adb375 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -20,7 +20,7 @@ import json import textwrap import time -from typing import TYPE_CHECKING, Any, Sequence, Callable +from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence from urllib.parse import urlencode from flask import request, url_for @@ -547,7 +547,7 @@ def pygment_html_render(s, lexer=lexers.TextLexer): return highlight(s, lexer(), HtmlFormatter(linenos=True)) -def render(obj: Any, lexer: Lexer, handler: Callable[[Any], str] = None): +def render(obj: Any, lexer: Lexer, handler: Optional[Callable[[Any], str]] = None): """Render a given Python object with a given Pygments lexer.""" if isinstance(obj, str): @@ -571,7 +571,7 @@ def render(obj: Any, lexer: Lexer, handler: Callable[[Any], str] = None): out += Markup("
" + pygment_html_render(v, lexer) + "
") return out - if obj and handler: + if handler is not None and obj is not None: return Markup(pygment_html_render(handler(obj), lexer)) # Return empty string otherwise diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 1deac393ab904..8de375d84aac3 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -23,7 +23,7 @@ import pytest from markupsafe import escape -from airflow.models import DAG, RenderedTaskInstanceFields, Variable, BaseOperator +from airflow.models import DAG, BaseOperator, RenderedTaskInstanceFields, Variable from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.serialization.serialized_objects import SerializedDAG @@ -68,7 +68,7 @@ def task2(dag): @pytest.fixture() def task3(dag): class TestOperator(BaseOperator): - template_fields = ('sql',) + template_fields = ("sql",) def __init__(self, *, sql, **kwargs): super().__init__(**kwargs) @@ -78,8 +78,8 @@ def execute(self, context): pass return TestOperator( - task_id='task3', - sql=['SELECT 1;', 'SELECT 2;'], + task_id="task3", + sql=["SELECT 1;", "SELECT 2;"], dag=dag, ) @@ -90,10 +90,10 @@ def func(*op_args): pass return PythonOperator( - task_id='task4', + task_id="task4", python_callable=func, - op_args=['{{ task_instance_key_str }}_args'], - op_kwargs={'0': '{{ task_instance_key_str }}_kwargs'}, + op_args=["{{ task_instance_key_str }}_args"], + op_kwargs={"0": "{{ task_instance_key_str }}_kwargs"}, dag=dag, ) @@ -142,9 +142,9 @@ def _create_dag_run(*, execution_date, session): ti2.state = TaskInstanceState.SCHEDULED ti3 = dag_run.get_task_instance(task_secret.task_id, session=session) ti3.state = TaskInstanceState.QUEUED - ti4 = dag_run.get_task_instance(task3.task_id, session=session) + ti4 = dag_run.get_task_instance(task3().task_id, session=session) ti4.state = TaskInstanceState.SUCCESS - ti5 = dag_run.get_task_instance(task4.task_id, session=session) + ti5 = dag_run.get_task_instance(task4().task_id, session=session) ti5.state = TaskInstanceState.SUCCESS session.flush() return dag_run @@ -335,12 +335,12 @@ def test_rendered_template_view_for_list_template_field_args(admin_client, creat """ Test that the Rendered View can show a list of syntax-highlighted SQL statements """ - assert task3.sql == ['SELECT 1;', 'SELECT 2;'] + assert task3.sql == ["SELECT 1;", "SELECT 2;"] with create_session() as session: create_dag_run(execution_date=DEFAULT_DATE, session=session) - url = f'rendered-templates?task_id=task3&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}' + url = f"rendered-templates?task_id=task3&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("List item #0", resp) @@ -352,15 +352,15 @@ def test_rendered_template_view_for_op_args(admin_client, create_dag_run, task4) """ Test that the Rendered View can show rendered values in op_args and op_kwargs """ - assert task4.op_args == ['{{ task_instance_key_str }}_args'] - assert list(task4.op_kwargs.values()) == ['{{ task_instance_key_str }}_kwargs'] + assert task4.op_args == ["{{ task_instance_key_str }}_args"] + assert list(task4.op_kwargs.values()) == ["{{ task_instance_key_str }}_kwargs"] with create_session() as session: create_dag_run(execution_date=DEFAULT_DATE, session=session) - url = f'rendered-templates?task_id=task4&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}' + url = f"rendered-templates?task_id=task4&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) - check_content_in_response('testdag__task4__20200301_args', resp) - check_content_in_response('testdag__task4__20200301_kwargs', resp) + check_content_in_response("testdag__task4__20200301_args", resp) + check_content_in_response("testdag__task4__20200301_kwargs", resp) From 59a3f4665772d15aac2f6b310352c64ca509b56d Mon Sep 17 00:00:00 2001 From: "clemens.valiente" Date: Fri, 30 Jun 2023 12:48:11 +0800 Subject: [PATCH 4/5] code review comments --- airflow/www/utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 709e960adb375..b11381c2289ac 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -553,29 +553,30 @@ def render(obj: Any, lexer: Lexer, handler: Optional[Callable[[Any], str]] = Non if isinstance(obj, str): return Markup(pygment_html_render(obj, lexer)) - if isinstance(obj, (tuple, list)): + elif isinstance(obj, (tuple, list)): out = "" for i, text_to_render in enumerate(obj): - if lexer == lexers.PythonLexer: + if lexer is lexers.PythonLexer: text_to_render = repr(text_to_render) out += Markup("
List item #{}
").format(i) out += Markup("
" + pygment_html_render(text_to_render, lexer) + "
") return out - if isinstance(obj, dict): + elif isinstance(obj, dict): out = "" for k, v in obj.items(): - if lexer == lexers.PythonLexer: + if lexer is lexers.PythonLexer: v = repr(v) out += Markup('
Dict item "{}"
').format(k) out += Markup("
" + pygment_html_render(v, lexer) + "
") return out - if handler is not None and obj is not None: + elif handler is not None and obj is not None: return Markup(pygment_html_render(handler(obj), lexer)) - # Return empty string otherwise - return "" + else: + # Return empty string otherwise + return "" def json_render(obj, lexer): From e1514be85ea85d375dff00b2299dc91dc79456ff Mon Sep 17 00:00:00 2001 From: "clemens.valiente" Date: Fri, 30 Jun 2023 16:57:39 +0800 Subject: [PATCH 5/5] pass tests --- airflow/www/utils.py | 7 +++---- tests/www/views/test_views_rendered.py | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/airflow/www/utils.py b/airflow/www/utils.py index b11381c2289ac..76914dd9cdcd4 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -20,7 +20,7 @@ import json import textwrap import time -from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence +from typing import TYPE_CHECKING, Any, Callable, Sequence from urllib.parse import urlencode from flask import request, url_for @@ -547,9 +547,8 @@ def pygment_html_render(s, lexer=lexers.TextLexer): return highlight(s, lexer(), HtmlFormatter(linenos=True)) -def render(obj: Any, lexer: Lexer, handler: Optional[Callable[[Any], str]] = None): +def render(obj: Any, lexer: Lexer, handler: Callable[[Any], str] | None = None): """Render a given Python object with a given Pygments lexer.""" - if isinstance(obj, str): return Markup(pygment_html_render(obj, lexer)) @@ -560,7 +559,7 @@ def render(obj: Any, lexer: Lexer, handler: Optional[Callable[[Any], str]] = Non text_to_render = repr(text_to_render) out += Markup("
List item #{}
").format(i) out += Markup("
" + pygment_html_render(text_to_render, lexer) + "
") - return out + return out elif isinstance(obj, dict): out = "" diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 8de375d84aac3..1557a083d50be 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -23,7 +23,8 @@ import pytest from markupsafe import escape -from airflow.models import DAG, BaseOperator, RenderedTaskInstanceFields, Variable +from airflow.models import DAG, RenderedTaskInstanceFields, Variable +from airflow.models.baseoperator import BaseOperator from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from airflow.serialization.serialized_objects import SerializedDAG @@ -119,7 +120,7 @@ def init_blank_db(): @pytest.fixture(autouse=True) -def reset_db(dag, task1, task2, task_secret): +def reset_db(dag, task1, task2, task3, task4, task_secret): yield clear_db_dags() clear_db_runs() @@ -127,7 +128,7 @@ def reset_db(dag, task1, task2, task_secret): @pytest.fixture() -def create_dag_run(dag, task1, task2, task_secret): +def create_dag_run(dag, task1, task2, task3, task4, task_secret): def _create_dag_run(*, execution_date, session): dag_run = dag.create_dagrun( state=DagRunState.RUNNING, @@ -142,9 +143,9 @@ def _create_dag_run(*, execution_date, session): ti2.state = TaskInstanceState.SCHEDULED ti3 = dag_run.get_task_instance(task_secret.task_id, session=session) ti3.state = TaskInstanceState.QUEUED - ti4 = dag_run.get_task_instance(task3().task_id, session=session) + ti4 = dag_run.get_task_instance(task3.task_id, session=session) ti4.state = TaskInstanceState.SUCCESS - ti5 = dag_run.get_task_instance(task4().task_id, session=session) + ti5 = dag_run.get_task_instance(task4.task_id, session=session) ti5.state = TaskInstanceState.SUCCESS session.flush() return dag_run @@ -363,4 +364,3 @@ def test_rendered_template_view_for_op_args(admin_client, create_dag_run, task4) resp = admin_client.get(url, follow_redirects=True) check_content_in_response("testdag__task4__20200301_args", resp) check_content_in_response("testdag__task4__20200301_kwargs", resp) -