From 55ff7e17d7091e24f0032be587036ff4558a245d Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Thu, 2 Sep 2021 16:06:28 +0300 Subject: [PATCH 01/20] Fix ses send email and make sender parameter consistent over the email backends --- airflow/providers/amazon/aws/utils/emailer.py | 20 +++- airflow/providers/sendgrid/utils/emailer.py | 7 ++ airflow/utils/email.py | 15 ++- .../amazon/aws/utils/test_emailer.py | 96 +++++++++++++++---- .../providers/sendgrid/utils/test_emailer.py | 20 ++++ tests/utils/test_email.py | 32 +++++++ 6 files changed, 166 insertions(+), 24 deletions(-) diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index d098892d224a2..e69e595914ec7 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -16,9 +16,11 @@ # specific language governing permissions and limitations # under the License. """Airflow module for email backend using AWS SES""" - +import os +from email.utils import formataddr from typing import List, Optional, Union +from airflow.configuration import conf from airflow.providers.amazon.aws.hooks.ses import SESHook @@ -35,9 +37,23 @@ def send_email( **kwargs, ) -> None: """Email backend for SES.""" + + from_email = kwargs.get('from_email') or os.environ.get('SES_MAIL_FROM') + if not from_email and conf.has_option('ses', 'ses_mail_from'): + from_email = conf.get('ses', 'ses_mail_from') + + from_name = kwargs.get('from_name') or os.environ.get('SES_MAIL_SENDER') + if not from_name and conf.has_option('ses', 'ses_mail_sender'): + from_name = conf.get('ses', 'ses_mail_sender') + + if from_email: + mail_from = formataddr((from_name, from_email)) + else: + mail_from = None + hook = SESHook(aws_conn_id=conn_id) hook.send_email( - mail_from=None, + mail_from=mail_from, to=to, subject=subject, html_content=html_content, diff --git a/airflow/providers/sendgrid/utils/emailer.py b/airflow/providers/sendgrid/utils/emailer.py index 58a1968180914..2c55ef3e534eb 100644 --- a/airflow/providers/sendgrid/utils/emailer.py +++ b/airflow/providers/sendgrid/utils/emailer.py @@ -37,6 +37,7 @@ SandBoxMode, ) +from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.utils.email import get_email_address_list @@ -68,7 +69,13 @@ def send_email( mail = Mail() from_email = kwargs.get('from_email') or os.environ.get('SENDGRID_MAIL_FROM') + if not from_email and conf.has_option('sendgrid', 'sendgrid_mail_from'): + from_email = conf.get('sendgrid', 'sendgrid_mail_from') + from_name = kwargs.get('from_name') or os.environ.get('SENDGRID_MAIL_SENDER') + if not from_name and conf.has_option('sendgrid', 'sendgrid_mail_sender'): + from_name = conf.get('sendgrid', 'sendgrid_mail_sender') + mail.from_email = Email(from_email, from_name) mail.subject = subject mail.mail_settings = MailSettings() diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 7d17027be4307..4d9ea949641cf 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -24,7 +24,7 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formatdate +from email.utils import formatdate, formataddr from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from airflow.configuration import conf @@ -85,7 +85,18 @@ def send_email_smtp( >>> send_email('test@example.com', 'foo', 'Foo bar', ['/dev/null'], dryrun=True) """ - smtp_mail_from = conf.get('smtp', 'SMTP_MAIL_FROM') + from_email = kwargs.get('from_email') or os.environ.get('SMTP_MAIL_FROM') + if not from_email and conf.has_option('smtp', 'smtp_mail_from'): + from_email = conf.get('smtp', 'smtp_mail_from') + + from_name = kwargs.get('from_name') or os.environ.get('SMTP_MAIL_SENDER') + if not from_name and conf.has_option('smtp', 'smtp_mail_sender'): + from_name = conf.get('smtp', 'smtp_mail_sender') + + if from_email: + smtp_mail_from = formataddr((from_name, from_email)) + else: + smtp_mail_from = None msg, recipients = build_mime_message( mail_from=smtp_mail_from, diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index 3d9957393fa41..7df77ef12c6a9 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -16,27 +16,83 @@ # specific language governing permissions and limitations # under the License. # - -from unittest import mock +from unittest import mock, TestCase from airflow.providers.amazon.aws.utils.emailer import send_email +from tests.test_utils.config import conf_vars + + +class TestSendEmailSes(TestCase): + def setUp(self): + pass + + @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") + def test_send_ses_email(self, mock_hook): + send_email( + to="to@test.com", + subject="subject", + html_content="content" + ) + + mock_hook.return_value.send_email.assert_called_once_with( + mail_from=None, + to="to@test.com", + subject="subject", + html_content="content", + bcc=None, + cc=None, + files=None, + mime_charset="utf-8", + mime_subtype="mixed", + ) + + @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") + def test_send_ses_email_sender_kwargs(self, mock_hook): + send_email( + to="to@test.com", + subject="subject", + html_content="content", + from_email='from.kwargs@test.com', + from_name='From Kwargs' + ) + + _, call_args = mock_hook.return_value.send_email.call_args + + assert call_args['mail_from'] == "From Kwargs " + + @mock.patch.dict('os.environ', SES_MAIL_FROM='from.env@test.com', SES_MAIL_SENDER='From Env') + @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") + def test_send_ses_email_sender_env(self, mock_hook): + send_email( + to="to@test.com", + subject="subject", + html_content="content", + ) + + _, call_args = mock_hook.return_value.send_email.call_args + assert call_args['mail_from'] == "From Env " + + @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com', + ('ses', 'ses_mail_sender'): 'From Conf'}) + @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") + def test_send_ses_email_sender_conf(self, mock_hook): + send_email( + to="to@test.com", + subject="subject", + html_content="content", + ) + + _, call_args = mock_hook.return_value.send_email.call_args + assert call_args['mail_from'] == "From Conf " + @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com'}) + @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") + def test_send_ses_email_sender_conf_without_name(self, mock_hook): + send_email( + to="to@test.com", + subject="subject", + html_content="content", + ) -@mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") -def test_send_email(mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content", - ) - mock_hook.return_value.send_email.assert_called_once_with( - mail_from=None, - to="to@test.com", - subject="subject", - html_content="content", - bcc=None, - cc=None, - files=None, - mime_charset="utf-8", - mime_subtype="mixed", - ) + _, call_args = mock_hook.return_value.send_email.call_args + assert call_args['mail_from'] == "from.conf@test.com" diff --git a/tests/providers/sendgrid/utils/test_emailer.py b/tests/providers/sendgrid/utils/test_emailer.py index a2b0f2823b5cc..84cbb83b67e69 100644 --- a/tests/providers/sendgrid/utils/test_emailer.py +++ b/tests/providers/sendgrid/utils/test_emailer.py @@ -24,6 +24,7 @@ from unittest import mock from airflow.providers.sendgrid.utils.emailer import send_email +from tests.test_utils.config import conf_vars class TestSendEmailSendGrid(unittest.TestCase): @@ -64,6 +65,12 @@ def setUp(self): 'name': 'Foo Bar', 'email': 'foo@foo.bar', } + # sender from conf + self.expected_mail_data_conf_sender = copy.deepcopy(self.expected_mail_data) + self.expected_mail_data_conf_sender['from'] = { + 'name': 'Foo Conf', + 'email': 'foo@conf.com', + } # Test the right email is constructed. @mock.patch.dict('os.environ', SENDGRID_MAIL_FROM='foo@bar.com') @@ -125,3 +132,16 @@ def test_send_email_sendgrid_sender(self, mock_post): from_name='Foo Bar', ) mock_post.assert_called_once_with(self.expected_mail_data_sender, "sendgrid_default") + + @mock.patch('airflow.providers.sendgrid.utils.emailer._post_sendgrid_mail') + @conf_vars({("sendgrid", 'sendgrid_mail_from'): 'foo@conf.com', + ("sendgrid", 'sendgrid_mail_sender'): 'Foo Conf'}) + def test_send_email_sendgrid_sender_conf(self, mock_post): + send_email( + self.recipients, + self.subject, + self.html_content, + cc=self.carbon_copy, + bcc=self.bcc + ) + mock_post.assert_called_once_with(self.expected_mail_data_conf_sender, "sendgrid_default") diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 28d43284ee8d7..91097069eba30 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -334,3 +334,35 @@ def test_send_mime_partial_failure(self, mock_smtp, mock_smtp_ssl): assert final_mock.starttls.called final_mock.sendmail.assert_called_once_with('from', 'to', msg.as_string()) assert final_mock.quit.called + + @mock.patch('airflow.utils.email.send_mime_email') + def test_send_smtp_correct_sender_from_kwargs(self, mock_send_mime): + utils.email.send_email_smtp('to', 'subject', 'content', from_email='from.kwargs@test.com', from_name='From Kwargs') + assert mock_send_mime.called + _, call_args = mock_send_mime.call_args + assert call_args['e_from'] == 'From Kwargs ' + + @mock.patch('airflow.utils.email.send_mime_email') + @mock.patch.dict('os.environ', SMTP_MAIL_FROM='from.env@test.com', SMTP_MAIL_SENDER='From Env') + def test_send_smtp_correct_sender_from_env(self, mock_send_mime): + utils.email.send_email_smtp('to', 'subject', 'content') + assert mock_send_mime.called + _, call_args = mock_send_mime.call_args + assert call_args['e_from'] == 'From Env ' + + @mock.patch('airflow.utils.email.send_mime_email') + @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com', + ('smtp', 'smtp_mail_sender'): 'From Conf'}) + def test_send_smtp_correct_sender_from_conf(self, mock_send_mime): + utils.email.send_email_smtp('to', 'subject', 'content') + assert mock_send_mime.called + _, call_args = mock_send_mime.call_args + assert call_args['e_from'] == 'From Conf ' + + @mock.patch('airflow.utils.email.send_mime_email') + @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com'}) + def test_send_smtp_correct_sender_from_conf_no_name(self, mock_send_mime): + utils.email.send_email_smtp('to', 'subject', 'content') + assert mock_send_mime.called + _, call_args = mock_send_mime.call_args + assert call_args['e_from'] == 'from.conf@test.com' From 1b3605a0849e075dc913e8f143e299b4fd2d085f Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Mon, 6 Sep 2021 10:03:57 +0300 Subject: [PATCH 02/20] Changes after running pre-commit hooks --- airflow/providers/amazon/aws/utils/emailer.py | 1 - airflow/utils/email.py | 2 +- docs/apache-airflow/howto/email-config.rst | 9 +++++++++ tests/providers/amazon/aws/utils/test_emailer.py | 13 ++++--------- tests/providers/sendgrid/utils/test_emailer.py | 13 ++++--------- tests/utils/test_email.py | 7 ++++--- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index e69e595914ec7..5b3b742cb450c 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -37,7 +37,6 @@ def send_email( **kwargs, ) -> None: """Email backend for SES.""" - from_email = kwargs.get('from_email') or os.environ.get('SES_MAIL_FROM') if not from_email and conf.has_option('ses', 'ses_mail_from'): from_email = conf.get('ses', 'ses_mail_from') diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 4d9ea949641cf..28ae7024150ae 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -24,7 +24,7 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formatdate, formataddr +from email.utils import formataddr, formatdate from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from airflow.configuration import conf diff --git a/docs/apache-airflow/howto/email-config.rst b/docs/apache-airflow/howto/email-config.rst index 67e26a7ca8a59..c34d66927374e 100644 --- a/docs/apache-airflow/howto/email-config.rst +++ b/docs/apache-airflow/howto/email-config.rst @@ -55,6 +55,9 @@ For example a ``html_content_template`` file could look like this: Host: {{ti.hostname}}
Mark success: Link
+You can configure sender's email address and name either by exporting the environment variables ``SMTP_MAIL_FROM`` and ``SMTP_MAIL_SENDER`` or +in your ``airflow.cfg`` by setting ``smtp_mail_from`` and ``smtp_mail_sender`` in the ``[smtp]`` section. + .. note:: For more information on setting the configuration, see :doc:`set-config` @@ -91,6 +94,9 @@ or name and set it in ``email_conn_id`` of 'Email' type. Only login and password are used from the connection. +4. Configure sender's email address and name either by exporting the environment variables ``SENDGRID_MAIL_FROM`` and ``SENDGRID_MAIL_SENDER`` or + in your ``airflow.cfg`` by setting ``sendgrid_mail_from`` and ``sendgrid_mail_sender`` in the ``[sendgrid]`` section. + .. _email-configuration-ses: Send email using AWS SES @@ -116,3 +122,6 @@ Follow the steps below to enable it: 3. Create a connection called ``aws_default``, or choose a custom connection name and set it in ``email_conn_id``. The type of connection should be ``Amazon Web Services``. + +4. Configure sender's email address and name either by exporting the environment variables ``SES_MAIL_FROM`` and ``SES_MAIL_SENDER`` or + in your ``airflow.cfg`` by setting ``ses_mail_from`` and ``ses_mail_sender`` in the ``[ses]`` section. diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index 7df77ef12c6a9..80c2e187a2089 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -from unittest import mock, TestCase +from unittest import TestCase, mock from airflow.providers.amazon.aws.utils.emailer import send_email from tests.test_utils.config import conf_vars @@ -28,11 +28,7 @@ def setUp(self): @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email(self, mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content" - ) + send_email(to="to@test.com", subject="subject", html_content="content") mock_hook.return_value.send_email.assert_called_once_with( mail_from=None, @@ -53,7 +49,7 @@ def test_send_ses_email_sender_kwargs(self, mock_hook): subject="subject", html_content="content", from_email='from.kwargs@test.com', - from_name='From Kwargs' + from_name='From Kwargs', ) _, call_args = mock_hook.return_value.send_email.call_args @@ -72,8 +68,7 @@ def test_send_ses_email_sender_env(self, mock_hook): _, call_args = mock_hook.return_value.send_email.call_args assert call_args['mail_from'] == "From Env " - @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com', - ('ses', 'ses_mail_sender'): 'From Conf'}) + @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com', ('ses', 'ses_mail_sender'): 'From Conf'}) @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email_sender_conf(self, mock_hook): send_email( diff --git a/tests/providers/sendgrid/utils/test_emailer.py b/tests/providers/sendgrid/utils/test_emailer.py index 84cbb83b67e69..969a5395b8691 100644 --- a/tests/providers/sendgrid/utils/test_emailer.py +++ b/tests/providers/sendgrid/utils/test_emailer.py @@ -134,14 +134,9 @@ def test_send_email_sendgrid_sender(self, mock_post): mock_post.assert_called_once_with(self.expected_mail_data_sender, "sendgrid_default") @mock.patch('airflow.providers.sendgrid.utils.emailer._post_sendgrid_mail') - @conf_vars({("sendgrid", 'sendgrid_mail_from'): 'foo@conf.com', - ("sendgrid", 'sendgrid_mail_sender'): 'Foo Conf'}) + @conf_vars( + {("sendgrid", 'sendgrid_mail_from'): 'foo@conf.com', ("sendgrid", 'sendgrid_mail_sender'): 'Foo Conf'} + ) def test_send_email_sendgrid_sender_conf(self, mock_post): - send_email( - self.recipients, - self.subject, - self.html_content, - cc=self.carbon_copy, - bcc=self.bcc - ) + send_email(self.recipients, self.subject, self.html_content, cc=self.carbon_copy, bcc=self.bcc) mock_post.assert_called_once_with(self.expected_mail_data_conf_sender, "sendgrid_default") diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 91097069eba30..70960efbdf85d 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -337,7 +337,9 @@ def test_send_mime_partial_failure(self, mock_smtp, mock_smtp_ssl): @mock.patch('airflow.utils.email.send_mime_email') def test_send_smtp_correct_sender_from_kwargs(self, mock_send_mime): - utils.email.send_email_smtp('to', 'subject', 'content', from_email='from.kwargs@test.com', from_name='From Kwargs') + utils.email.send_email_smtp( + 'to', 'subject', 'content', from_email='from.kwargs@test.com', from_name='From Kwargs' + ) assert mock_send_mime.called _, call_args = mock_send_mime.call_args assert call_args['e_from'] == 'From Kwargs ' @@ -351,8 +353,7 @@ def test_send_smtp_correct_sender_from_env(self, mock_send_mime): assert call_args['e_from'] == 'From Env ' @mock.patch('airflow.utils.email.send_mime_email') - @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com', - ('smtp', 'smtp_mail_sender'): 'From Conf'}) + @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com', ('smtp', 'smtp_mail_sender'): 'From Conf'}) def test_send_smtp_correct_sender_from_conf(self, mock_send_mime): utils.email.send_email_smtp('to', 'subject', 'content') assert mock_send_mime.called From 93b398242757590e08791eeb94abd8663b9f98f4 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Mon, 6 Sep 2021 16:59:36 +0300 Subject: [PATCH 03/20] Moved repeatable logic to single code piece --- airflow/providers/amazon/aws/utils/emailer.py | 19 ++---- airflow/providers/sendgrid/utils/emailer.py | 5 -- airflow/utils/email.py | 26 ++++---- .../amazon/aws/utils/test_emailer.py | 61 +++---------------- .../providers/sendgrid/utils/test_emailer.py | 9 --- tests/utils/test_email.py | 55 ++++++----------- 6 files changed, 45 insertions(+), 130 deletions(-) diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index 5b3b742cb450c..66cf695fcc24d 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -16,15 +16,15 @@ # specific language governing permissions and limitations # under the License. """Airflow module for email backend using AWS SES""" -import os from email.utils import formataddr from typing import List, Optional, Union -from airflow.configuration import conf from airflow.providers.amazon.aws.hooks.ses import SESHook def send_email( + from_email: str, + from_name: str, to: Union[List[str], str], subject: str, html_content: str, @@ -37,22 +37,11 @@ def send_email( **kwargs, ) -> None: """Email backend for SES.""" - from_email = kwargs.get('from_email') or os.environ.get('SES_MAIL_FROM') - if not from_email and conf.has_option('ses', 'ses_mail_from'): - from_email = conf.get('ses', 'ses_mail_from') - - from_name = kwargs.get('from_name') or os.environ.get('SES_MAIL_SENDER') - if not from_name and conf.has_option('ses', 'ses_mail_sender'): - from_name = conf.get('ses', 'ses_mail_sender') - - if from_email: - mail_from = formataddr((from_name, from_email)) - else: - mail_from = None + from_formatted = formataddr((from_name, from_email)) hook = SESHook(aws_conn_id=conn_id) hook.send_email( - mail_from=mail_from, + mail_from=from_formatted, to=to, subject=subject, html_content=html_content, diff --git a/airflow/providers/sendgrid/utils/emailer.py b/airflow/providers/sendgrid/utils/emailer.py index 2c55ef3e534eb..6ae2440f9d341 100644 --- a/airflow/providers/sendgrid/utils/emailer.py +++ b/airflow/providers/sendgrid/utils/emailer.py @@ -37,7 +37,6 @@ SandBoxMode, ) -from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.hooks.base import BaseHook from airflow.utils.email import get_email_address_list @@ -69,12 +68,8 @@ def send_email( mail = Mail() from_email = kwargs.get('from_email') or os.environ.get('SENDGRID_MAIL_FROM') - if not from_email and conf.has_option('sendgrid', 'sendgrid_mail_from'): - from_email = conf.get('sendgrid', 'sendgrid_mail_from') from_name = kwargs.get('from_name') or os.environ.get('SENDGRID_MAIL_SENDER') - if not from_name and conf.has_option('sendgrid', 'sendgrid_mail_sender'): - from_name = conf.get('sendgrid', 'sendgrid_mail_sender') mail.from_email = Email(from_email, from_name) mail.subject = subject diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 28ae7024150ae..f89e048fe33b2 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -49,10 +49,17 @@ def send_email( """Send email using backend specified in EMAIL_BACKEND.""" backend = conf.getimport('email', 'EMAIL_BACKEND') backend_conn_id = conn_id or conf.get("email", "EMAIL_CONN_ID") + from_email = ( + conf.get('email', 'email_from_email') if conf.has_option('email', 'email_from_email') else None + ) + from_name = conf.get('email', 'email_from_name') if conf.has_option('email', 'email_from_name') else None + to_list = get_email_address_list(to) to_comma_separated = ", ".join(to_list) return backend( + from_email, + from_name, to_comma_separated, subject, html_content, @@ -68,6 +75,8 @@ def send_email( def send_email_smtp( + from_email: str, + from_name: str, to: Union[str, Iterable[str]], subject: str, html_content: str, @@ -85,21 +94,12 @@ def send_email_smtp( >>> send_email('test@example.com', 'foo', 'Foo bar', ['/dev/null'], dryrun=True) """ - from_email = kwargs.get('from_email') or os.environ.get('SMTP_MAIL_FROM') - if not from_email and conf.has_option('smtp', 'smtp_mail_from'): - from_email = conf.get('smtp', 'smtp_mail_from') - - from_name = kwargs.get('from_name') or os.environ.get('SMTP_MAIL_SENDER') - if not from_name and conf.has_option('smtp', 'smtp_mail_sender'): - from_name = conf.get('smtp', 'smtp_mail_sender') + smtp_mail_from = conf.get('smtp', 'SMTP_MAIL_FROM') - if from_email: - smtp_mail_from = formataddr((from_name, from_email)) - else: - smtp_mail_from = None + from_formatted = formataddr((from_name, smtp_mail_from or from_email)) msg, recipients = build_mime_message( - mail_from=smtp_mail_from, + mail_from=from_formatted, to=to, subject=subject, html_content=html_content, @@ -110,7 +110,7 @@ def send_email_smtp( mime_charset=mime_charset, ) - send_mime_email(e_from=smtp_mail_from, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun) + send_mime_email(e_from=from_formatted, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun) def build_mime_message( diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index 80c2e187a2089..ded7b2d8bdc8f 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -26,12 +26,19 @@ class TestSendEmailSes(TestCase): def setUp(self): pass + @conf_vars({('email', 'email_from_email'): 'from@test.com', ('email', 'email_from_name'): 'From Test'}) @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email(self, mock_hook): - send_email(to="to@test.com", subject="subject", html_content="content") + send_email( + from_email="from@test.com", + from_name="From Test", + to="to@test.com", + subject="subject", + html_content="content", + ) mock_hook.return_value.send_email.assert_called_once_with( - mail_from=None, + mail_from="From Test ", to="to@test.com", subject="subject", html_content="content", @@ -41,53 +48,3 @@ def test_send_ses_email(self, mock_hook): mime_charset="utf-8", mime_subtype="mixed", ) - - @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") - def test_send_ses_email_sender_kwargs(self, mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content", - from_email='from.kwargs@test.com', - from_name='From Kwargs', - ) - - _, call_args = mock_hook.return_value.send_email.call_args - - assert call_args['mail_from'] == "From Kwargs " - - @mock.patch.dict('os.environ', SES_MAIL_FROM='from.env@test.com', SES_MAIL_SENDER='From Env') - @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") - def test_send_ses_email_sender_env(self, mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content", - ) - - _, call_args = mock_hook.return_value.send_email.call_args - assert call_args['mail_from'] == "From Env " - - @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com', ('ses', 'ses_mail_sender'): 'From Conf'}) - @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") - def test_send_ses_email_sender_conf(self, mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content", - ) - - _, call_args = mock_hook.return_value.send_email.call_args - assert call_args['mail_from'] == "From Conf " - - @conf_vars({('ses', 'ses_mail_from'): 'from.conf@test.com'}) - @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") - def test_send_ses_email_sender_conf_without_name(self, mock_hook): - send_email( - to="to@test.com", - subject="subject", - html_content="content", - ) - - _, call_args = mock_hook.return_value.send_email.call_args - assert call_args['mail_from'] == "from.conf@test.com" diff --git a/tests/providers/sendgrid/utils/test_emailer.py b/tests/providers/sendgrid/utils/test_emailer.py index 969a5395b8691..5d2d625906765 100644 --- a/tests/providers/sendgrid/utils/test_emailer.py +++ b/tests/providers/sendgrid/utils/test_emailer.py @@ -24,7 +24,6 @@ from unittest import mock from airflow.providers.sendgrid.utils.emailer import send_email -from tests.test_utils.config import conf_vars class TestSendEmailSendGrid(unittest.TestCase): @@ -132,11 +131,3 @@ def test_send_email_sendgrid_sender(self, mock_post): from_name='Foo Bar', ) mock_post.assert_called_once_with(self.expected_mail_data_sender, "sendgrid_default") - - @mock.patch('airflow.providers.sendgrid.utils.emailer._post_sendgrid_mail') - @conf_vars( - {("sendgrid", 'sendgrid_mail_from'): 'foo@conf.com', ("sendgrid", 'sendgrid_mail_sender'): 'Foo Conf'} - ) - def test_send_email_sendgrid_sender_conf(self, mock_post): - send_email(self.recipients, self.subject, self.html_content, cc=self.carbon_copy, bcc=self.bcc) - mock_post.assert_called_once_with(self.expected_mail_data_conf_sender, "sendgrid_default") diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 70960efbdf85d..4d331113df6a1 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -89,6 +89,8 @@ def test_custom_backend(self, mock_send_email): with conf_vars({('email', 'email_backend'): 'tests.utils.test_email.send_email_test'}): utils.email.send_email('to', 'subject', 'content') send_email_test.assert_called_once_with( + None, + None, 'to', 'subject', 'content', @@ -102,6 +104,20 @@ def test_custom_backend(self, mock_send_email): ) assert not mock_send_email.called + @mock.patch('airflow.utils.email.send_email_smtp') + @conf_vars( + { + ('email', 'email_backend'): 'tests.utils.test_email.send_email_test', + ('email', 'email_from_email'): 'from@test.com', + ('email', 'email_from_name'): 'From Test', + } + ) + def test_custom_backend_sender(self, mock_send_email_smtp): + utils.email.send_email('to', 'subject', 'content') + call_args, _ = send_email_test.call_args + assert call_args == ('from@test.com', 'From Test', 'to', 'subject', 'content') + assert not mock_send_email_smtp.called + def test_build_mime_message(self): mail_from = 'from@example.com' mail_to = 'to@example.com' @@ -131,7 +147,7 @@ def test_send_smtp(self, mock_send_mime): with tempfile.NamedTemporaryFile() as attachment: attachment.write(b'attachment') attachment.seek(0) - utils.email.send_email_smtp('to', 'subject', 'content', files=[attachment.name]) + utils.email.send_email_smtp(None, None, 'to', 'subject', 'content', files=[attachment.name]) assert mock_send_mime.called _, call_args = mock_send_mime.call_args assert conf.get('smtp', 'SMTP_MAIL_FROM') == call_args['e_from'] @@ -147,7 +163,7 @@ def test_send_smtp(self, mock_send_mime): @mock.patch('airflow.utils.email.send_mime_email') def test_send_smtp_with_multibyte_content(self, mock_send_mime): - utils.email.send_email_smtp('to', 'subject', '🔥', mime_charset='utf-8') + utils.email.send_email_smtp(None, None, 'to', 'subject', '🔥', mime_charset='utf-8') assert mock_send_mime.called _, call_args = mock_send_mime.call_args msg = call_args['mime_msg'] @@ -160,7 +176,7 @@ def test_send_bcc_smtp(self, mock_send_mime): attachment.write(b'attachment') attachment.seek(0) utils.email.send_email_smtp( - 'to', 'subject', 'content', files=[attachment.name], cc='cc', bcc='bcc' + None, None, 'to', 'subject', 'content', files=[attachment.name], cc='cc', bcc='bcc' ) assert mock_send_mime.called _, call_args = mock_send_mime.call_args @@ -334,36 +350,3 @@ def test_send_mime_partial_failure(self, mock_smtp, mock_smtp_ssl): assert final_mock.starttls.called final_mock.sendmail.assert_called_once_with('from', 'to', msg.as_string()) assert final_mock.quit.called - - @mock.patch('airflow.utils.email.send_mime_email') - def test_send_smtp_correct_sender_from_kwargs(self, mock_send_mime): - utils.email.send_email_smtp( - 'to', 'subject', 'content', from_email='from.kwargs@test.com', from_name='From Kwargs' - ) - assert mock_send_mime.called - _, call_args = mock_send_mime.call_args - assert call_args['e_from'] == 'From Kwargs ' - - @mock.patch('airflow.utils.email.send_mime_email') - @mock.patch.dict('os.environ', SMTP_MAIL_FROM='from.env@test.com', SMTP_MAIL_SENDER='From Env') - def test_send_smtp_correct_sender_from_env(self, mock_send_mime): - utils.email.send_email_smtp('to', 'subject', 'content') - assert mock_send_mime.called - _, call_args = mock_send_mime.call_args - assert call_args['e_from'] == 'From Env ' - - @mock.patch('airflow.utils.email.send_mime_email') - @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com', ('smtp', 'smtp_mail_sender'): 'From Conf'}) - def test_send_smtp_correct_sender_from_conf(self, mock_send_mime): - utils.email.send_email_smtp('to', 'subject', 'content') - assert mock_send_mime.called - _, call_args = mock_send_mime.call_args - assert call_args['e_from'] == 'From Conf ' - - @mock.patch('airflow.utils.email.send_mime_email') - @conf_vars({('smtp', 'smtp_mail_from'): 'from.conf@test.com'}) - def test_send_smtp_correct_sender_from_conf_no_name(self, mock_send_mime): - utils.email.send_email_smtp('to', 'subject', 'content') - assert mock_send_mime.called - _, call_args = mock_send_mime.call_args - assert call_args['e_from'] == 'from.conf@test.com' From 2b4fd5d0b081fdc796fcd52f302656fac9d43bf5 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Mon, 6 Sep 2021 17:25:49 +0300 Subject: [PATCH 04/20] Updated documentation --- docs/apache-airflow/howto/email-config.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/apache-airflow/howto/email-config.rst b/docs/apache-airflow/howto/email-config.rst index c34d66927374e..2c67970e93163 100644 --- a/docs/apache-airflow/howto/email-config.rst +++ b/docs/apache-airflow/howto/email-config.rst @@ -29,6 +29,8 @@ in the ``[email]`` section. subject_template = /path/to/my_subject_template_file html_content_template = /path/to/my_html_content_template_file +You can configure sender's email address and name by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. + To configure SMTP settings, checkout the :ref:`SMTP ` section in the standard configuration. If you do not want to store the SMTP credentials in the config or in the environment variables, you can create a connection called ``smtp_default`` of ``Email`` type, or choose a custom connection name and set the ``email_conn_id`` with it's name in @@ -55,9 +57,6 @@ For example a ``html_content_template`` file could look like this: Host: {{ti.hostname}}
Mark success: Link
-You can configure sender's email address and name either by exporting the environment variables ``SMTP_MAIL_FROM`` and ``SMTP_MAIL_SENDER`` or -in your ``airflow.cfg`` by setting ``smtp_mail_from`` and ``smtp_mail_sender`` in the ``[smtp]`` section. - .. note:: For more information on setting the configuration, see :doc:`set-config` @@ -95,7 +94,7 @@ or are used from the connection. 4. Configure sender's email address and name either by exporting the environment variables ``SENDGRID_MAIL_FROM`` and ``SENDGRID_MAIL_SENDER`` or - in your ``airflow.cfg`` by setting ``sendgrid_mail_from`` and ``sendgrid_mail_sender`` in the ``[sendgrid]`` section. + in your ``airflow.cfg`` by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. .. _email-configuration-ses: @@ -123,5 +122,4 @@ Follow the steps below to enable it: 3. Create a connection called ``aws_default``, or choose a custom connection name and set it in ``email_conn_id``. The type of connection should be ``Amazon Web Services``. -4. Configure sender's email address and name either by exporting the environment variables ``SES_MAIL_FROM`` and ``SES_MAIL_SENDER`` or - in your ``airflow.cfg`` by setting ``ses_mail_from`` and ``ses_mail_sender`` in the ``[ses]`` section. +4. Configure sender's email address and name in your ``airflow.cfg`` by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. From 973ed58b8e2a6865b4da2d5ab7478d0168457c6c Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Mon, 6 Sep 2021 17:46:43 +0300 Subject: [PATCH 05/20] Removed blank lines --- airflow/providers/sendgrid/utils/emailer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airflow/providers/sendgrid/utils/emailer.py b/airflow/providers/sendgrid/utils/emailer.py index 6ae2440f9d341..58a1968180914 100644 --- a/airflow/providers/sendgrid/utils/emailer.py +++ b/airflow/providers/sendgrid/utils/emailer.py @@ -68,9 +68,7 @@ def send_email( mail = Mail() from_email = kwargs.get('from_email') or os.environ.get('SENDGRID_MAIL_FROM') - from_name = kwargs.get('from_name') or os.environ.get('SENDGRID_MAIL_SENDER') - mail.from_email = Email(from_email, from_name) mail.subject = subject mail.mail_settings = MailSettings() From 46136473dabca71a588dd8868669b6a4977f4077 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Thu, 16 Sep 2021 13:04:19 +0300 Subject: [PATCH 06/20] Added email sender config to config templates; Added fallback value when getting email config to have clearer code --- airflow/config_templates/config.yml | 14 ++++++++++++++ airflow/config_templates/default_airflow.cfg | 8 ++++++++ airflow/utils/email.py | 6 ++---- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 6af5a7dec4280..941e55b45e6ed 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1350,6 +1350,20 @@ example: "/path/to/my_html_content_template_file" default: ~ see_also: ":doc:`Email Configuration `" + - name: email_from_email + description: | + Email address that will be used as sender address. + version_added: 2.2.0 + type: string + example: "airflow@example.com" + default: ~ + - name: email_from_name + description: | + Display name for the sender address. + version_added: 2.2.0 + type: string + example: "Airflow Notifications" + default: ~ - name: smtp description: | diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index d8d5fa250f353..a6d1ce8ad63cc 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -678,6 +678,14 @@ default_email_on_failure = True # Example: html_content_template = /path/to/my_html_content_template_file # html_content_template = +# Email address that will be used as sender address. +# Example: email_from_email = airflow@example.com +# from_email = + +# Display name for the sender address. +# Example: email_from_name = Airflow Notifications +# from_name = + [smtp] # If you want airflow to send emails on retries, failure, and you want to use diff --git a/airflow/utils/email.py b/airflow/utils/email.py index f89e048fe33b2..eb8f66ae55e89 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -49,10 +49,8 @@ def send_email( """Send email using backend specified in EMAIL_BACKEND.""" backend = conf.getimport('email', 'EMAIL_BACKEND') backend_conn_id = conn_id or conf.get("email", "EMAIL_CONN_ID") - from_email = ( - conf.get('email', 'email_from_email') if conf.has_option('email', 'email_from_email') else None - ) - from_name = conf.get('email', 'email_from_name') if conf.has_option('email', 'email_from_name') else None + from_email = conf.get('email', 'email_from_email', fallback=None) + from_name = conf.get('email', 'email_from_name', fallback=None) to_list = get_email_address_list(to) to_comma_separated = ", ".join(to_list) From 4849562c88b1c8c9efcdf8fcae505acbddac32a1 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 08:37:43 +0300 Subject: [PATCH 07/20] Fixed mismatching config names --- airflow/config_templates/default_airflow.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index a6d1ce8ad63cc..f61cfd154e34d 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -680,11 +680,11 @@ default_email_on_failure = True # Email address that will be used as sender address. # Example: email_from_email = airflow@example.com -# from_email = +# email_from_email = # Display name for the sender address. # Example: email_from_name = Airflow Notifications -# from_name = +# email_from_name = [smtp] From 8a4e48f26204859bf5efcbb44ddc18a1971f31aa Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 08:58:45 +0300 Subject: [PATCH 08/20] Added backwards compatibility to send_email_smtp --- airflow/utils/email.py | 8 ++++---- tests/utils/test_email.py | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/airflow/utils/email.py b/airflow/utils/email.py index eb8f66ae55e89..9b9f9b1e21ca0 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -56,8 +56,6 @@ def send_email( to_comma_separated = ", ".join(to_list) return backend( - from_email, - from_name, to_comma_separated, subject, html_content, @@ -68,13 +66,13 @@ def send_email( mime_subtype=mime_subtype, mime_charset=mime_charset, conn_id=backend_conn_id, + from_email=from_email, + from_name=from_name, **kwargs, ) def send_email_smtp( - from_email: str, - from_name: str, to: Union[str, Iterable[str]], subject: str, html_content: str, @@ -85,6 +83,8 @@ def send_email_smtp( mime_subtype: str = 'mixed', mime_charset: str = 'utf-8', conn_id: str = "smtp_default", + from_email: str = None, + from_name: str = None, **kwargs, ): """ diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 4d331113df6a1..71616866cf9b7 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -89,8 +89,6 @@ def test_custom_backend(self, mock_send_email): with conf_vars({('email', 'email_backend'): 'tests.utils.test_email.send_email_test'}): utils.email.send_email('to', 'subject', 'content') send_email_test.assert_called_once_with( - None, - None, 'to', 'subject', 'content', @@ -101,6 +99,8 @@ def test_custom_backend(self, mock_send_email): mime_charset='utf-8', mime_subtype='mixed', conn_id='smtp_default', + from_email=None, + from_name=None ) assert not mock_send_email.called @@ -114,8 +114,9 @@ def test_custom_backend(self, mock_send_email): ) def test_custom_backend_sender(self, mock_send_email_smtp): utils.email.send_email('to', 'subject', 'content') - call_args, _ = send_email_test.call_args - assert call_args == ('from@test.com', 'From Test', 'to', 'subject', 'content') + _, call_kwargs = send_email_test.call_args + assert call_kwargs['from_email'] == 'from@test.com' + assert call_kwargs['from_name'] == 'From Test' assert not mock_send_email_smtp.called def test_build_mime_message(self): @@ -147,7 +148,7 @@ def test_send_smtp(self, mock_send_mime): with tempfile.NamedTemporaryFile() as attachment: attachment.write(b'attachment') attachment.seek(0) - utils.email.send_email_smtp(None, None, 'to', 'subject', 'content', files=[attachment.name]) + utils.email.send_email_smtp('to', 'subject', 'content', files=[attachment.name]) assert mock_send_mime.called _, call_args = mock_send_mime.call_args assert conf.get('smtp', 'SMTP_MAIL_FROM') == call_args['e_from'] @@ -163,7 +164,7 @@ def test_send_smtp(self, mock_send_mime): @mock.patch('airflow.utils.email.send_mime_email') def test_send_smtp_with_multibyte_content(self, mock_send_mime): - utils.email.send_email_smtp(None, None, 'to', 'subject', '🔥', mime_charset='utf-8') + utils.email.send_email_smtp('to', 'subject', '🔥', mime_charset='utf-8') assert mock_send_mime.called _, call_args = mock_send_mime.call_args msg = call_args['mime_msg'] @@ -176,7 +177,7 @@ def test_send_bcc_smtp(self, mock_send_mime): attachment.write(b'attachment') attachment.seek(0) utils.email.send_email_smtp( - None, None, 'to', 'subject', 'content', files=[attachment.name], cc='cc', bcc='bcc' + 'to', 'subject', 'content', files=[attachment.name], cc='cc', bcc='bcc' ) assert mock_send_mime.called _, call_args = mock_send_mime.call_args From ec024b9be1cd86b59c71863d4e46e9c978d1fee4 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 09:42:44 +0300 Subject: [PATCH 09/20] Fixed method call --- tests/utils/test_email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 71616866cf9b7..0090fde66e0a9 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -100,7 +100,7 @@ def test_custom_backend(self, mock_send_email): mime_subtype='mixed', conn_id='smtp_default', from_email=None, - from_name=None + from_name=None, ) assert not mock_send_email.called From 56c841d8d747dae54514501ff1dffe980cc12a4a Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 10:45:06 +0300 Subject: [PATCH 10/20] Removed email_from_name argument and renamed email_from_email to from_email --- airflow/config_templates/config.yml | 9 +-------- airflow/config_templates/default_airflow.cfg | 8 ++------ airflow/providers/amazon/aws/utils/emailer.py | 4 +--- airflow/utils/email.py | 11 ++++------- docs/apache-airflow/howto/email-config.rst | 6 +++--- tests/utils/test_email.py | 5 +---- 6 files changed, 12 insertions(+), 31 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 941e55b45e6ed..33036481c4a02 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1350,20 +1350,13 @@ example: "/path/to/my_html_content_template_file" default: ~ see_also: ":doc:`Email Configuration `" - - name: email_from_email + - name: from_email description: | Email address that will be used as sender address. version_added: 2.2.0 type: string example: "airflow@example.com" default: ~ - - name: email_from_name - description: | - Display name for the sender address. - version_added: 2.2.0 - type: string - example: "Airflow Notifications" - default: ~ - name: smtp description: | diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index f61cfd154e34d..aaab2a4d8af49 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -679,12 +679,8 @@ default_email_on_failure = True # html_content_template = # Email address that will be used as sender address. -# Example: email_from_email = airflow@example.com -# email_from_email = - -# Display name for the sender address. -# Example: email_from_name = Airflow Notifications -# email_from_name = +# Example: from_email = airflow@example.com +# from_email = [smtp] diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index 66cf695fcc24d..0740097283253 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -24,7 +24,6 @@ def send_email( from_email: str, - from_name: str, to: Union[List[str], str], subject: str, html_content: str, @@ -37,11 +36,10 @@ def send_email( **kwargs, ) -> None: """Email backend for SES.""" - from_formatted = formataddr((from_name, from_email)) hook = SESHook(aws_conn_id=conn_id) hook.send_email( - mail_from=from_formatted, + mail_from=from_email, to=to, subject=subject, html_content=html_content, diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 9b9f9b1e21ca0..c1a7e5abead28 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -49,8 +49,7 @@ def send_email( """Send email using backend specified in EMAIL_BACKEND.""" backend = conf.getimport('email', 'EMAIL_BACKEND') backend_conn_id = conn_id or conf.get("email", "EMAIL_CONN_ID") - from_email = conf.get('email', 'email_from_email', fallback=None) - from_name = conf.get('email', 'email_from_name', fallback=None) + from_email = conf.get('email', 'from_email', fallback=None) to_list = get_email_address_list(to) to_comma_separated = ", ".join(to_list) @@ -67,7 +66,6 @@ def send_email( mime_charset=mime_charset, conn_id=backend_conn_id, from_email=from_email, - from_name=from_name, **kwargs, ) @@ -84,7 +82,6 @@ def send_email_smtp( mime_charset: str = 'utf-8', conn_id: str = "smtp_default", from_email: str = None, - from_name: str = None, **kwargs, ): """ @@ -94,10 +91,10 @@ def send_email_smtp( """ smtp_mail_from = conf.get('smtp', 'SMTP_MAIL_FROM') - from_formatted = formataddr((from_name, smtp_mail_from or from_email)) + mail_from = smtp_mail_from or from_email msg, recipients = build_mime_message( - mail_from=from_formatted, + mail_from=mail_from, to=to, subject=subject, html_content=html_content, @@ -108,7 +105,7 @@ def send_email_smtp( mime_charset=mime_charset, ) - send_mime_email(e_from=from_formatted, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun) + send_mime_email(e_from=mail_from, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun) def build_mime_message( diff --git a/docs/apache-airflow/howto/email-config.rst b/docs/apache-airflow/howto/email-config.rst index 2c67970e93163..af7b3cf877818 100644 --- a/docs/apache-airflow/howto/email-config.rst +++ b/docs/apache-airflow/howto/email-config.rst @@ -29,7 +29,7 @@ in the ``[email]`` section. subject_template = /path/to/my_subject_template_file html_content_template = /path/to/my_html_content_template_file -You can configure sender's email address and name by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. +You can configure sender's email address by setting ``from_email`` in the ``[email]`` section. To configure SMTP settings, checkout the :ref:`SMTP ` section in the standard configuration. If you do not want to store the SMTP credentials in the config or in the environment variables, you can create a @@ -94,7 +94,7 @@ or are used from the connection. 4. Configure sender's email address and name either by exporting the environment variables ``SENDGRID_MAIL_FROM`` and ``SENDGRID_MAIL_SENDER`` or - in your ``airflow.cfg`` by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. + in your ``airflow.cfg`` by setting ``from_email`` in the ``[email]`` section. .. _email-configuration-ses: @@ -122,4 +122,4 @@ Follow the steps below to enable it: 3. Create a connection called ``aws_default``, or choose a custom connection name and set it in ``email_conn_id``. The type of connection should be ``Amazon Web Services``. -4. Configure sender's email address and name in your ``airflow.cfg`` by setting ``email_from_email`` and ``email_from_name`` in the ``[email]`` section. +4. Configure sender's email address in your ``airflow.cfg`` by setting ``from_email`` in the ``[email]`` section. diff --git a/tests/utils/test_email.py b/tests/utils/test_email.py index 0090fde66e0a9..b458bbdcc8fa7 100644 --- a/tests/utils/test_email.py +++ b/tests/utils/test_email.py @@ -100,7 +100,6 @@ def test_custom_backend(self, mock_send_email): mime_subtype='mixed', conn_id='smtp_default', from_email=None, - from_name=None, ) assert not mock_send_email.called @@ -108,15 +107,13 @@ def test_custom_backend(self, mock_send_email): @conf_vars( { ('email', 'email_backend'): 'tests.utils.test_email.send_email_test', - ('email', 'email_from_email'): 'from@test.com', - ('email', 'email_from_name'): 'From Test', + ('email', 'from_email'): 'from@test.com', } ) def test_custom_backend_sender(self, mock_send_email_smtp): utils.email.send_email('to', 'subject', 'content') _, call_kwargs = send_email_test.call_args assert call_kwargs['from_email'] == 'from@test.com' - assert call_kwargs['from_name'] == 'From Test' assert not mock_send_email_smtp.called def test_build_mime_message(self): From 8f4dcd5be1cd823b80d50804312b7d5670157e12 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 10:47:04 +0300 Subject: [PATCH 11/20] Removed unused imports --- airflow/providers/amazon/aws/utils/emailer.py | 1 - airflow/utils/email.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index 0740097283253..0d8e1a8cd98fb 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. """Airflow module for email backend using AWS SES""" -from email.utils import formataddr from typing import List, Optional, Union from airflow.providers.amazon.aws.hooks.ses import SESHook diff --git a/airflow/utils/email.py b/airflow/utils/email.py index c1a7e5abead28..50f24150ec56c 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -24,7 +24,7 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formataddr, formatdate +from email.utils import formatdate from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from airflow.configuration import conf From e50f4bc0fdaedb2e83da919cbb9c23aad44bdde7 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 10:47:37 +0300 Subject: [PATCH 12/20] Removed unused imports --- airflow/providers/amazon/aws/utils/emailer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index 0d8e1a8cd98fb..da88e741be2e9 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. """Airflow module for email backend using AWS SES""" + from typing import List, Optional, Union from airflow.providers.amazon.aws.hooks.ses import SESHook From 3c1b253db94e798ee7494d3759c2bf7f32df7fca Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Fri, 17 Sep 2021 11:14:04 +0300 Subject: [PATCH 13/20] Improved config description --- airflow/config_templates/config.yml | 3 ++- airflow/config_templates/default_airflow.cfg | 3 ++- airflow/providers/amazon/aws/utils/emailer.py | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 33036481c4a02..85e495f363584 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1353,9 +1353,10 @@ - name: from_email description: | Email address that will be used as sender address. + It can either be raw email or the complete address in a format ``Sender Name `` version_added: 2.2.0 type: string - example: "airflow@example.com" + example: "Airflow " default: ~ - name: smtp diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index aaab2a4d8af49..4452ab7b781de 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -679,7 +679,8 @@ default_email_on_failure = True # html_content_template = # Email address that will be used as sender address. -# Example: from_email = airflow@example.com +# Can either be raw email or the complete address in a format "Sender Name " +# Example: from_email = "Airflow " # from_email = [smtp] diff --git a/airflow/providers/amazon/aws/utils/emailer.py b/airflow/providers/amazon/aws/utils/emailer.py index da88e741be2e9..fc34835993304 100644 --- a/airflow/providers/amazon/aws/utils/emailer.py +++ b/airflow/providers/amazon/aws/utils/emailer.py @@ -36,7 +36,6 @@ def send_email( **kwargs, ) -> None: """Email backend for SES.""" - hook = SESHook(aws_conn_id=conn_id) hook.send_email( mail_from=from_email, From 16f86255effe218fcfd204ea16c7339df262872d Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Mon, 20 Sep 2021 09:40:42 +0300 Subject: [PATCH 14/20] Sync airflow config --- airflow/config_templates/default_airflow.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 4452ab7b781de..32b0e5c310883 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -679,7 +679,7 @@ default_email_on_failure = True # html_content_template = # Email address that will be used as sender address. -# Can either be raw email or the complete address in a format "Sender Name " +# It can either be raw email or the complete address in a format ``Sender Name `` # Example: from_email = "Airflow " # from_email = From 615a33dd1b45a087b51b959943109df61074ec1d Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Tue, 21 Sep 2021 11:10:14 +0300 Subject: [PATCH 15/20] Sync airflow config with the yaml template --- airflow/config_templates/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 85e495f363584..941c76ce74a3f 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1356,7 +1356,7 @@ It can either be raw email or the complete address in a format ``Sender Name `` version_added: 2.2.0 type: string - example: "Airflow " + example: "\"Airflow \"" default: ~ - name: smtp From e7e616153fdcaba1c7161e4d33b34eda6998d04b Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Wed, 27 Oct 2021 13:33:37 +0300 Subject: [PATCH 16/20] Updated version_added and removed redundant quotes --- airflow/config_templates/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 941c76ce74a3f..0e56875738a5c 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1354,9 +1354,9 @@ description: | Email address that will be used as sender address. It can either be raw email or the complete address in a format ``Sender Name `` - version_added: 2.2.0 + version_added: 2.3.0 type: string - example: "\"Airflow \"" + example: "Airflow " default: ~ - name: smtp From 1fe74cf639756602cc00d2dc82c502900f06bf17 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Wed, 27 Oct 2021 14:16:29 +0300 Subject: [PATCH 17/20] Sync default cfg with config yml --- airflow/config_templates/default_airflow.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg index 32b0e5c310883..f733c2db0ef9b 100644 --- a/airflow/config_templates/default_airflow.cfg +++ b/airflow/config_templates/default_airflow.cfg @@ -680,7 +680,7 @@ default_email_on_failure = True # Email address that will be used as sender address. # It can either be raw email or the complete address in a format ``Sender Name `` -# Example: from_email = "Airflow " +# Example: from_email = Airflow # from_email = [smtp] From b005bfa34b81c8f521577dd7eabb98803d00a3a3 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Wed, 27 Oct 2021 16:59:36 +0300 Subject: [PATCH 18/20] Fix ses test --- tests/providers/amazon/aws/utils/test_emailer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index ded7b2d8bdc8f..5cecae4300b3d 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -19,19 +19,16 @@ from unittest import TestCase, mock from airflow.providers.amazon.aws.utils.emailer import send_email -from tests.test_utils.config import conf_vars class TestSendEmailSes(TestCase): def setUp(self): pass - @conf_vars({('email', 'email_from_email'): 'from@test.com', ('email', 'email_from_name'): 'From Test'}) @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email(self, mock_hook): send_email( - from_email="from@test.com", - from_name="From Test", + from_email="From Test ", to="to@test.com", subject="subject", html_content="content", From 02586fde3bdbaefa86cd95ed2ee443ff187997d3 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Thu, 28 Oct 2021 11:04:34 +0300 Subject: [PATCH 19/20] Clean up tests --- tests/providers/amazon/aws/utils/test_emailer.py | 2 -- tests/providers/sendgrid/utils/test_emailer.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index 5cecae4300b3d..ab9bd86ba6436 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -22,8 +22,6 @@ class TestSendEmailSes(TestCase): - def setUp(self): - pass @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email(self, mock_hook): diff --git a/tests/providers/sendgrid/utils/test_emailer.py b/tests/providers/sendgrid/utils/test_emailer.py index 5d2d625906765..a2b0f2823b5cc 100644 --- a/tests/providers/sendgrid/utils/test_emailer.py +++ b/tests/providers/sendgrid/utils/test_emailer.py @@ -64,12 +64,6 @@ def setUp(self): 'name': 'Foo Bar', 'email': 'foo@foo.bar', } - # sender from conf - self.expected_mail_data_conf_sender = copy.deepcopy(self.expected_mail_data) - self.expected_mail_data_conf_sender['from'] = { - 'name': 'Foo Conf', - 'email': 'foo@conf.com', - } # Test the right email is constructed. @mock.patch.dict('os.environ', SENDGRID_MAIL_FROM='foo@bar.com') From fb5eed3a65c79b3f3159f24118e7a48394d34080 Mon Sep 17 00:00:00 2001 From: "ignas.kizelevicius" Date: Thu, 28 Oct 2021 11:36:07 +0300 Subject: [PATCH 20/20] Clean up tests --- tests/providers/amazon/aws/utils/test_emailer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/amazon/aws/utils/test_emailer.py b/tests/providers/amazon/aws/utils/test_emailer.py index ab9bd86ba6436..bcbbd4ebe6899 100644 --- a/tests/providers/amazon/aws/utils/test_emailer.py +++ b/tests/providers/amazon/aws/utils/test_emailer.py @@ -22,7 +22,6 @@ class TestSendEmailSes(TestCase): - @mock.patch("airflow.providers.amazon.aws.utils.emailer.SESHook") def test_send_ses_email(self, mock_hook): send_email(