diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py index eab772d233b13..22f37e95555d6 100644 --- a/airflow/providers/databricks/operators/databricks.py +++ b/airflow/providers/databricks/operators/databricks.py @@ -207,6 +207,7 @@ class DatabricksCreateJobsOperator(BaseOperator): .. seealso:: For more information about templating see :ref:`concepts:jinja-templating`. :param name: An optional name for the job. + :param description: An optional description for the job. :param tags: A map of tags associated with the job. :param tasks: A list of task specifications to be executed by this job. Array of objects (JobTaskSettings). @@ -214,6 +215,7 @@ class DatabricksCreateJobsOperator(BaseOperator): tasks of this job. Array of objects (JobCluster). :param email_notifications: Object (JobEmailNotifications). :param webhook_notifications: Object (WebhookNotifications). + :param notification_settings: Optional notification settings. :param timeout_seconds: An optional timeout applied to each run of this job. :param schedule: Object (CronSchedule). :param max_concurrent_runs: An optional maximum allowed number of concurrent runs of the job. @@ -249,11 +251,13 @@ def __init__( *, json: Any | None = None, name: str | None = None, + description: str | None = None, tags: dict[str, str] | None = None, tasks: list[dict] | None = None, job_clusters: list[dict] | None = None, email_notifications: dict | None = None, webhook_notifications: dict | None = None, + notification_settings: dict | None = None, timeout_seconds: int | None = None, schedule: dict | None = None, max_concurrent_runs: int | None = None, @@ -276,6 +280,8 @@ def __init__( self.databricks_retry_args = databricks_retry_args if name is not None: self.json["name"] = name + if description is not None: + self.json["description"] = description if tags is not None: self.json["tags"] = tags if tasks is not None: @@ -286,6 +292,8 @@ def __init__( self.json["email_notifications"] = email_notifications if webhook_notifications is not None: self.json["webhook_notifications"] = webhook_notifications + if notification_settings is not None: + self.json["notification_settings"] = notification_settings if timeout_seconds is not None: self.json["timeout_seconds"] = timeout_seconds if schedule is not None: diff --git a/docs/apache-airflow-providers-databricks/operators/jobs_create.rst b/docs/apache-airflow-providers-databricks/operators/jobs_create.rst index 779095e92cd6b..7e6765eba420a 100644 --- a/docs/apache-airflow-providers-databricks/operators/jobs_create.rst +++ b/docs/apache-airflow-providers-databricks/operators/jobs_create.rst @@ -44,11 +44,13 @@ override the top level ``json`` keys. Currently the named parameters that ``DatabricksCreateJobsOperator`` supports are: - ``name`` + - ``description`` - ``tags`` - ``tasks`` - ``job_clusters`` - ``email_notifications`` - ``webhook_notifications`` + - ``notification_settings`` - ``timeout_seconds`` - ``schedule`` - ``max_concurrent_runs`` diff --git a/tests/providers/databricks/operators/test_databricks.py b/tests/providers/databricks/operators/test_databricks.py index 46e14a917ab4e..26d59baa61610 100644 --- a/tests/providers/databricks/operators/test_databricks.py +++ b/tests/providers/databricks/operators/test_databricks.py @@ -63,6 +63,7 @@ RUN_PAGE_URL = "run-page-url" JOB_ID = "42" JOB_NAME = "job-name" +JOB_DESCRIPTION = "job-description" NOTEBOOK_PARAMS = {"dry-run": "true", "oldest-time-to-consider": "1457570074236"} JAR_PARAMS = ["param1", "param2"] RENDERED_TEMPLATED_JAR_PARAMS = [f"/test-{DATE}"] @@ -202,6 +203,7 @@ } ], } +NOTIFICATION_SETTINGS = {"no_alert_for_canceled_runs": True, "no_alert_for_skipped_runs": True} TIMEOUT_SECONDS = 86400 SCHEDULE = { "quartz_cron_expression": "20 30 * * * ?", @@ -409,11 +411,13 @@ def test_exec_create(self, db_mock_class): """ json = { "name": JOB_NAME, + "description": JOB_DESCRIPTION, "tags": TAGS, "tasks": TASKS, "job_clusters": JOB_CLUSTERS, "email_notifications": EMAIL_NOTIFICATIONS, "webhook_notifications": WEBHOOK_NOTIFICATIONS, + "notification_settings": NOTIFICATION_SETTINGS, "timeout_seconds": TIMEOUT_SECONDS, "schedule": SCHEDULE, "max_concurrent_runs": MAX_CONCURRENT_RUNS, @@ -431,11 +435,13 @@ def test_exec_create(self, db_mock_class): expected = utils.normalise_json_content( { "name": JOB_NAME, + "description": JOB_DESCRIPTION, "tags": TAGS, "tasks": TASKS, "job_clusters": JOB_CLUSTERS, "email_notifications": EMAIL_NOTIFICATIONS, "webhook_notifications": WEBHOOK_NOTIFICATIONS, + "notification_settings": NOTIFICATION_SETTINGS, "timeout_seconds": TIMEOUT_SECONDS, "schedule": SCHEDULE, "max_concurrent_runs": MAX_CONCURRENT_RUNS, @@ -461,11 +467,13 @@ def test_exec_reset(self, db_mock_class): """ json = { "name": JOB_NAME, + "description": JOB_DESCRIPTION, "tags": TAGS, "tasks": TASKS, "job_clusters": JOB_CLUSTERS, "email_notifications": EMAIL_NOTIFICATIONS, "webhook_notifications": WEBHOOK_NOTIFICATIONS, + "notification_settings": NOTIFICATION_SETTINGS, "timeout_seconds": TIMEOUT_SECONDS, "schedule": SCHEDULE, "max_concurrent_runs": MAX_CONCURRENT_RUNS, @@ -481,11 +489,13 @@ def test_exec_reset(self, db_mock_class): expected = utils.normalise_json_content( { "name": JOB_NAME, + "description": JOB_DESCRIPTION, "tags": TAGS, "tasks": TASKS, "job_clusters": JOB_CLUSTERS, "email_notifications": EMAIL_NOTIFICATIONS, "webhook_notifications": WEBHOOK_NOTIFICATIONS, + "notification_settings": NOTIFICATION_SETTINGS, "timeout_seconds": TIMEOUT_SECONDS, "schedule": SCHEDULE, "max_concurrent_runs": MAX_CONCURRENT_RUNS,