Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion airflow/providers/amazon/aws/operators/rds.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,13 +370,15 @@ def __init__(
*,
export_task_identifier: str,
wait_for_completion: bool = True,
check_interval: int = 30,
aws_conn_id: str = "aws_default",
**kwargs,
):
super().__init__(aws_conn_id=aws_conn_id, **kwargs)

self.export_task_identifier = export_task_identifier
self.wait_for_completion = wait_for_completion
self.check_interval = check_interval

def execute(self, context: Context) -> str:
self.log.info("Canceling export task %s", self.export_task_identifier)
Expand All @@ -386,7 +388,9 @@ def execute(self, context: Context) -> str:
)

if self.wait_for_completion:
self.hook.wait_for_export_task_state(self.export_task_identifier, target_state="canceled")
self.hook.wait_for_export_task_state(
self.export_task_identifier, target_state="canceled", check_interval=self.check_interval
)
return json.dumps(cancel_export, default=str)


Expand Down
2 changes: 2 additions & 0 deletions tests/system/providers/amazon/aws/example_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ def delete_job_queue(job_queue_name):
job_id=submit_batch_job.output,
)
# [END howto_sensor_batch]
wait_for_batch_job.poke_interval = 10

wait_for_compute_environment_disabled = BatchComputeEnvironmentSensor(
task_id="wait_for_compute_environment_disabled",
Expand All @@ -224,6 +225,7 @@ def delete_job_queue(job_queue_name):
task_id="wait_for_job_queue_deleted",
job_queue=batch_job_queue_name,
treat_non_existing_as_deleted=True,
poke_interval=10,
)

log_cleanup = prune_logs(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@
trigger_rule=TriggerRule.ALL_DONE,
cluster_name=cluster_name,
target_state=ClusterStates.NONEXISTENT,
poke_interval=10,
)

chain(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@
fargate_profile_name=fargate_profile_name,
target_state=FargateProfileStates.NONEXISTENT,
trigger_rule=TriggerRule.ALL_DONE,
poke_interval=10,
)

delete_cluster = EksDeleteClusterOperator(
Expand All @@ -169,6 +170,7 @@
task_id="wait_for_delete_cluster",
cluster_name=cluster_name,
target_state=ClusterStates.NONEXISTENT,
poke_interval=10,
)

chain(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
cluster_name=cluster_name,
nodegroup_name=nodegroup_name,
target_state=NodegroupStates.ACTIVE,
poke_interval=10,
)

start_pod = EksPodOperator(
Expand Down Expand Up @@ -105,6 +106,7 @@
trigger_rule=TriggerRule.ALL_DONE,
cluster_name=cluster_name,
target_state=ClusterStates.NONEXISTENT,
poke_interval=10,
)

chain(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
target_state=NodegroupStates.ACTIVE,
)
# [END howto_sensor_eks_nodegroup]
await_create_nodegroup.poke_interval = 10

# [START howto_operator_eks_pod_operator]
start_pod = EksPodOperator(
Expand Down Expand Up @@ -140,6 +141,7 @@
trigger_rule=TriggerRule.ALL_DONE,
cluster_name=cluster_name,
target_state=ClusterStates.NONEXISTENT,
poke_interval=10,
)

chain(
Expand Down
1 change: 1 addition & 0 deletions tests/system/providers/amazon/aws/example_emr.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ def delete_security_config(config_name: str):
# [START howto_sensor_emr_job_flow]
check_job_flow = EmrJobFlowSensor(task_id="check_job_flow", job_flow_id=create_job_flow.output)
# [END howto_sensor_emr_job_flow]
check_job_flow.poke_interval = 10

delete_security_configuration = delete_security_config(config_name)

Expand Down
2 changes: 2 additions & 0 deletions tests/system/providers/amazon/aws/example_emr_eks.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ def delete_virtual_cluster(virtual_cluster_id):
cluster_name=eks_cluster_name,
nodegroup_name=nodegroup_name,
target_state=NodegroupStates.ACTIVE,
poke_interval=10,
)

emr_access_on_eks = enable_access_emr_on_eks(eks_cluster_name, eks_namespace)
Expand Down Expand Up @@ -260,6 +261,7 @@ def delete_virtual_cluster(virtual_cluster_id):
cluster_name=eks_cluster_name,
target_state=ClusterStates.NONEXISTENT,
trigger_rule=TriggerRule.ALL_DONE,
poke_interval=10,
)

delete_bucket = S3DeleteBucketOperator(
Expand Down
3 changes: 3 additions & 0 deletions tests/system/providers/amazon/aws/example_emr_serverless.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
application_id=emr_serverless_app_id,
)
# [END howto_sensor_emr_serverless_application]
wait_for_app_creation.poke_interval = 1

# [START howto_operator_emr_serverless_start_job]
start_job = EmrServerlessStartJobOperator(
Expand All @@ -98,6 +99,7 @@
configuration_overrides=SPARK_CONFIGURATION_OVERRIDES,
)
# [END howto_operator_emr_serverless_start_job]
start_job.waiter_check_interval_seconds = 10

# [START howto_sensor_emr_serverless_job]
wait_for_job = EmrServerlessJobSensor(
Expand All @@ -113,6 +115,7 @@
application_id=emr_serverless_app_id,
)
# [END howto_operator_emr_serverless_delete_application]
delete_app.waiter_check_interval_seconds = 1
delete_app.trigger_rule = TriggerRule.ALL_DONE

delete_s3_bucket = S3DeleteBucketOperator(
Expand Down
1 change: 1 addition & 0 deletions tests/system/providers/amazon/aws/example_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ def glue_cleanup(crawler_name: str, job_name: str, db_name: str) -> None:
run_id=submit_glue_job.output,
)
# [END howto_sensor_glue]
wait_for_job.poke_interval = 10

delete_bucket = S3DeleteBucketOperator(
task_id="delete_bucket",
Expand Down
1 change: 1 addition & 0 deletions tests/system/providers/amazon/aws/example_rds_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ def get_snapshot_arn(snapshot_name: str) -> str:
export_task_identifier=rds_export_task_id,
)
# [END howto_operator_rds_cancel_export]
cancel_export.check_interval = 10

# [START howto_sensor_rds_export_task_existence]
export_sensor = RdsExportTaskExistenceSensor(
Expand Down