Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow-core/src/airflow/models/mappedoperator.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
from airflow.sdk.definitions._internal.node import DAGNode as TaskSDKDAGNode
from airflow.sdk.definitions.operator_resources import Resources
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.serialized_objects import XComOperatorLink
from airflow.serialization.definitions.operatorlink import XComOperatorLink
from airflow.task.trigger_rule import TriggerRule
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
from airflow.triggers.base import StartTriggerArgs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@
from airflow.models.taskinstance import TaskInstance
from airflow.sdk import Context
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.definitions.operatorlink import XComOperatorLink
from airflow.serialization.definitions.taskgroup import SerializedMappedTaskGroup, SerializedTaskGroup
from airflow.serialization.serialized_objects import XComOperatorLink
from airflow.task.trigger_rule import TriggerRule
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
from airflow.triggers.base import StartTriggerArgs
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from __future__ import annotations

from typing import TYPE_CHECKING, TypeAlias

import attrs

from airflow.models.xcom import XComModel
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.session import create_session

if TYPE_CHECKING:
from airflow.models.mappedoperator import MappedOperator as SerializedMappedOperator
from airflow.models.taskinstancekey import TaskInstanceKey
from airflow.serialization.definitions.baseoperator import SerializedBaseOperator

Operator: TypeAlias = "SerializedMappedOperator | SerializedBaseOperator"


@attrs.define()
class XComOperatorLink(LoggingMixin):
"""
Generic operator link class that can retrieve link only using XCOMs.

Used while deserializing operators.
"""

name: str
xcom_key: str

def get_link(self, operator: Operator, *, ti_key: TaskInstanceKey) -> str:
"""
Retrieve the link from the XComs.

:param operator: The Airflow operator object this link is associated to.
:param ti_key: TaskInstance ID to return link for.
:return: link to external system, but by pulling it from XComs
"""
self.log.info(
"Attempting to retrieve link from XComs with key: %s for task id: %s", self.xcom_key, ti_key
)
with create_session() as session:
value = session.execute(
XComModel.get_many(
key=self.xcom_key,
run_id=ti_key.run_id,
dag_ids=ti_key.dag_id,
task_ids=ti_key.task_id,
map_indexes=ti_key.map_index,
).with_only_columns(XComModel.value)
).first()
if not value:
self.log.debug(
"No link with name: %s present in XCom as key: %s, returning empty link",
self.name,
self.xcom_key,
)
return ""
return XComModel.deserialize_value(value)
46 changes: 1 addition & 45 deletions airflow-core/src/airflow/serialization/serialized_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
from airflow.models.connection import Connection
from airflow.models.expandinput import create_expand_input
from airflow.models.taskinstancekey import TaskInstanceKey
from airflow.models.xcom import XComModel
from airflow.models.xcom_arg import SchedulerXComArg, deserialize_xcom_arg
from airflow.sdk import DAG, Asset, AssetAlias, BaseOperator, XComArg
from airflow.sdk.bases.operator import OPERATOR_DEFAULTS # TODO: Copy this into the scheduler?
Expand Down Expand Up @@ -76,6 +75,7 @@
from airflow.serialization.definitions.baseoperator import SerializedBaseOperator
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.definitions.node import DAGNode
from airflow.serialization.definitions.operatorlink import XComOperatorLink
from airflow.serialization.definitions.param import SerializedParam, SerializedParamsDict
from airflow.serialization.definitions.taskgroup import SerializedMappedTaskGroup, SerializedTaskGroup
from airflow.serialization.encoders import (
Expand All @@ -100,8 +100,6 @@
from airflow.utils.code_utils import get_python_source
from airflow.utils.context import ConnectionAccessor, Context, VariableAccessor
from airflow.utils.db import LazySelectSequence
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.session import create_session

if TYPE_CHECKING:
from inspect import Parameter
Expand Down Expand Up @@ -2363,48 +2361,6 @@ def owner(self) -> str:
)


@attrs.define()
class XComOperatorLink(LoggingMixin):
"""
Generic operator link class that can retrieve link only using XCOMs.

Used while deserializing operators.
"""

name: str
xcom_key: str

def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey) -> str:
"""
Retrieve the link from the XComs.

:param operator: The Airflow operator object this link is associated to.
:param ti_key: TaskInstance ID to return link for.
:return: link to external system, but by pulling it from XComs
"""
self.log.info(
"Attempting to retrieve link from XComs with key: %s for task id: %s", self.xcom_key, ti_key
)
with create_session() as session:
value = session.execute(
XComModel.get_many(
key=self.xcom_key,
run_id=ti_key.run_id,
dag_ids=ti_key.dag_id,
task_ids=ti_key.task_id,
map_indexes=ti_key.map_index,
).with_only_columns(XComModel.value)
).first()
if not value:
self.log.debug(
"No link with name: %s present in XCom as key: %s, returning empty link",
self.name,
self.xcom_key,
)
return ""
return XComModel.deserialize_value(value)


@overload
def create_scheduler_operator(op: BaseOperator | SerializedBaseOperator) -> SerializedBaseOperator: ...

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,18 @@
from airflow.sdk.definitions.param import Param, ParamsDict
from airflow.security import permissions
from airflow.serialization.definitions.assets import SerializedAssetUniqueKey
from airflow.serialization.definitions.baseoperator import SerializedBaseOperator
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.definitions.notset import NOTSET
from airflow.serialization.definitions.operatorlink import XComOperatorLink
from airflow.serialization.definitions.param import SerializedParam
from airflow.serialization.encoders import ensure_serialized_asset
from airflow.serialization.enums import Encoding
from airflow.serialization.json_schema import load_dag_schema_dict
from airflow.serialization.serialized_objects import (
BaseSerialization,
DagSerialization,
OperatorSerialization,
SerializedBaseOperator,
SerializedParam,
XComOperatorLink,
)
from airflow.task.priority_strategy import _AbsolutePriorityWeightStrategy, _DownstreamPriorityWeightStrategy
from airflow.ti_deps.deps.ready_to_reschedule import ReadyToRescheduleDep
Expand Down