|
25 | 25 | import pytest
|
26 | 26 |
|
27 | 27 | from airflow.decorators import task as task_decorator
|
28 |
| -from airflow.exceptions import AirflowException, TaskDeferralTimeout |
| 28 | +from airflow.exceptions import TaskDeferralTimeout |
29 | 29 | from airflow.models.baseoperator import (
|
30 | 30 | BaseOperator,
|
31 |
| - chain, |
32 |
| - chain_linear, |
33 |
| - cross_downstream, |
34 | 31 | )
|
35 | 32 | from airflow.models.dag import DAG
|
36 | 33 | from airflow.models.dagrun import DagRun
|
37 | 34 | from airflow.models.taskinstance import TaskInstance
|
38 | 35 | from airflow.models.trigger import TriggerFailureReason
|
39 | 36 | from airflow.providers.common.compat.lineage.entities import File
|
40 | 37 | from airflow.providers.common.sql.operators import sql
|
41 |
| -from airflow.utils.edgemodifier import Label |
42 | 38 | from airflow.utils.task_group import TaskGroup
|
43 | 39 | from airflow.utils.trigger_rule import TriggerRule
|
44 | 40 | from airflow.utils.types import DagRunType
|
@@ -91,89 +87,6 @@ def test_trigger_rule_validation(self):
|
91 | 87 | task_id="test_valid_trigger_rule", dag=non_fail_fast_dag, trigger_rule=TriggerRule.ALWAYS
|
92 | 88 | )
|
93 | 89 |
|
94 |
| - def test_cross_downstream(self): |
95 |
| - """Test if all dependencies between tasks are all set correctly.""" |
96 |
| - dag = DAG(dag_id="test_dag", schedule=None, start_date=datetime.now()) |
97 |
| - start_tasks = [BaseOperator(task_id=f"t{i}", dag=dag) for i in range(1, 4)] |
98 |
| - end_tasks = [BaseOperator(task_id=f"t{i}", dag=dag) for i in range(4, 7)] |
99 |
| - cross_downstream(from_tasks=start_tasks, to_tasks=end_tasks) |
100 |
| - |
101 |
| - for start_task in start_tasks: |
102 |
| - assert set(start_task.get_direct_relatives(upstream=False)) == set(end_tasks) |
103 |
| - |
104 |
| - # Begin test for `XComArgs` |
105 |
| - xstart_tasks = [ |
106 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
107 |
| - for i in range(1, 4) |
108 |
| - ] |
109 |
| - xend_tasks = [ |
110 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
111 |
| - for i in range(4, 7) |
112 |
| - ] |
113 |
| - cross_downstream(from_tasks=xstart_tasks, to_tasks=xend_tasks) |
114 |
| - |
115 |
| - for xstart_task in xstart_tasks: |
116 |
| - assert set(xstart_task.operator.get_direct_relatives(upstream=False)) == { |
117 |
| - xend_task.operator for xend_task in xend_tasks |
118 |
| - } |
119 |
| - |
120 |
| - def test_chain(self): |
121 |
| - dag = DAG(dag_id="test_chain", schedule=None, start_date=datetime.now()) |
122 |
| - |
123 |
| - # Begin test for classic operators with `EdgeModifiers` |
124 |
| - [label1, label2] = [Label(label=f"label{i}") for i in range(1, 3)] |
125 |
| - [op1, op2, op3, op4, op5, op6] = [BaseOperator(task_id=f"t{i}", dag=dag) for i in range(1, 7)] |
126 |
| - chain(op1, [label1, label2], [op2, op3], [op4, op5], op6) |
127 |
| - |
128 |
| - assert {op2, op3} == set(op1.get_direct_relatives(upstream=False)) |
129 |
| - assert [op4] == op2.get_direct_relatives(upstream=False) |
130 |
| - assert [op5] == op3.get_direct_relatives(upstream=False) |
131 |
| - assert {op4, op5} == set(op6.get_direct_relatives(upstream=True)) |
132 |
| - |
133 |
| - assert dag.get_edge_info(upstream_task_id=op1.task_id, downstream_task_id=op2.task_id) == { |
134 |
| - "label": "label1" |
135 |
| - } |
136 |
| - assert dag.get_edge_info(upstream_task_id=op1.task_id, downstream_task_id=op3.task_id) == { |
137 |
| - "label": "label2" |
138 |
| - } |
139 |
| - |
140 |
| - # Begin test for `XComArgs` with `EdgeModifiers` |
141 |
| - [xlabel1, xlabel2] = [Label(label=f"xcomarg_label{i}") for i in range(1, 3)] |
142 |
| - [xop1, xop2, xop3, xop4, xop5, xop6] = [ |
143 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
144 |
| - for i in range(1, 7) |
145 |
| - ] |
146 |
| - chain(xop1, [xlabel1, xlabel2], [xop2, xop3], [xop4, xop5], xop6) |
147 |
| - |
148 |
| - assert {xop2.operator, xop3.operator} == set(xop1.operator.get_direct_relatives(upstream=False)) |
149 |
| - assert [xop4.operator] == xop2.operator.get_direct_relatives(upstream=False) |
150 |
| - assert [xop5.operator] == xop3.operator.get_direct_relatives(upstream=False) |
151 |
| - assert {xop4.operator, xop5.operator} == set(xop6.operator.get_direct_relatives(upstream=True)) |
152 |
| - |
153 |
| - assert dag.get_edge_info( |
154 |
| - upstream_task_id=xop1.operator.task_id, downstream_task_id=xop2.operator.task_id |
155 |
| - ) == {"label": "xcomarg_label1"} |
156 |
| - assert dag.get_edge_info( |
157 |
| - upstream_task_id=xop1.operator.task_id, downstream_task_id=xop3.operator.task_id |
158 |
| - ) == {"label": "xcomarg_label2"} |
159 |
| - |
160 |
| - # Begin test for `TaskGroups` |
161 |
| - [tg1, tg2] = [TaskGroup(group_id=f"tg{i}", dag=dag) for i in range(1, 3)] |
162 |
| - [op1, op2] = [BaseOperator(task_id=f"task{i}", dag=dag) for i in range(1, 3)] |
163 |
| - [tgop1, tgop2] = [ |
164 |
| - BaseOperator(task_id=f"task_group_task{i}", task_group=tg1, dag=dag) for i in range(1, 3) |
165 |
| - ] |
166 |
| - [tgop3, tgop4] = [ |
167 |
| - BaseOperator(task_id=f"task_group_task{i}", task_group=tg2, dag=dag) for i in range(1, 3) |
168 |
| - ] |
169 |
| - chain(op1, tg1, tg2, op2) |
170 |
| - |
171 |
| - assert {tgop1, tgop2} == set(op1.get_direct_relatives(upstream=False)) |
172 |
| - assert {tgop3, tgop4} == set(tgop1.get_direct_relatives(upstream=False)) |
173 |
| - assert {tgop3, tgop4} == set(tgop2.get_direct_relatives(upstream=False)) |
174 |
| - assert [op2] == tgop3.get_direct_relatives(upstream=False) |
175 |
| - assert [op2] == tgop4.get_direct_relatives(upstream=False) |
176 |
| - |
177 | 90 | def test_baseoperator_raises_exception_when_task_id_plus_taskgroup_id_exceeds_250_chars(self):
|
178 | 91 | """Test exception is raised when operator task id + taskgroup id > 250 chars."""
|
179 | 92 | dag = DAG(dag_id="foo", schedule=None, start_date=datetime.now())
|
@@ -201,110 +114,6 @@ def test_baseoperator_with_task_id_less_than_250_chars(self):
|
201 | 114 | except Exception as e:
|
202 | 115 | pytest.fail(f"Exception raised: {e}")
|
203 | 116 |
|
204 |
| - def test_chain_linear(self): |
205 |
| - dag = DAG(dag_id="test_chain_linear", schedule=None, start_date=datetime.now()) |
206 |
| - |
207 |
| - t1, t2, t3, t4, t5, t6, t7 = (BaseOperator(task_id=f"t{i}", dag=dag) for i in range(1, 8)) |
208 |
| - chain_linear(t1, [t2, t3, t4], [t5, t6], t7) |
209 |
| - |
210 |
| - assert set(t1.get_direct_relatives(upstream=False)) == {t2, t3, t4} |
211 |
| - assert set(t2.get_direct_relatives(upstream=False)) == {t5, t6} |
212 |
| - assert set(t3.get_direct_relatives(upstream=False)) == {t5, t6} |
213 |
| - assert set(t7.get_direct_relatives(upstream=True)) == {t5, t6} |
214 |
| - |
215 |
| - t1, t2, t3, t4, t5, t6 = ( |
216 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
217 |
| - for i in range(1, 7) |
218 |
| - ) |
219 |
| - chain_linear(t1, [t2, t3], [t4, t5], t6) |
220 |
| - |
221 |
| - assert set(t1.operator.get_direct_relatives(upstream=False)) == {t2.operator, t3.operator} |
222 |
| - assert set(t2.operator.get_direct_relatives(upstream=False)) == {t4.operator, t5.operator} |
223 |
| - assert set(t3.operator.get_direct_relatives(upstream=False)) == {t4.operator, t5.operator} |
224 |
| - assert set(t6.operator.get_direct_relatives(upstream=True)) == {t4.operator, t5.operator} |
225 |
| - |
226 |
| - # Begin test for `TaskGroups` |
227 |
| - tg1, tg2 = (TaskGroup(group_id=f"tg{i}", dag=dag) for i in range(1, 3)) |
228 |
| - op1, op2 = (BaseOperator(task_id=f"task{i}", dag=dag) for i in range(1, 3)) |
229 |
| - tgop1, tgop2 = ( |
230 |
| - BaseOperator(task_id=f"task_group_task{i}", task_group=tg1, dag=dag) for i in range(1, 3) |
231 |
| - ) |
232 |
| - tgop3, tgop4 = ( |
233 |
| - BaseOperator(task_id=f"task_group_task{i}", task_group=tg2, dag=dag) for i in range(1, 3) |
234 |
| - ) |
235 |
| - chain_linear(op1, tg1, tg2, op2) |
236 |
| - |
237 |
| - assert set(op1.get_direct_relatives(upstream=False)) == {tgop1, tgop2} |
238 |
| - assert set(tgop1.get_direct_relatives(upstream=False)) == {tgop3, tgop4} |
239 |
| - assert set(tgop2.get_direct_relatives(upstream=False)) == {tgop3, tgop4} |
240 |
| - assert set(tgop3.get_direct_relatives(upstream=False)) == {op2} |
241 |
| - assert set(tgop4.get_direct_relatives(upstream=False)) == {op2} |
242 |
| - |
243 |
| - t1, t2 = (BaseOperator(task_id=f"t-{i}", dag=dag) for i in range(1, 3)) |
244 |
| - with pytest.raises(ValueError, match="Labels are not supported"): |
245 |
| - chain_linear(t1, Label("hi"), t2) |
246 |
| - |
247 |
| - with pytest.raises(ValueError, match="nothing to do"): |
248 |
| - chain_linear() |
249 |
| - |
250 |
| - with pytest.raises(ValueError, match="Did you forget to expand"): |
251 |
| - chain_linear(t1) |
252 |
| - |
253 |
| - def test_chain_not_support_type(self): |
254 |
| - dag = DAG(dag_id="test_chain", schedule=None, start_date=datetime.now()) |
255 |
| - [op1, op2] = [BaseOperator(task_id=f"t{i}", dag=dag) for i in range(1, 3)] |
256 |
| - with pytest.raises(TypeError): |
257 |
| - chain([op1, op2], 1) |
258 |
| - |
259 |
| - # Begin test for `XComArgs` |
260 |
| - [xop1, xop2] = [ |
261 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
262 |
| - for i in range(1, 3) |
263 |
| - ] |
264 |
| - |
265 |
| - with pytest.raises(TypeError): |
266 |
| - chain([xop1, xop2], 1) |
267 |
| - |
268 |
| - # Begin test for `EdgeModifiers` |
269 |
| - with pytest.raises(TypeError): |
270 |
| - chain([Label("labe1"), Label("label2")], 1) |
271 |
| - |
272 |
| - # Begin test for `TaskGroups` |
273 |
| - [tg1, tg2] = [TaskGroup(group_id=f"tg{i}", dag=dag) for i in range(1, 3)] |
274 |
| - |
275 |
| - with pytest.raises(TypeError): |
276 |
| - chain([tg1, tg2], 1) |
277 |
| - |
278 |
| - def test_chain_different_length_iterable(self): |
279 |
| - dag = DAG(dag_id="test_chain", schedule=None, start_date=datetime.now()) |
280 |
| - [label1, label2] = [Label(label=f"label{i}") for i in range(1, 3)] |
281 |
| - [op1, op2, op3, op4, op5] = [BaseOperator(task_id=f"t{i}", dag=dag) for i in range(1, 6)] |
282 |
| - |
283 |
| - with pytest.raises(AirflowException): |
284 |
| - chain([op1, op2], [op3, op4, op5]) |
285 |
| - |
286 |
| - with pytest.raises(AirflowException): |
287 |
| - chain([op1, op2, op3], [label1, label2]) |
288 |
| - |
289 |
| - # Begin test for `XComArgs` with `EdgeModifiers` |
290 |
| - [label3, label4] = [Label(label=f"xcomarg_label{i}") for i in range(1, 3)] |
291 |
| - [xop1, xop2, xop3, xop4, xop5] = [ |
292 |
| - task_decorator(task_id=f"xcomarg_task{i}", python_callable=lambda: None, dag=dag)() |
293 |
| - for i in range(1, 6) |
294 |
| - ] |
295 |
| - |
296 |
| - with pytest.raises(AirflowException): |
297 |
| - chain([xop1, xop2], [xop3, xop4, xop5]) |
298 |
| - |
299 |
| - with pytest.raises(AirflowException): |
300 |
| - chain([xop1, xop2, xop3], [label1, label2]) |
301 |
| - |
302 |
| - # Begin test for `TaskGroups` |
303 |
| - [tg1, tg2, tg3, tg4, tg5] = [TaskGroup(group_id=f"tg{i}", dag=dag) for i in range(1, 6)] |
304 |
| - |
305 |
| - with pytest.raises(AirflowException): |
306 |
| - chain([tg1, tg2], [tg3, tg4, tg5]) |
307 |
| - |
308 | 117 | def test_lineage_composition(self):
|
309 | 118 | """
|
310 | 119 | Test composition with lineage
|
|
0 commit comments