Index

A

add_child() (ditto.api.DAGFragment method) add_child_in_place() (ditto.api.DAGFragment method) add_dag_fragment_to_dag() (ditto.utils.TransformerUtils class method) add_downstream_dag_fragment() (ditto.utils.TransformerUtils static method)

add_parent() (ditto.api.DAGFragment method) AirflowDagTransformer (class in ditto.ditto) AncestralClassTransformerResolver (class in ditto.resolvers) assign_task_to_dag() (ditto.utils.TransformerUtils static method)

C

callable_transformers (ditto.resolvers.PythonCallTransformerResolver parameter), [1] check_for_existing_emr_cluster (ditto.transformers.subdag.CheckClusterSubDagTransformer attribute) CheckClusterEmr2HdiDagTransformerTemplate (class in ditto.templates) CheckClusterSubDagTransformer (class in ditto.transformers.subdag) child (ditto.api.DAGFragment.add_child_in_place parameter)

children (ditto.api.DAGFragment parameter) (ditto.api.TaskMatcher attribute) ClassTaskMatcher (class in ditto.matchers) ClassTransformerResolver (class in ditto.resolvers) copy_op_attrs() (ditto.api.OperatorTransformer static method) CopyTransformer (class in ditto.transformers)

D

dag (ditto.api.SubDagTransformer parameter) (ditto.transformers.subdag.CheckClusterSubDagTransformer parameter), [1] (ditto.utils.TransformerUtils.add_dag_fragment_to_dag parameter) (ditto.utils.TransformerUtils.assign_task_to_dag parameter) (ditto.utils.TransformerUtils.get_digraph_from_airflow_dag parameter) (ditto.utils.TransformerUtils.get_digraph_from_matcher_dag parameter) (ditto.utils.TransformerUtils.remove_task_from_dag parameter) dag_fragment (ditto.api.DAGFragment.add_child parameter) (ditto.api.DAGFragment.add_parent parameter) (ditto.utils.TransformerUtils.find_op_in_dag_fragment parameter) dag_nodes (ditto.utils.TransformerUtils.remove_task_from_dag parameter) DAGFragment (class in ditto.api) debug_colorer() (in module ditto.rendering) debug_dags() (in module ditto.rendering) debug_legender() (in module ditto.rendering) debug_mode (ditto.ditto.AirflowDagTransformer parameter) (ditto.templates.CheckClusterEmr2HdiDagTransformerTemplate parameter), [1] (ditto.templates.EmrHdiDagTransformerTemplate parameter), [1] debug_relabeler() (in module ditto.rendering) default_operator (ditto.api.TransformerDefaults attribute) DEFAULT_PROXY_USER (ditto.transformers.emr.EmrAddStepsOperatorTransformer attribute) DEFAULT_SPARK_CONF (ditto.transformers.emr.EmrAddStepsOperatorTransformer attribute) defaults (ditto.api.OperatorTransformer parameter) (ditto.api.SubDagTransformer parameter) (ditto.transformers.CopyTransformer parameter) (ditto.transformers.emr.EmrAddStepsOperatorTransformer parameter), [1] (ditto.transformers.emr.EmrCreateJobFlowOperatorTransformer parameter), [1] (ditto.transformers.emr.EmrJobFlowSensorTransformer parameter), [1] (ditto.transformers.emr.EmrStepSensorTransformer parameter), [1] (ditto.transformers.emr.EmrTerminateJobFlowOperatorTransformer parameter), [1] (ditto.transformers.IdentityTransformer parameter) (ditto.transformers.s3.S3KeySensorAdlsGen1OperatorTransformer parameter), [1] (ditto.transformers.s3.S3KeySensorBlobOperatorTransformer parameter), [1] (ditto.transformers.subdag.CheckClusterSubDagTransformer parameter), [1]

ditto.api module ditto.ditto module ditto.rendering module ditto.transformers module ditto.utils module does_match() (ditto.api.TaskMatcher method) (ditto.matchers.ClassTaskMatcher method) (ditto.matchers.PythonCallTaskMatcher method) draw_dag_graphiviz_rendering() (in module ditto.rendering)

E

EmrAddStepsOperatorTransformer (class in ditto.transformers.emr) EmrCreateJobFlowOperatorTransformer (class in ditto.transformers.emr) EmrHdiDagTransformerTemplate (class in ditto.templates)

EmrJobFlowSensorTransformer (class in ditto.transformers.emr) EmrStepSensorTransformer (class in ditto.transformers.emr) EmrTerminateJobFlowOperatorTransformer (class in ditto.transformers.emr)

F

find_callable (ditto.matchers.PythonCallTaskMatcher parameter), [1] find_matching_tasks() (ditto.utils.TransformerUtils class method) find_op_in_dag_fragment() (ditto.utils.TransformerUtils static method) find_op_in_fragment_list() (ditto.utils.TransformerUtils class method) find_op_in_fragment_list_strict() (ditto.utils.TransformerUtils class method) find_op_in_parent_fragment_chain() (ditto.utils.TransformerUtils class method)

find_sub_dag() (ditto.utils.TransformerUtils class method) frag (ditto.utils.TransformerUtils.add_dag_fragment_to_dag parameter) fragment_down (ditto.utils.TransformerUtils.add_downstream_dag_fragment parameter) fragment_list (ditto.utils.TransformerUtils.find_op_in_fragment_list parameter) (ditto.utils.TransformerUtils.find_op_in_fragment_list_strict parameter) fragment_up (ditto.utils.TransformerUtils.add_downstream_dag_fragment parameter) from_op (ditto.api.OperatorTransformer.copy_op_attrs parameter)

G

get_cluster_name() (ditto.transformers.emr.EmrCreateJobFlowOperatorTransformer method) get_default_op() (ditto.api.OperatorTransformer method) get_digraph_from_airflow_dag() (ditto.utils.TransformerUtils static method) get_digraph_from_matcher_dag() (ditto.utils.TransformerUtils static method) get_list_index_from_xcom_pull() (ditto.utils.TransformerUtils static method)

get_sub_dag_matcher() (ditto.api.SubDagTransformer method) (ditto.transformers.subdag.CheckClusterSubDagTransformer method) get_target_step_task_id() (ditto.transformers.emr.EmrAddStepsOperatorTransformer static method) get_task_id_from_xcom_pull() (ditto.utils.TransformerUtils static method) get_transformer_for_class() (ditto.resolvers.ClassTransformerResolver method)

H

HADOOP_DISTCP_DEFAULT_MAPPERS (ditto.transformers.emr.EmrAddStepsOperatorTransformer attribute)

I

IdentityTransformer (class in ditto.transformers)

M

matcher (ditto.utils.TransformerUtils.find_matching_tasks parameter) (ditto.utils.TransformerUtils.find_sub_dag parameter) module ditto.api ditto.ditto ditto.rendering ditto.transformers ditto.utils

N

nested_search (ditto.matchers.PythonCallTaskMatcher parameter), [1] (ditto.resolvers.PythonCallTransformerResolver parameter), [1]

O

op (ditto.utils.TransformerUtils.assign_task_to_dag parameter) operator_class (ditto.matchers.ClassTaskMatcher parameter), [1] operator_transformers (ditto.resolvers.AncestralClassTransformerResolver parameter), [1] (ditto.resolvers.ClassTransformerResolver parameter), [1] operator_type (ditto.utils.TransformerUtils.find_op_in_dag_fragment parameter) (ditto.utils.TransformerUtils.find_op_in_fragment_list parameter) (ditto.utils.TransformerUtils.find_op_in_fragment_list_strict parameter) (ditto.utils.TransformerUtils.find_op_in_parent_fragment_chain parameter)

OperatorTransformer (class in ditto.api) other (ditto.api.TaskMatcher.set_downstream parameter) (ditto.api.TaskMatcher.set_upstream parameter) other_defaults (ditto.api.TransformerDefaults attribute) output_op (ditto.api.OperatorTransformer.sign_op parameter)

P

parent_fragment (ditto.api.OperatorTransformer.transform parameter) (ditto.api.SubDagTransformer.transform parameter) (ditto.transformers.CopyTransformer.transform parameter) (ditto.transformers.IdentityTransformer.transform parameter) (ditto.transformers.subdag.CheckClusterSubDagTransformer.transform parameter) (ditto.utils.TransformerUtils.find_op_in_parent_fragment_chain parameter)

parents (ditto.api.DAGFragment parameter) (ditto.api.TaskMatcher attribute) PythonCallTaskMatcher (class in ditto.matchers) PythonCallTransformerResolver (class in ditto.resolvers)

R

remove_task_from_dag() (ditto.utils.TransformerUtils static method) resolve_transformer() (ditto.api.TransformerResolver method) (ditto.resolvers.AncestralClassTransformerResolver method) (ditto.resolvers.ClassTransformerResolver method) (ditto.resolvers.PythonCallTransformerResolver method)

S

S3KeySensorAdlsGen1OperatorTransformer (class in ditto.transformers.s3) S3KeySensorBlobOperatorTransformer (class in ditto.transformers.s3) set_downstream() (ditto.api.TaskMatcher method) set_upstream() (ditto.api.TaskMatcher method) show_multi_dag_graphviz() (in module ditto.rendering) show_single_dag_graphviz() (in module ditto.rendering) sign_op() (ditto.api.OperatorTransformer method) src_dag (ditto.ditto.AirflowDagTransformer.transform parameter) (ditto.ditto.AirflowDagTransformer.transform_operators parameter) (ditto.ditto.AirflowDagTransformer.transform_sub_dags parameter)

src_operator (ditto.api.OperatorTransformer.get_default_op parameter) (ditto.api.OperatorTransformer.transform parameter) (ditto.transformers.CopyTransformer.transform parameter) (ditto.transformers.IdentityTransformer.transform parameter) subdag (ditto.api.SubDagTransformer.transform parameter) (ditto.transformers.subdag.CheckClusterSubDagTransformer.transform parameter) (ditto.utils.TransformerUtils.find_matching_tasks parameter) subdag_transformers (ditto.ditto.AirflowDagTransformer parameter) (ditto.templates.CheckClusterEmr2HdiDagTransformerTemplate parameter), [1] (ditto.templates.EmrHdiDagTransformerTemplate parameter), [1] SubDagTransformer (class in ditto.api)

T

target_dag (ditto.api.OperatorTransformer parameter) (ditto.ditto.AirflowDagTransformer parameter) (ditto.templates.CheckClusterEmr2HdiDagTransformerTemplate parameter), [1] (ditto.templates.EmrHdiDagTransformerTemplate parameter), [1] (ditto.transformers.CopyTransformer parameter) (ditto.transformers.emr.EmrAddStepsOperatorTransformer parameter), [1] (ditto.transformers.emr.EmrCreateJobFlowOperatorTransformer parameter), [1] (ditto.transformers.emr.EmrJobFlowSensorTransformer parameter), [1] (ditto.transformers.emr.EmrStepSensorTransformer parameter), [1] (ditto.transformers.emr.EmrTerminateJobFlowOperatorTransformer parameter), [1] (ditto.transformers.IdentityTransformer parameter) (ditto.transformers.s3.S3KeySensorAdlsGen1OperatorTransformer parameter), [1] (ditto.transformers.s3.S3KeySensorBlobOperatorTransformer parameter), [1] task (ditto.api.TaskMatcher.does_match parameter) (ditto.api.TransformerResolver.resolve_transformer parameter) (ditto.matchers.ClassTaskMatcher.does_match parameter) (ditto.matchers.PythonCallTaskMatcher.does_match parameter) (ditto.resolvers.AncestralClassTransformerResolver.resolve_transformer parameter) (ditto.resolvers.ClassTransformerResolver.resolve_transformer parameter) (ditto.resolvers.PythonCallTransformerResolver.resolve_transformer parameter) (ditto.utils.TransformerUtils.find_sub_dag parameter) (ditto.utils.TransformerUtils.remove_task_from_dag parameter) task_cls (ditto.resolvers.ClassTransformerResolver.get_transformer_for_class parameter) task_id (ditto.utils.TransformerUtils.find_op_in_dag_fragment parameter) (ditto.utils.TransformerUtils.find_op_in_fragment_list parameter) (ditto.utils.TransformerUtils.find_op_in_fragment_list_strict parameter) (ditto.utils.TransformerUtils.find_op_in_parent_fragment_chain parameter) TaskMatcher (class in ditto.api) tasks (ditto.api.DAGFragment parameter)

to_op (ditto.api.OperatorTransformer.copy_op_attrs parameter) transform() (ditto.api.OperatorTransformer method) (ditto.api.SubDagTransformer method) (ditto.ditto.AirflowDagTransformer method) (ditto.transformers.CopyTransformer method) (ditto.transformers.emr.EmrAddStepsOperatorTransformer method) (ditto.transformers.emr.EmrCreateJobFlowOperatorTransformer method) (ditto.transformers.emr.EmrJobFlowSensorTransformer method) (ditto.transformers.emr.EmrStepSensorTransformer method) (ditto.transformers.emr.EmrTerminateJobFlowOperatorTransformer method) (ditto.transformers.IdentityTransformer method) (ditto.transformers.s3.S3KeySensorAdlsGen1OperatorTransformer method) (ditto.transformers.s3.S3KeySensorBlobOperatorTransformer method) (ditto.transformers.subdag.CheckClusterSubDagTransformer method) transform_operators() (ditto.ditto.AirflowDagTransformer method) transform_sub_dags() (ditto.ditto.AirflowDagTransformer method) TRANSFORMED_BY_HEADER (ditto.api.Transformer attribute) Transformer (class in ditto.api) transformer_defaults (ditto.ditto.AirflowDagTransformer parameter) (ditto.templates.CheckClusterEmr2HdiDagTransformerTemplate parameter), [1] (ditto.templates.EmrHdiDagTransformerTemplate parameter), [1] transformer_resolvers (ditto.ditto.AirflowDagTransformer parameter) (ditto.templates.CheckClusterEmr2HdiDagTransformerTemplate parameter), [1] (ditto.templates.EmrHdiDagTransformerTemplate parameter), [1] TransformerDefaults (class in ditto.api) TransformerDefaultsConf (class in ditto.api) TransformerException TransformerResolver (class in ditto.api) TransformerUtils (class in ditto.utils)

U

upstream (ditto.utils.TransformerUtils.find_op_in_dag_fragment parameter) upstream_fragments (ditto.api.OperatorTransformer.transform parameter) (ditto.transformers.CopyTransformer.transform parameter) (ditto.transformers.IdentityTransformer.transform parameter)

UpstreamOperatorNotFoundException ut_colorer() (in module ditto.rendering) ut_relabeler() (in module ditto.rendering)

X

xcom_template (ditto.utils.TransformerUtils.get_list_index_from_xcom_pull parameter) (ditto.utils.TransformerUtils.get_task_id_from_xcom_pull parameter)