dlt.extract.pipe
ForkPipe Objects
class ForkPipe(ItemTransform[ResolvablePipeItem, Dict[str, Any]])
__init__
def __init__(pipe: "Pipe", step: int = -1, copy_on_fork: bool = False) -> None
A transformer that forks the pipe and sends the data items to forks added via add_pipe method.
Pipe Objects
class Pipe(SupportsPipe)
is_empty
@property
def is_empty() -> bool
Checks if pipe contains any steps
is_data_bound
@property
def is_data_bound() -> bool
Checks if pipe is bound to data and can be iterated. Pipe is bound if has a parent that is bound xor is not empty.
gen
@property
def gen() -> TPipeStep
A data generating step
find
def find(*step_type: AnyType) -> int
Finds a step with object of type step_type
find_all
def find_all(*step_type: AnyType) -> List[int]
Finds all steps with object of type step_type
get_by_type
def get_by_type(*step_type: AnyType) -> Optional[TPipeStep]
Gets first step found with object of type step_type
remove_by_type
def remove_by_type(*step_type: AnyType) -> int
Deletes first step found with object of type step_type, returns previous index
append_step
def append_step(step: TPipeStep) -> "Pipe"
Appends pipeline step. On first added step performs additional verification if step is a valid data generator
insert_step
def insert_step(step: TPipeStep, index: int) -> "Pipe"
Inserts step at a given index in the pipeline. Allows prepending only for transformers
remove_step
def remove_step(index: int) -> None
Removes steps at a given index. Gen step cannot be removed
replace_gen
def replace_gen(gen: TPipeStep) -> None
Replaces data generating step. Assumes that you know what are you doing
close
def close() -> None
Closes pipe generator
full_pipe
def full_pipe() -> "Pipe"
Creates a pipe that from the current and all the parent pipes.
ensure_gen_bound
def ensure_gen_bound() -> None
Verifies that gen step is bound to data
evaluate_gen
def evaluate_gen() -> None
Lazily evaluate gen of the pipe when creating PipeIterator. Allows creating multiple use pipes from generator functions and lists
bind_gen
def bind_gen(*args: Any, **kwargs: Any) -> Any
Finds and wraps with args + kwargs the callable generating step in the resource pipe and then replaces the pipe gen with the wrapped one
sim_gen
def sim_gen(
*args: Any, **kwargs: Any
) -> Tuple[inspect.Signature, inspect.Signature, inspect.BoundArguments]
Simulates calling the gen element