extract.utils

get_data_item_format

def get_data_item_format(items: TDataItems) -> TDataItemFormat

[view_source]

Detect the format of the data item from items.

Reverts to object for empty lists

Returns:

The data file format.

resolve_column_value

def resolve_column_value(column_hint: TTableHintTemplate[TColumnNames],
                         item: TDataItem) -> Union[Any, List[Any]]

[view_source]

Extract values from the data item given a column hint. Returns either a single value or list of values when hint is a composite.

ensure_table_schema_columns

def ensure_table_schema_columns(
        columns: TAnySchemaColumns) -> TTableSchemaColumns

[view_source]

Convert supported column schema types to a column dict which can be used in resource schema.

Arguments:

• columns - A dict of column schemas, a list of column schemas, or a pydantic model

ensure_table_schema_columns_hint

def ensure_table_schema_columns_hint(
    columns: TTableHintTemplate[TAnySchemaColumns]
) -> TTableHintTemplate[TTableSchemaColumns]

[view_source]

Convert column schema hint to a hint returning TTableSchemaColumns. A callable hint is wrapped in another function which converts the original result.

reset_pipe_state

def reset_pipe_state(pipe: SupportsPipe,
                     source_state_: Optional[DictStrAny] = None) -> None

[view_source]

Resets the resource state for a pipe and all its parent pipes

simulate_func_call

def simulate_func_call(
    f: Union[Any, AnyFun], args_to_skip: int, *args: Any, **kwargs: Any
) -> Tuple[inspect.Signature, inspect.Signature, inspect.BoundArguments]

[view_source]

Simulates a call to a resource or transformer function before it will be wrapped for later execution in the pipe

Returns a tuple with a f signature, modified signature in case of transformers and bound arguments

wrap_async_iterator

def wrap_async_iterator(
    gen: AsyncIterator[TDataItems]
) -> Generator[Awaitable[TDataItems], None, None]

[view_source]

Wraps an async generator into a list of awaitables

wrap_parallel_iterator

def wrap_parallel_iterator(f: TAnyFunOrGenerator) -> TAnyFunOrGenerator

[view_source]

Wraps a generator for parallel extraction

wrap_compat_transformer

def wrap_compat_transformer(name: str, f: AnyFun, sig: inspect.Signature,
                            *args: Any, **kwargs: Any) -> AnyFun

[view_source]

Creates a compatible wrapper over transformer function. A pure transformer function expects data item in first argument and one keyword argument called meta

wrap_resource_gen

def wrap_resource_gen(name: str, f: AnyFun, sig: inspect.Signature, *args: Any,
                      **kwargs: Any) -> AnyFun

[view_source]

Wraps a generator or generator function so it is evaluated on extraction