dlt.destinations.impl.filesystem.filesystem

FilesystemLoadJob Objects

class FilesystemLoadJob(RunnableLoadJob)

[view_source]

make_remote_path

def make_remote_path() -> str

[view_source]

Returns path on the remote filesystem to which copy the file, without scheme. For local filesystem a native path is used

make_remote_url

def make_remote_url() -> str

[view_source]

Returns path on a remote filesystem as a full url including scheme.

FilesystemClient Objects

class FilesystemClient(FSClientBase, WithSqlClient, JobClientBase,
                       WithStagingDataset, WithStateSync)

[view_source]

filesystem client storing jobs in memory

dataset_path

@property
def dataset_path() -> str

[view_source]

A path within a bucket to tables in a dataset NOTE: dataset_name changes if with_staging_dataset is active

get_storage_tables

def get_storage_tables(
        table_names: Iterable[str]
) -> Iterable[Tuple[str, TTableSchemaColumns]]

[view_source]

Yields tables that have files in storage, does not return column schemas

truncate_tables

def truncate_tables(table_names: List[str]) -> None

[view_source]

Truncate a set of regular tables with given table_names

get_table_dirs

def get_table_dirs(table_names: Iterable[str],
                   remote: bool = False) -> List[str]

[view_source]

Gets directories where table data is stored.

list_table_files

def list_table_files(table_name: str) -> List[str]

[view_source]

gets list of files associated with one table

list_files_with_prefixes

def list_files_with_prefixes(table_dir: str, prefixes: List[str]) -> List[str]

[view_source]

returns all files in a directory that match given prefixes

make_remote_url

def make_remote_url(remote_path: str) -> str

[view_source]

Returns uri to the remote filesystem to which copy the file

get_stored_schema

def get_stored_schema(schema_name: str = None) -> Optional[StorageSchemaInfo]

[view_source]

Retrieves newest schema from destination storage