dlt.destinations.impl.filesystem.filesystem
FilesystemLoadJob Objects
class FilesystemLoadJob(RunnableLoadJob)
make_remote_path
def make_remote_path() -> str
Returns path on the remote filesystem to which copy the file, without scheme. For local filesystem a native path is used
make_remote_url
def make_remote_url() -> str
Returns path on a remote filesystem as a full url including scheme.
FilesystemClient Objects
class FilesystemClient(FSClientBase, WithSqlClient, JobClientBase,
WithStagingDataset, WithStateSync, SupportsOpenTables)
dataset_path
@property
def dataset_path() -> str
A path within a bucket to tables in a dataset NOTE: dataset_name changes if with_staging_dataset is active
get_storage_tables
def get_storage_tables(
table_names: Iterable[str]
) -> Iterable[Tuple[str, TTableSchemaColumns]]
Yields tables that have files in storage, does not return column schemas
truncate_tables
def truncate_tables(table_names: List[str]) -> None
Truncate a set of regular tables with given table_names
get_table_dirs
def get_table_dirs(table_names: Iterable[str],
remote: bool = False) -> List[str]
Gets directories where table data is stored.
list_table_files
def list_table_files(table_name: str) -> List[str]
gets list of files associated with one table
list_files_with_prefixes
def list_files_with_prefixes(table_dir: str, prefixes: List[str]) -> List[str]
returns all files in a directory that match given prefixes
make_remote_url
def make_remote_url(remote_path: str) -> str
Returns uri to the remote filesystem to which copy the file
get_stored_schema
def get_stored_schema(schema_name: str = None) -> Optional[StorageSchemaInfo]
Retrieves newest schema from destination storage
load_open_table
def load_open_table(table_format: TTableFormat, table_name: str,
**kwargs: Any) -> Any
Locates, loads and returns native table client for table table_name in delta or iceberg formats
get_open_table_catalog
def get_open_table_catalog(table_format: TTableFormat,
catalog_name: str = None) -> Any
Gets a native catalog for a table table_name with format table_format
Returns: currently pyiceberg Catalog is supported
get_open_table_location
def get_open_table_location(table_format: TTableFormat,
table_name: str) -> str
All tables have location, also those in "native" table format.