dlt.common.storages.load_storage
LoadItemStorage Objects
class LoadItemStorage(DataItemStorage)
__init__
def __init__(package_storage: PackageStorage,
writer_spec: FileWriterSpec) -> None
Data item storage using storage to manage load packages
LoadStorage Objects
class LoadStorage(VersionedStorage)
NORMALIZED_FOLDER
folder within the volume where load packages are stored
LOADED_FOLDER
folder to keep the loads that were completely processed
NEW_PACKAGES_FOLDER
folder where new packages are created
list_new_jobs
def list_new_jobs(load_id: str) -> Sequence[str]
Lists all jobs in new jobs folder of normalized package storage and checks if file formats are supported
list_normalized_packages
def list_normalized_packages() -> Sequence[str]
Lists all packages that are normalized and will be loaded or are currently loaded
list_loaded_packages
def list_loaded_packages() -> Sequence[str]
List packages that are completely loaded
list_failed_jobs_in_loaded_package
def list_failed_jobs_in_loaded_package(load_id: str) -> Sequence[LoadJobInfo]
List all failed jobs and associated error messages for a completed load package with load_id
commit_schema_update
def commit_schema_update(load_id: str, applied_update: TSchemaTables) -> None
Marks schema update as processed by removing schema update file and saving the applied (effective) updates as reported by destination
import_new_job
def import_new_job(load_id: str,
job_file_path: str,
job_state: TPackageJobState = "new_jobs") -> None
Adds new job by moving the job_file_path into new_jobs of package load_id
maybe_remove_completed_jobs
def maybe_remove_completed_jobs(load_id: str) -> None
Deletes completed jobs if delete_completed_jobs config flag is set. If package has failed jobs, nothing gets deleted.
get_package_storage
def get_package_storage(load_id: str) -> PackageStorage
Returns the loaded or normalized package storage that holds load_id.
get_load_package_info
def get_load_package_info(load_id: str) -> LoadPackageInfo
Gets information on normalized OR loaded package with given load_id, all jobs and their statuses.
get_load_package_state
def get_load_package_state(load_id: str) -> TLoadPackageState
Gets state of normalized or loaded package with given load_id, all jobs and their statuses.