dlt.common.runtime.collector_base
Collector Objects
class Collector(ABC, SupportsTracking)
update
@abstractmethod
def update(name: str,
inc: int = 1,
total: int = None,
inc_total: int = None,
message: str = None,
label: str = None) -> None
Creates or updates a counter
This function updates a counter name
with a value inc
. If counter does not exist, it is
created with optional total value of total
. Labels are used to create sub-counters.
Examples for counters by stage:
# Extract Stage
collector.update("users", inc=5) # 5 rows added to users table
collector.update("Resources", inc=1) # 1 resource processed
collector.update("Resources", inc=1, label="Completed") # 1 resource completed
# Normalize Stage
collector.update("Files", inc=1, total=10) # 1/10 files processed
collector.update("Items", inc=100) # 100 items processed
# Load Stage
collector.update("Jobs", inc=1, total=5) # 1/5 jobs processed
collector.update("Jobs", inc=1, label="Failed") # 1 job failed
__call__
def __call__(step: str) -> TCollector
Syntactic sugar for nicer context managers