destinations.sql_client
TJobQueryTags Objects
class TJobQueryTags(TypedDict)
Applied to sql client when a job using it starts. Using to tag queries
SqlClientBase Objects
class SqlClientBase(ABC, Generic[TNativeConn])
database_name
Database or catalog name, optional
dataset_name
Normalized dataset name
staging_dataset_name
Normalized staging dataset name
capabilities
Instance of adjusted destination capabilities
drop_tables
def drop_tables(*tables: str) -> None
Drops a set of tables if they exist
execute_fragments
def execute_fragments(fragments: Sequence[AnyStr], *args: Any,
**kwargs: Any) -> Optional[Sequence[Sequence[Any]]]
Executes several SQL fragments as efficiently as possible to prevent data copying. Default implementation just joins the strings and executes them together.
execute_many
def execute_many(statements: Sequence[str], *args: Any,
**kwargs: Any) -> Optional[Sequence[Sequence[Any]]]
Executes multiple SQL statements as efficiently as possible. When client supports multiple statements in a single query they are executed together in as few database calls as possible.
make_qualified_table_name_path
def make_qualified_table_name_path(table_name: Optional[str],
escape: bool = True) -> List[str]
Returns a list with path components leading from catalog to table_name.
Used to construct fully qualified names. table_name is optional.
get_qualified_table_names
def get_qualified_table_names(table_name: str,
escape: bool = True) -> Tuple[str, str]
Returns qualified names for table and corresponding staging table as tuple.
with_alternative_dataset_name
@contextmanager
def with_alternative_dataset_name(
dataset_name: str) -> Iterator["SqlClientBase[TNativeConn]"]
Sets the dataset_name as the default dataset during the lifetime of the context. Does not modify any search paths in the existing connection.
with_staging_dataset
def with_staging_dataset() -> ContextManager["SqlClientBase[TNativeConn]"]
Temporarily switch sql client to staging dataset name
is_staging_dataset_active
@property
def is_staging_dataset_active() -> bool
Checks if staging dataset is currently active
set_query_tags
def set_query_tags(tags: TJobQueryTags) -> None
Sets current schema (source), resource, load_id and table name when a job starts
DBApiCursorImpl Objects
class DBApiCursorImpl(DBApiCursor)
A DBApi Cursor wrapper with dataframes reading functionality
df
def df(chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]
Fetches results as data frame in full or in specified chunks.
May use native pandas/arrow reader if available. Depending on the native implementation chunk size may vary.
arrow
def arrow(chunk_size: int = None, **kwargs: Any) -> Optional[ArrowTable]
Fetches results as data frame in full or in specified chunks.
May use native pandas/arrow reader if available. Depending on the native implementation chunk size may vary.
iter_df
def iter_df(chunk_size: int) -> Generator[DataFrame, None, None]
Default implementation converts arrow to df
iter_arrow
def iter_arrow(chunk_size: int) -> Generator[ArrowTable, None, None]
Default implementation converts query result to arrow table