sqlmesh.dbt.source
1from __future__ import annotations 2 3import typing as t 4 5from dbt.contracts.relation import RelationType 6from pydantic import Field, validator 7 8from sqlmesh.core.config.base import UpdateStrategy 9from sqlmesh.dbt.column import ColumnConfig, yaml_to_columns 10from sqlmesh.dbt.common import GeneralConfig 11from sqlmesh.utils.conversions import ensure_bool 12 13 14class SourceConfig(GeneralConfig): 15 """ 16 Args: 17 config_name: The schema.table_name names declared in source config 18 name: The name of the source or table 19 database: Name of the database where the table is stored. By default, the project's target database is used. 20 schema: The scehma name as stored in the database. If not specified, the source name is used. 21 identifier: The table name as stored in the database. If not specified, the source table name is used 22 loader: Describes the tool that loads the source into the warehouse 23 overrides: Override a source defined in the specified package 24 freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh 25 loaded_at_field: Column name or expression that returns a timestamp indicating freshness 26 quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method 27 external: Dictionary of metadata properties specific to sources that point to external tables 28 columns: Columns within the source 29 """ 30 31 # sqlmesh fields 32 config_name: str = "" 33 34 # DBT configuration fields 35 name: t.Optional[str] = None 36 database: t.Optional[str] = None 37 schema_: t.Optional[str] = Field(None, alias="schema") 38 identifier: t.Optional[str] = None 39 loader: t.Optional[str] = None 40 overrides: t.Optional[str] = None 41 freshness: t.Optional[t.Dict[str, t.Any]] = {} 42 loaded_at_field: t.Optional[str] = None 43 quoting: t.Optional[t.Dict[str, bool]] = {} 44 external: t.Optional[t.Dict[str, t.Any]] = {} 45 columns: t.Dict[str, ColumnConfig] = {} 46 47 @validator("quoting", pre=True) 48 def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]: 49 return {key: ensure_bool(val) for key, val in v.items()} 50 51 @validator("columns", pre=True) 52 def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]: 53 return yaml_to_columns(v) 54 55 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 56 **GeneralConfig._FIELD_UPDATE_STRATEGY, 57 **{"columns": UpdateStrategy.KEY_EXTEND}, 58 } 59 60 @property 61 def table_name(self) -> t.Optional[str]: 62 return self.identifier or self.name 63 64 @property 65 def source_name(self) -> str: 66 return ".".join(part for part in (self.schema_, self.table_name) if part) 67 68 @property 69 def relation_info(self) -> t.Dict[str, t.Any]: 70 return { 71 "database": self.database, 72 "schema": self.schema_, 73 "identifier": self.table_name, 74 "type": RelationType.External.value, 75 }
15class SourceConfig(GeneralConfig): 16 """ 17 Args: 18 config_name: The schema.table_name names declared in source config 19 name: The name of the source or table 20 database: Name of the database where the table is stored. By default, the project's target database is used. 21 schema: The scehma name as stored in the database. If not specified, the source name is used. 22 identifier: The table name as stored in the database. If not specified, the source table name is used 23 loader: Describes the tool that loads the source into the warehouse 24 overrides: Override a source defined in the specified package 25 freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh 26 loaded_at_field: Column name or expression that returns a timestamp indicating freshness 27 quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method 28 external: Dictionary of metadata properties specific to sources that point to external tables 29 columns: Columns within the source 30 """ 31 32 # sqlmesh fields 33 config_name: str = "" 34 35 # DBT configuration fields 36 name: t.Optional[str] = None 37 database: t.Optional[str] = None 38 schema_: t.Optional[str] = Field(None, alias="schema") 39 identifier: t.Optional[str] = None 40 loader: t.Optional[str] = None 41 overrides: t.Optional[str] = None 42 freshness: t.Optional[t.Dict[str, t.Any]] = {} 43 loaded_at_field: t.Optional[str] = None 44 quoting: t.Optional[t.Dict[str, bool]] = {} 45 external: t.Optional[t.Dict[str, t.Any]] = {} 46 columns: t.Dict[str, ColumnConfig] = {} 47 48 @validator("quoting", pre=True) 49 def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]: 50 return {key: ensure_bool(val) for key, val in v.items()} 51 52 @validator("columns", pre=True) 53 def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]: 54 return yaml_to_columns(v) 55 56 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 57 **GeneralConfig._FIELD_UPDATE_STRATEGY, 58 **{"columns": UpdateStrategy.KEY_EXTEND}, 59 } 60 61 @property 62 def table_name(self) -> t.Optional[str]: 63 return self.identifier or self.name 64 65 @property 66 def source_name(self) -> str: 67 return ".".join(part for part in (self.schema_, self.table_name) if part) 68 69 @property 70 def relation_info(self) -> t.Dict[str, t.Any]: 71 return { 72 "database": self.database, 73 "schema": self.schema_, 74 "identifier": self.table_name, 75 "type": RelationType.External.value, 76 }
Arguments:
- config_name: The schema.table_name names declared in source config
- name: The name of the source or table
- database: Name of the database where the table is stored. By default, the project's target database is used.
- schema: The scehma name as stored in the database. If not specified, the source name is used.
- identifier: The table name as stored in the database. If not specified, the source table name is used
- loader: Describes the tool that loads the source into the warehouse
- overrides: Override a source defined in the specified package
- freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
- loaded_at_field: Column name or expression that returns a timestamp indicating freshness
- quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
- external: Dictionary of metadata properties specific to sources that point to external tables
- columns: Columns within the source
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs