sqlmesh.dbt.common
1from __future__ import annotations 2 3import typing as t 4from dataclasses import dataclass, field, replace 5from pathlib import Path 6 7from pydantic import validator 8from sqlglot.helper import ensure_list 9 10from sqlmesh.core.config.base import BaseConfig, UpdateStrategy 11from sqlmesh.core.engine_adapter import EngineAdapter 12from sqlmesh.dbt.target import TargetConfig 13from sqlmesh.utils import AttributeDict 14from sqlmesh.utils.conversions import ensure_bool, try_str_to_bool 15from sqlmesh.utils.errors import ConfigError 16from sqlmesh.utils.jinja import JinjaGlobalAttribute, JinjaMacroRegistry 17from sqlmesh.utils.pydantic import PydanticModel 18from sqlmesh.utils.yaml import load 19 20if t.TYPE_CHECKING: 21 from jinja2 import Environment 22 23 from sqlmesh.dbt.model import ModelConfig 24 from sqlmesh.dbt.seed import SeedConfig 25 from sqlmesh.dbt.source import SourceConfig 26 27T = t.TypeVar("T", bound="GeneralConfig") 28 29 30PROJECT_FILENAME = "dbt_project.yml" 31 32JINJA_ONLY = { 33 "adapter", 34 "api", 35 "exceptions", 36 "flags", 37 "load_result", 38 "modules", 39 "run_query", 40 "statement", 41 "store_result", 42 "target", 43} 44 45 46def load_yaml(source: str | Path) -> t.OrderedDict: 47 return load(source, render_jinja=False) 48 49 50@dataclass 51class DbtContext: 52 """Context for DBT environment""" 53 54 project_root: Path = Path() 55 target_name: t.Optional[str] = None 56 project_name: t.Optional[str] = None 57 profile_name: t.Optional[str] = None 58 project_schema: t.Optional[str] = None 59 jinja_macros: JinjaMacroRegistry = field( 60 default_factory=lambda: JinjaMacroRegistry(create_builtins_module="sqlmesh.dbt") 61 ) 62 63 engine_adapter: t.Optional[EngineAdapter] = None 64 65 _variables: t.Dict[str, t.Any] = field(default_factory=dict) 66 _models: t.Dict[str, ModelConfig] = field(default_factory=dict) 67 _seeds: t.Dict[str, SeedConfig] = field(default_factory=dict) 68 _sources: t.Dict[str, SourceConfig] = field(default_factory=dict) 69 _refs: t.Dict[str, str] = field(default_factory=dict) 70 71 _target: t.Optional[TargetConfig] = None 72 73 _jinja_environment: t.Optional[Environment] = None 74 75 @property 76 def dialect(self) -> str: 77 return self.engine_adapter.dialect if self.engine_adapter is not None else "" 78 79 @property 80 def variables(self) -> t.Dict[str, t.Any]: 81 return self._variables 82 83 @variables.setter 84 def variables(self, variables: t.Dict[str, t.Any]) -> None: 85 self._variables = {} 86 self.add_variables(variables) 87 88 def add_variables(self, variables: t.Dict[str, t.Any]) -> None: 89 self._variables.update(variables) 90 self._jinja_environment = None 91 92 @property 93 def models(self) -> t.Dict[str, ModelConfig]: 94 return self._models 95 96 @models.setter 97 def models(self, models: t.Dict[str, ModelConfig]) -> None: 98 self._models = {} 99 self._refs = {} 100 self.add_models(models) 101 102 def add_models(self, models: t.Dict[str, ModelConfig]) -> None: 103 self._refs = {} 104 self._models.update(models) 105 self._jinja_environment = None 106 107 @property 108 def seeds(self) -> t.Dict[str, SeedConfig]: 109 return self._seeds 110 111 @seeds.setter 112 def seeds(self, seeds: t.Dict[str, SeedConfig]) -> None: 113 self._seeds = {} 114 self._refs = {} 115 self.add_seeds(seeds) 116 117 def add_seeds(self, seeds: t.Dict[str, SeedConfig]) -> None: 118 self._refs = {} 119 self._seeds.update(seeds) 120 self._jinja_environment = None 121 122 @property 123 def sources(self) -> t.Dict[str, SourceConfig]: 124 return self._sources 125 126 @sources.setter 127 def sources(self, sources: t.Dict[str, SourceConfig]) -> None: 128 self._sources = {} 129 self.add_sources(sources) 130 131 def add_sources(self, sources: t.Dict[str, SourceConfig]) -> None: 132 self._sources.update(sources) 133 self._jinja_environment = None 134 135 @property 136 def refs(self) -> t.Dict[str, str]: 137 if not self._refs: 138 self._refs = {k: v.model_name for k, v in {**self._seeds, **self._models}.items()} # type: ignore 139 return self._refs 140 141 @property 142 def target(self) -> TargetConfig: 143 if not self._target: 144 raise ConfigError(f"Target not set for {self.project_name}") 145 return self._target 146 147 @target.setter 148 def target(self, value: TargetConfig) -> None: 149 if not self.project_name: 150 raise ConfigError("Project name must be set in the context in order to use a target.") 151 152 self._target = value 153 self.engine_adapter = self._target.to_sqlmesh().create_engine_adapter() 154 self._jinja_environment = None 155 156 def render(self, source: str, **kwargs: t.Any) -> str: 157 return self.jinja_environment.from_string(source).render(**kwargs) 158 159 def copy(self) -> DbtContext: 160 return replace(self) 161 162 @property 163 def jinja_environment(self) -> Environment: 164 if self._jinja_environment is None: 165 self._jinja_environment = self.jinja_macros.build_environment( 166 **self.jinja_globals, engine_adapter=self.engine_adapter 167 ) 168 return self._jinja_environment 169 170 @property 171 def jinja_globals(self) -> t.Dict[str, JinjaGlobalAttribute]: 172 refs: t.Dict[str, t.Union[ModelConfig, SeedConfig]] = {**self.models, **self.seeds} 173 output: t.Dict[str, JinjaGlobalAttribute] = { 174 "vars": AttributeDict(self.variables), 175 "refs": AttributeDict({k: v.relation_info for k, v in refs.items()}), 176 "sources": AttributeDict({k: v.relation_info for k, v in self.sources.items()}), 177 } 178 if self.project_name is not None: 179 output["project_name"] = self.project_name 180 if self._target is not None and self.project_name is not None: 181 output["target"] = self._target.target_jinja(self.project_name) 182 return output 183 184 185class SqlStr(str): 186 pass 187 188 189class DbtConfig(PydanticModel): 190 class Config: 191 extra = "allow" 192 allow_mutation = True 193 194 195class GeneralConfig(DbtConfig, BaseConfig): 196 """ 197 General DBT configuration properties for models, sources, seeds, columns, etc. 198 199 Args: 200 description: Description of element 201 tests: Tests for the element 202 enabled: When false, the element is ignored 203 docs: Documentation specific configuration 204 perist_docs: Persist resource descriptions as column and/or relation comments in the database 205 tags: List of tags that can be used for element grouping 206 meta: Dictionary of metadata for the element 207 """ 208 209 start: t.Optional[str] = None 210 description: t.Optional[str] = None 211 # TODO add test support 212 tests: t.Dict[str, t.Any] = {} 213 enabled: bool = True 214 docs: t.Dict[str, t.Any] = {"show": True} 215 persist_docs: t.Dict[str, t.Any] = {} 216 tags: t.List[str] = [] 217 meta: t.Dict[str, t.Any] = {} 218 219 @validator("enabled", pre=True) 220 def _validate_bool(cls, v: str) -> bool: 221 return ensure_bool(v) 222 223 @validator("docs", pre=True) 224 def _validate_dict(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: 225 for key, value in v.items(): 226 if isinstance(value, str): 227 v[key] = try_str_to_bool(value) 228 229 return v 230 231 @validator("persist_docs", pre=True) 232 def _validate_persist_docs(cls, v: t.Dict[str, str]) -> t.Dict[str, bool]: 233 return {key: bool(value) for key, value in v.items()} 234 235 @validator("tags", pre=True) 236 def _validate_list(cls, v: t.Union[str, t.List[str]]) -> t.List[str]: 237 return ensure_list(v) 238 239 @validator("meta", pre=True) 240 def _validate_meta(cls, v: t.Dict[str, t.Union[str, t.Any]]) -> t.Dict[str, t.Any]: 241 return parse_meta(v) 242 243 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 244 **BaseConfig._FIELD_UPDATE_STRATEGY, 245 **{ 246 "tests": UpdateStrategy.KEY_UPDATE, 247 "docs": UpdateStrategy.KEY_UPDATE, 248 "persist_docs": UpdateStrategy.KEY_UPDATE, 249 "tags": UpdateStrategy.EXTEND, 250 "meta": UpdateStrategy.KEY_UPDATE, 251 }, 252 } 253 254 _SQL_FIELDS: t.ClassVar[t.List[str]] = [] 255 256 def replace(self, other: T) -> None: 257 """ 258 Replace the contents of this instance with the passed in instance. 259 260 Args: 261 other: The instance to apply to this instance 262 """ 263 for field in other.__fields_set__: 264 setattr(self, field, getattr(other, field)) 265 266 def render_config(self: T, context: DbtContext) -> T: 267 def render_value(val: t.Any) -> t.Any: 268 if type(val) is not SqlStr and type(val) is str: 269 val = context.render(val) 270 elif isinstance(val, GeneralConfig): 271 for name in val.__fields__: 272 setattr(val, name, render_value(getattr(val, name))) 273 elif isinstance(val, list): 274 for i in range(len(val)): 275 val[i] = render_value(val[i]) 276 elif isinstance(val, set): 277 for set_val in val: 278 val.remove(set_val) 279 val.add(render_value(set_val)) 280 elif isinstance(val, dict): 281 for k in val: 282 val[k] = render_value(val[k]) 283 284 return val 285 286 rendered = self.copy(deep=True) 287 for name in rendered.__fields__: 288 setattr(rendered, name, render_value(getattr(rendered, name))) 289 290 return rendered 291 292 293def parse_meta(v: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: 294 for key, value in v.items(): 295 if isinstance(value, str): 296 v[key] = try_str_to_bool(value) 297 298 return v
def
load_yaml(source: str | pathlib.Path) -> OrderedDict:
@dataclass
class
DbtContext:
51@dataclass 52class DbtContext: 53 """Context for DBT environment""" 54 55 project_root: Path = Path() 56 target_name: t.Optional[str] = None 57 project_name: t.Optional[str] = None 58 profile_name: t.Optional[str] = None 59 project_schema: t.Optional[str] = None 60 jinja_macros: JinjaMacroRegistry = field( 61 default_factory=lambda: JinjaMacroRegistry(create_builtins_module="sqlmesh.dbt") 62 ) 63 64 engine_adapter: t.Optional[EngineAdapter] = None 65 66 _variables: t.Dict[str, t.Any] = field(default_factory=dict) 67 _models: t.Dict[str, ModelConfig] = field(default_factory=dict) 68 _seeds: t.Dict[str, SeedConfig] = field(default_factory=dict) 69 _sources: t.Dict[str, SourceConfig] = field(default_factory=dict) 70 _refs: t.Dict[str, str] = field(default_factory=dict) 71 72 _target: t.Optional[TargetConfig] = None 73 74 _jinja_environment: t.Optional[Environment] = None 75 76 @property 77 def dialect(self) -> str: 78 return self.engine_adapter.dialect if self.engine_adapter is not None else "" 79 80 @property 81 def variables(self) -> t.Dict[str, t.Any]: 82 return self._variables 83 84 @variables.setter 85 def variables(self, variables: t.Dict[str, t.Any]) -> None: 86 self._variables = {} 87 self.add_variables(variables) 88 89 def add_variables(self, variables: t.Dict[str, t.Any]) -> None: 90 self._variables.update(variables) 91 self._jinja_environment = None 92 93 @property 94 def models(self) -> t.Dict[str, ModelConfig]: 95 return self._models 96 97 @models.setter 98 def models(self, models: t.Dict[str, ModelConfig]) -> None: 99 self._models = {} 100 self._refs = {} 101 self.add_models(models) 102 103 def add_models(self, models: t.Dict[str, ModelConfig]) -> None: 104 self._refs = {} 105 self._models.update(models) 106 self._jinja_environment = None 107 108 @property 109 def seeds(self) -> t.Dict[str, SeedConfig]: 110 return self._seeds 111 112 @seeds.setter 113 def seeds(self, seeds: t.Dict[str, SeedConfig]) -> None: 114 self._seeds = {} 115 self._refs = {} 116 self.add_seeds(seeds) 117 118 def add_seeds(self, seeds: t.Dict[str, SeedConfig]) -> None: 119 self._refs = {} 120 self._seeds.update(seeds) 121 self._jinja_environment = None 122 123 @property 124 def sources(self) -> t.Dict[str, SourceConfig]: 125 return self._sources 126 127 @sources.setter 128 def sources(self, sources: t.Dict[str, SourceConfig]) -> None: 129 self._sources = {} 130 self.add_sources(sources) 131 132 def add_sources(self, sources: t.Dict[str, SourceConfig]) -> None: 133 self._sources.update(sources) 134 self._jinja_environment = None 135 136 @property 137 def refs(self) -> t.Dict[str, str]: 138 if not self._refs: 139 self._refs = {k: v.model_name for k, v in {**self._seeds, **self._models}.items()} # type: ignore 140 return self._refs 141 142 @property 143 def target(self) -> TargetConfig: 144 if not self._target: 145 raise ConfigError(f"Target not set for {self.project_name}") 146 return self._target 147 148 @target.setter 149 def target(self, value: TargetConfig) -> None: 150 if not self.project_name: 151 raise ConfigError("Project name must be set in the context in order to use a target.") 152 153 self._target = value 154 self.engine_adapter = self._target.to_sqlmesh().create_engine_adapter() 155 self._jinja_environment = None 156 157 def render(self, source: str, **kwargs: t.Any) -> str: 158 return self.jinja_environment.from_string(source).render(**kwargs) 159 160 def copy(self) -> DbtContext: 161 return replace(self) 162 163 @property 164 def jinja_environment(self) -> Environment: 165 if self._jinja_environment is None: 166 self._jinja_environment = self.jinja_macros.build_environment( 167 **self.jinja_globals, engine_adapter=self.engine_adapter 168 ) 169 return self._jinja_environment 170 171 @property 172 def jinja_globals(self) -> t.Dict[str, JinjaGlobalAttribute]: 173 refs: t.Dict[str, t.Union[ModelConfig, SeedConfig]] = {**self.models, **self.seeds} 174 output: t.Dict[str, JinjaGlobalAttribute] = { 175 "vars": AttributeDict(self.variables), 176 "refs": AttributeDict({k: v.relation_info for k, v in refs.items()}), 177 "sources": AttributeDict({k: v.relation_info for k, v in self.sources.items()}), 178 } 179 if self.project_name is not None: 180 output["project_name"] = self.project_name 181 if self._target is not None and self.project_name is not None: 182 output["target"] = self._target.target_jinja(self.project_name) 183 return output
Context for DBT environment
DbtContext( project_root: pathlib.Path = PosixPath('.'), target_name: Optional[str] = None, project_name: Optional[str] = None, profile_name: Optional[str] = None, project_schema: Optional[str] = None, jinja_macros: sqlmesh.utils.jinja.JinjaMacroRegistry = <factory>, engine_adapter: Optional[sqlmesh.core.engine_adapter.base.EngineAdapter] = None, _variables: Dict[str, Any] = <factory>, _models: Dict[str, sqlmesh.dbt.model.ModelConfig] = <factory>, _seeds: Dict[str, sqlmesh.dbt.seed.SeedConfig] = <factory>, _sources: Dict[str, sqlmesh.dbt.source.SourceConfig] = <factory>, _refs: Dict[str, str] = <factory>, _target: Optional[sqlmesh.dbt.target.TargetConfig] = None, _jinja_environment: Optional[jinja2.environment.Environment] = None)
class
SqlStr(builtins.str):
str(object='') -> str str(bytes_or_buffer[, encoding[, errors]]) -> str
Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to 'strict'.
Inherited Members
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
class
DbtConfig.Config:
196class GeneralConfig(DbtConfig, BaseConfig): 197 """ 198 General DBT configuration properties for models, sources, seeds, columns, etc. 199 200 Args: 201 description: Description of element 202 tests: Tests for the element 203 enabled: When false, the element is ignored 204 docs: Documentation specific configuration 205 perist_docs: Persist resource descriptions as column and/or relation comments in the database 206 tags: List of tags that can be used for element grouping 207 meta: Dictionary of metadata for the element 208 """ 209 210 start: t.Optional[str] = None 211 description: t.Optional[str] = None 212 # TODO add test support 213 tests: t.Dict[str, t.Any] = {} 214 enabled: bool = True 215 docs: t.Dict[str, t.Any] = {"show": True} 216 persist_docs: t.Dict[str, t.Any] = {} 217 tags: t.List[str] = [] 218 meta: t.Dict[str, t.Any] = {} 219 220 @validator("enabled", pre=True) 221 def _validate_bool(cls, v: str) -> bool: 222 return ensure_bool(v) 223 224 @validator("docs", pre=True) 225 def _validate_dict(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: 226 for key, value in v.items(): 227 if isinstance(value, str): 228 v[key] = try_str_to_bool(value) 229 230 return v 231 232 @validator("persist_docs", pre=True) 233 def _validate_persist_docs(cls, v: t.Dict[str, str]) -> t.Dict[str, bool]: 234 return {key: bool(value) for key, value in v.items()} 235 236 @validator("tags", pre=True) 237 def _validate_list(cls, v: t.Union[str, t.List[str]]) -> t.List[str]: 238 return ensure_list(v) 239 240 @validator("meta", pre=True) 241 def _validate_meta(cls, v: t.Dict[str, t.Union[str, t.Any]]) -> t.Dict[str, t.Any]: 242 return parse_meta(v) 243 244 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 245 **BaseConfig._FIELD_UPDATE_STRATEGY, 246 **{ 247 "tests": UpdateStrategy.KEY_UPDATE, 248 "docs": UpdateStrategy.KEY_UPDATE, 249 "persist_docs": UpdateStrategy.KEY_UPDATE, 250 "tags": UpdateStrategy.EXTEND, 251 "meta": UpdateStrategy.KEY_UPDATE, 252 }, 253 } 254 255 _SQL_FIELDS: t.ClassVar[t.List[str]] = [] 256 257 def replace(self, other: T) -> None: 258 """ 259 Replace the contents of this instance with the passed in instance. 260 261 Args: 262 other: The instance to apply to this instance 263 """ 264 for field in other.__fields_set__: 265 setattr(self, field, getattr(other, field)) 266 267 def render_config(self: T, context: DbtContext) -> T: 268 def render_value(val: t.Any) -> t.Any: 269 if type(val) is not SqlStr and type(val) is str: 270 val = context.render(val) 271 elif isinstance(val, GeneralConfig): 272 for name in val.__fields__: 273 setattr(val, name, render_value(getattr(val, name))) 274 elif isinstance(val, list): 275 for i in range(len(val)): 276 val[i] = render_value(val[i]) 277 elif isinstance(val, set): 278 for set_val in val: 279 val.remove(set_val) 280 val.add(render_value(set_val)) 281 elif isinstance(val, dict): 282 for k in val: 283 val[k] = render_value(val[k]) 284 285 return val 286 287 rendered = self.copy(deep=True) 288 for name in rendered.__fields__: 289 setattr(rendered, name, render_value(getattr(rendered, name))) 290 291 return rendered
General DBT configuration properties for models, sources, seeds, columns, etc.
Arguments:
- description: Description of element
- tests: Tests for the element
- enabled: When false, the element is ignored
- docs: Documentation specific configuration
- perist_docs: Persist resource descriptions as column and/or relation comments in the database
- tags: List of tags that can be used for element grouping
- meta: Dictionary of metadata for the element
def
replace(self, other: ~T) -> None:
257 def replace(self, other: T) -> None: 258 """ 259 Replace the contents of this instance with the passed in instance. 260 261 Args: 262 other: The instance to apply to this instance 263 """ 264 for field in other.__fields_set__: 265 setattr(self, field, getattr(other, field))
Replace the contents of this instance with the passed in instance.
Arguments:
- other: The instance to apply to this instance
267 def render_config(self: T, context: DbtContext) -> T: 268 def render_value(val: t.Any) -> t.Any: 269 if type(val) is not SqlStr and type(val) is str: 270 val = context.render(val) 271 elif isinstance(val, GeneralConfig): 272 for name in val.__fields__: 273 setattr(val, name, render_value(getattr(val, name))) 274 elif isinstance(val, list): 275 for i in range(len(val)): 276 val[i] = render_value(val[i]) 277 elif isinstance(val, set): 278 for set_val in val: 279 val.remove(set_val) 280 val.add(render_value(set_val)) 281 elif isinstance(val, dict): 282 for k in val: 283 val[k] = render_value(val[k]) 284 285 return val 286 287 rendered = self.copy(deep=True) 288 for name in rendered.__fields__: 289 setattr(rendered, name, render_value(getattr(rendered, name))) 290 291 return rendered
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
def
parse_meta(v: Dict[str, Any]) -> Dict[str, Any]: