sqlmesh.dbt.basemodel
1from __future__ import annotations 2 3import typing as t 4from abc import abstractmethod 5from enum import Enum 6from pathlib import Path 7 8from dbt.adapters.base import BaseRelation 9from dbt.contracts.relation import RelationType 10from pydantic import Field, validator 11from sqlglot.helper import ensure_list 12 13from sqlmesh.core import constants as c 14from sqlmesh.core import dialect as d 15from sqlmesh.core.config.base import UpdateStrategy 16from sqlmesh.core.model import Model 17from sqlmesh.dbt.adapter import ParsetimeAdapter 18from sqlmesh.dbt.builtin import create_builtin_globals 19from sqlmesh.dbt.column import ( 20 ColumnConfig, 21 column_descriptions_to_sqlmesh, 22 column_types_to_sqlmesh, 23 yaml_to_columns, 24) 25from sqlmesh.dbt.common import DbtContext, GeneralConfig, SqlStr 26from sqlmesh.utils import AttributeDict 27from sqlmesh.utils.conversions import ensure_bool 28from sqlmesh.utils.date import date_dict 29from sqlmesh.utils.errors import ConfigError 30from sqlmesh.utils.jinja import MacroReference, extract_macro_references 31from sqlmesh.utils.pydantic import PydanticModel 32 33BMC = t.TypeVar("BMC", bound="BaseModelConfig") 34 35 36class Dependencies(PydanticModel): 37 """ 38 DBT dependencies for a model, macro, etc. 39 40 Args: 41 macros: The references to macros 42 sources: The "source_name.table_name" for source tables used 43 refs: The table_name for models used 44 variables: The names of variables used, mapped to a flag that indicates whether their 45 definition is optional or not. 46 """ 47 48 macros: t.Set[MacroReference] = set() 49 sources: t.Set[str] = set() 50 refs: t.Set[str] = set() 51 variables: t.Set[str] = set() 52 53 def union(self, other: Dependencies) -> Dependencies: 54 dependencies = Dependencies() 55 dependencies.macros = self.macros | other.macros 56 dependencies.sources = self.sources | other.sources 57 dependencies.refs = self.refs | other.refs 58 dependencies.variables = self.variables | other.variables 59 60 return dependencies 61 62 63class Materialization(str, Enum): 64 """DBT model materializations""" 65 66 TABLE = "table" 67 VIEW = "view" 68 INCREMENTAL = "incremental" 69 EPHEMERAL = "ephemeral" 70 71 72class BaseModelConfig(GeneralConfig): 73 """ 74 Args: 75 owner: The owner of the model. 76 stamp: An optional arbitrary string sequence used to create new model versions without making 77 changes to any of the functional components of the definition. 78 storage_format: The storage format used to store the physical table, only applicable in certain engines. 79 (eg. 'parquet') 80 path: The file path of the model 81 target_schema: The schema for the profile target 82 database: Database the model is stored in 83 schema: Custom schema name added to the model schema name 84 alias: Relation identifier for this model instead of the filename 85 pre-hook: List of SQL statements to run before the model is built 86 post-hook: List of SQL statements to run after the model is built 87 full_refresh: Forces the model to always do a full refresh or never do a full refresh 88 grants: Set or revoke permissions to the database object for this model 89 columns: Column information for the model 90 """ 91 92 # sqlmesh fields 93 owner: t.Optional[str] = None 94 stamp: t.Optional[str] = None 95 storage_format: t.Optional[str] = None 96 path: Path = Path() 97 target_schema: str = "" 98 _dependencies: Dependencies = Dependencies() 99 _variables: t.Dict[str, bool] = {} 100 101 # DBT configuration fields 102 database: t.Optional[str] = None 103 schema_: t.Optional[str] = Field(None, alias="schema") 104 alias: t.Optional[str] = None 105 pre_hook: t.List[SqlStr] = Field([], alias="pre-hook") 106 post_hook: t.List[SqlStr] = Field([], alias="post-hook") 107 full_refresh: t.Optional[bool] = None 108 grants: t.Dict[str, t.List[str]] = {} 109 columns: t.Dict[str, ColumnConfig] = {} 110 111 @validator("pre_hook", "post_hook", pre=True) 112 def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]: 113 return [SqlStr(val) for val in ensure_list(v)] 114 115 @validator("full_refresh", pre=True) 116 def _validate_bool(cls, v: str) -> bool: 117 return ensure_bool(v) 118 119 @validator("grants", pre=True) 120 def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]: 121 return {key: ensure_list(value) for key, value in v.items()} 122 123 @validator("columns", pre=True) 124 def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]: 125 if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()): 126 return v 127 128 return yaml_to_columns(v) 129 130 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 131 **GeneralConfig._FIELD_UPDATE_STRATEGY, 132 **{ 133 "grants": UpdateStrategy.KEY_EXTEND, 134 "path": UpdateStrategy.IMMUTABLE, 135 "pre-hook": UpdateStrategy.EXTEND, 136 "post-hook": UpdateStrategy.EXTEND, 137 "columns": UpdateStrategy.KEY_EXTEND, 138 }, 139 } 140 141 @property 142 def all_sql(self) -> SqlStr: 143 return SqlStr("") 144 145 @property 146 def table_schema(self) -> str: 147 """ 148 Get the full schema name 149 """ 150 return "_".join(part for part in (self.target_schema, self.schema_) if part) 151 152 @property 153 def table_name(self) -> str: 154 """ 155 Get the table name 156 """ 157 return self.alias or self.path.stem 158 159 @property 160 def model_name(self) -> str: 161 """ 162 Get the sqlmesh model name 163 164 Returns: 165 The sqlmesh model name 166 """ 167 return ".".join( 168 part for part in (self.database, self.table_schema, self.table_name) if part 169 ) 170 171 @property 172 def model_materialization(self) -> Materialization: 173 return Materialization.TABLE 174 175 @property 176 def relation_info(self) -> AttributeDict[str, t.Any]: 177 if self.model_materialization == Materialization.VIEW: 178 relation_type = RelationType.View 179 elif self.model_materialization == Materialization.EPHEMERAL: 180 relation_type = RelationType.CTE 181 else: 182 relation_type = RelationType.Table 183 184 return AttributeDict( 185 { 186 "database": self.database, 187 "schema": self.table_schema, 188 "identifier": self.table_name, 189 "type": relation_type.value, 190 } 191 ) 192 193 def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]: 194 """Get common sqlmesh model parameters""" 195 jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros) 196 jinja_macros.global_objs.update( 197 { 198 "this": self.relation_info, 199 "schema": self.table_schema, 200 **model_context.jinja_globals, # type: ignore 201 } 202 ) 203 204 optional_kwargs: t.Dict[str, t.Any] = {} 205 for field in ("description", "owner", "stamp", "storage_format"): 206 field_val = getattr(self, field, None) or self.meta.get(field, None) 207 if field_val: 208 optional_kwargs[field] = field_val 209 210 return { 211 "columns": column_types_to_sqlmesh(self.columns) or None, 212 "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None, 213 "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs}, 214 "jinja_macros": jinja_macros, 215 "path": self.path, 216 "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)], 217 "post": [exp for hook in self.post_hook for exp in d.parse(hook)], 218 **optional_kwargs, 219 } 220 221 def render_config(self: BMC, context: DbtContext) -> BMC: 222 rendered = super().render_config(context) 223 rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql)) 224 rendered = ModelSqlRenderer(context, rendered).enriched_config 225 226 rendered_dependencies = rendered._dependencies 227 for dependency in rendered_dependencies.refs: 228 model = context.models.get(dependency) 229 if model and model.materialized == Materialization.EPHEMERAL: 230 rendered._dependencies = rendered._dependencies.union( 231 model.render_config(context)._dependencies 232 ) 233 rendered._dependencies.refs.discard(dependency) 234 235 return rendered 236 237 @abstractmethod 238 def to_sqlmesh(self, context: DbtContext) -> Model: 239 """Convert DBT model into sqlmesh Model""" 240 241 def _context_for_dependencies( 242 self, context: DbtContext, dependencies: Dependencies 243 ) -> DbtContext: 244 model_context = context.copy() 245 246 model_context.sources = { 247 name: value for name, value in context.sources.items() if name in dependencies.sources 248 } 249 model_context.seeds = { 250 name: value for name, value in context.seeds.items() if name in dependencies.refs 251 } 252 model_context.models = { 253 name: value for name, value in context.models.items() if name in dependencies.refs 254 } 255 model_context.variables = { 256 name: value 257 for name, value in context.variables.items() 258 if name in dependencies.variables 259 } 260 261 return model_context 262 263 264class ModelSqlRenderer(t.Generic[BMC]): 265 def __init__(self, context: DbtContext, config: BMC): 266 self.context = context 267 self.config = config 268 269 self._captured_dependencies: Dependencies = Dependencies() 270 self._rendered_sql: t.Optional[str] = None 271 self._enriched_config: BMC = config.copy() 272 273 self._jinja_globals = create_builtin_globals( 274 jinja_macros=context.jinja_macros, 275 jinja_globals={ 276 **context.jinja_globals, 277 **date_dict(c.EPOCH, c.EPOCH, c.EPOCH), 278 "config": self._config, 279 "ref": self._ref, 280 "var": self._var, 281 "source": self._source, 282 "this": self.config.relation_info, 283 "schema": self.config.table_schema, 284 }, 285 engine_adapter=None, 286 ) 287 288 # Set the adapter separately since it requires jinja globals to passed into it. 289 self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter( 290 self, 291 context.jinja_macros, 292 jinja_globals=self._jinja_globals, 293 dialect=context.engine_adapter.dialect if context.engine_adapter else "", 294 ) 295 296 @property 297 def enriched_config(self) -> BMC: 298 if self._rendered_sql is None: 299 self.render() 300 self._enriched_config._dependencies = self._enriched_config._dependencies.union( 301 self._captured_dependencies 302 ) 303 return self._enriched_config 304 305 def render(self) -> str: 306 if self._rendered_sql is None: 307 registry = self.context.jinja_macros 308 self._rendered_sql = ( 309 registry.build_environment(**self._jinja_globals) 310 .from_string(self.config.all_sql) 311 .render() 312 ) 313 return self._rendered_sql 314 315 def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation: 316 if package_name in self.context.models: 317 relation = BaseRelation.create(**self.context.models[package_name].relation_info) 318 elif package_name in self.context.seeds: 319 relation = BaseRelation.create(**self.context.seeds[package_name].relation_info) 320 else: 321 raise ConfigError( 322 f"Model '{package_name}' was not found for model '{self.config.table_name}'." 323 ) 324 self._captured_dependencies.refs.add(package_name) 325 return relation 326 327 def _var(self, name: str, default: t.Optional[str] = None) -> t.Any: 328 if default is None and name not in self.context.variables: 329 raise ConfigError( 330 f"Variable '{name}' was not found for model '{self.config.table_name}'." 331 ) 332 self._captured_dependencies.variables.add(name) 333 return self.context.variables.get(name, default) 334 335 def _source(self, source_name: str, table_name: str) -> BaseRelation: 336 full_name = ".".join([source_name, table_name]) 337 if full_name not in self.context.sources: 338 raise ConfigError( 339 f"Source '{full_name}' was not found for model '{self.config.table_name}'." 340 ) 341 self._captured_dependencies.sources.add(full_name) 342 return BaseRelation.create(**self.context.sources[full_name].relation_info) 343 344 def _config(self, *args: t.Any, **kwargs: t.Any) -> str: 345 if args and isinstance(args[0], dict): 346 self._enriched_config = self._enriched_config.update_with(args[0]) 347 if kwargs: 348 self._enriched_config = self._enriched_config.update_with(kwargs) 349 return "" 350 351 class TrackingAdapter(ParsetimeAdapter): 352 def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any): 353 super().__init__(*args, **kwargs) 354 self.outer_self = outer_self 355 self.context = outer_self.context 356 357 def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable: 358 macros = ( 359 self.context.jinja_macros.packages.get(package, {}) 360 if package is not None 361 else self.context.jinja_macros.root_macros 362 ) 363 for target_name in macros: 364 if target_name.endswith(f"__{name}"): 365 self.outer_self._captured_dependencies.macros.add( 366 MacroReference(package=package, name=target_name) 367 ) 368 return super().dispatch(name, package=package)
37class Dependencies(PydanticModel): 38 """ 39 DBT dependencies for a model, macro, etc. 40 41 Args: 42 macros: The references to macros 43 sources: The "source_name.table_name" for source tables used 44 refs: The table_name for models used 45 variables: The names of variables used, mapped to a flag that indicates whether their 46 definition is optional or not. 47 """ 48 49 macros: t.Set[MacroReference] = set() 50 sources: t.Set[str] = set() 51 refs: t.Set[str] = set() 52 variables: t.Set[str] = set() 53 54 def union(self, other: Dependencies) -> Dependencies: 55 dependencies = Dependencies() 56 dependencies.macros = self.macros | other.macros 57 dependencies.sources = self.sources | other.sources 58 dependencies.refs = self.refs | other.refs 59 dependencies.variables = self.variables | other.variables 60 61 return dependencies
DBT dependencies for a model, macro, etc.
Arguments:
- macros: The references to macros
- sources: The "source_name.table_name" for source tables used
- refs: The table_name for models used
- variables: The names of variables used, mapped to a flag that indicates whether their definition is optional or not.
54 def union(self, other: Dependencies) -> Dependencies: 55 dependencies = Dependencies() 56 dependencies.macros = self.macros | other.macros 57 dependencies.sources = self.sources | other.sources 58 dependencies.refs = self.refs | other.refs 59 dependencies.variables = self.variables | other.variables 60 61 return dependencies
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
64class Materialization(str, Enum): 65 """DBT model materializations""" 66 67 TABLE = "table" 68 VIEW = "view" 69 INCREMENTAL = "incremental" 70 EPHEMERAL = "ephemeral"
DBT model materializations
Inherited Members
- enum.Enum
- name
- value
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
73class BaseModelConfig(GeneralConfig): 74 """ 75 Args: 76 owner: The owner of the model. 77 stamp: An optional arbitrary string sequence used to create new model versions without making 78 changes to any of the functional components of the definition. 79 storage_format: The storage format used to store the physical table, only applicable in certain engines. 80 (eg. 'parquet') 81 path: The file path of the model 82 target_schema: The schema for the profile target 83 database: Database the model is stored in 84 schema: Custom schema name added to the model schema name 85 alias: Relation identifier for this model instead of the filename 86 pre-hook: List of SQL statements to run before the model is built 87 post-hook: List of SQL statements to run after the model is built 88 full_refresh: Forces the model to always do a full refresh or never do a full refresh 89 grants: Set or revoke permissions to the database object for this model 90 columns: Column information for the model 91 """ 92 93 # sqlmesh fields 94 owner: t.Optional[str] = None 95 stamp: t.Optional[str] = None 96 storage_format: t.Optional[str] = None 97 path: Path = Path() 98 target_schema: str = "" 99 _dependencies: Dependencies = Dependencies() 100 _variables: t.Dict[str, bool] = {} 101 102 # DBT configuration fields 103 database: t.Optional[str] = None 104 schema_: t.Optional[str] = Field(None, alias="schema") 105 alias: t.Optional[str] = None 106 pre_hook: t.List[SqlStr] = Field([], alias="pre-hook") 107 post_hook: t.List[SqlStr] = Field([], alias="post-hook") 108 full_refresh: t.Optional[bool] = None 109 grants: t.Dict[str, t.List[str]] = {} 110 columns: t.Dict[str, ColumnConfig] = {} 111 112 @validator("pre_hook", "post_hook", pre=True) 113 def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]: 114 return [SqlStr(val) for val in ensure_list(v)] 115 116 @validator("full_refresh", pre=True) 117 def _validate_bool(cls, v: str) -> bool: 118 return ensure_bool(v) 119 120 @validator("grants", pre=True) 121 def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]: 122 return {key: ensure_list(value) for key, value in v.items()} 123 124 @validator("columns", pre=True) 125 def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]: 126 if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()): 127 return v 128 129 return yaml_to_columns(v) 130 131 _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = { 132 **GeneralConfig._FIELD_UPDATE_STRATEGY, 133 **{ 134 "grants": UpdateStrategy.KEY_EXTEND, 135 "path": UpdateStrategy.IMMUTABLE, 136 "pre-hook": UpdateStrategy.EXTEND, 137 "post-hook": UpdateStrategy.EXTEND, 138 "columns": UpdateStrategy.KEY_EXTEND, 139 }, 140 } 141 142 @property 143 def all_sql(self) -> SqlStr: 144 return SqlStr("") 145 146 @property 147 def table_schema(self) -> str: 148 """ 149 Get the full schema name 150 """ 151 return "_".join(part for part in (self.target_schema, self.schema_) if part) 152 153 @property 154 def table_name(self) -> str: 155 """ 156 Get the table name 157 """ 158 return self.alias or self.path.stem 159 160 @property 161 def model_name(self) -> str: 162 """ 163 Get the sqlmesh model name 164 165 Returns: 166 The sqlmesh model name 167 """ 168 return ".".join( 169 part for part in (self.database, self.table_schema, self.table_name) if part 170 ) 171 172 @property 173 def model_materialization(self) -> Materialization: 174 return Materialization.TABLE 175 176 @property 177 def relation_info(self) -> AttributeDict[str, t.Any]: 178 if self.model_materialization == Materialization.VIEW: 179 relation_type = RelationType.View 180 elif self.model_materialization == Materialization.EPHEMERAL: 181 relation_type = RelationType.CTE 182 else: 183 relation_type = RelationType.Table 184 185 return AttributeDict( 186 { 187 "database": self.database, 188 "schema": self.table_schema, 189 "identifier": self.table_name, 190 "type": relation_type.value, 191 } 192 ) 193 194 def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]: 195 """Get common sqlmesh model parameters""" 196 jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros) 197 jinja_macros.global_objs.update( 198 { 199 "this": self.relation_info, 200 "schema": self.table_schema, 201 **model_context.jinja_globals, # type: ignore 202 } 203 ) 204 205 optional_kwargs: t.Dict[str, t.Any] = {} 206 for field in ("description", "owner", "stamp", "storage_format"): 207 field_val = getattr(self, field, None) or self.meta.get(field, None) 208 if field_val: 209 optional_kwargs[field] = field_val 210 211 return { 212 "columns": column_types_to_sqlmesh(self.columns) or None, 213 "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None, 214 "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs}, 215 "jinja_macros": jinja_macros, 216 "path": self.path, 217 "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)], 218 "post": [exp for hook in self.post_hook for exp in d.parse(hook)], 219 **optional_kwargs, 220 } 221 222 def render_config(self: BMC, context: DbtContext) -> BMC: 223 rendered = super().render_config(context) 224 rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql)) 225 rendered = ModelSqlRenderer(context, rendered).enriched_config 226 227 rendered_dependencies = rendered._dependencies 228 for dependency in rendered_dependencies.refs: 229 model = context.models.get(dependency) 230 if model and model.materialized == Materialization.EPHEMERAL: 231 rendered._dependencies = rendered._dependencies.union( 232 model.render_config(context)._dependencies 233 ) 234 rendered._dependencies.refs.discard(dependency) 235 236 return rendered 237 238 @abstractmethod 239 def to_sqlmesh(self, context: DbtContext) -> Model: 240 """Convert DBT model into sqlmesh Model""" 241 242 def _context_for_dependencies( 243 self, context: DbtContext, dependencies: Dependencies 244 ) -> DbtContext: 245 model_context = context.copy() 246 247 model_context.sources = { 248 name: value for name, value in context.sources.items() if name in dependencies.sources 249 } 250 model_context.seeds = { 251 name: value for name, value in context.seeds.items() if name in dependencies.refs 252 } 253 model_context.models = { 254 name: value for name, value in context.models.items() if name in dependencies.refs 255 } 256 model_context.variables = { 257 name: value 258 for name, value in context.variables.items() 259 if name in dependencies.variables 260 } 261 262 return model_context
Arguments:
- owner: The owner of the model.
- stamp: An optional arbitrary string sequence used to create new model versions without making changes to any of the functional components of the definition.
- storage_format: The storage format used to store the physical table, only applicable in certain engines. (eg. 'parquet')
- path: The file path of the model
- target_schema: The schema for the profile target
- database: Database the model is stored in
- schema: Custom schema name added to the model schema name
- alias: Relation identifier for this model instead of the filename
- pre-hook: List of SQL statements to run before the model is built
- post-hook: List of SQL statements to run after the model is built
- full_refresh: Forces the model to always do a full refresh or never do a full refresh
- grants: Set or revoke permissions to the database object for this model
- columns: Column information for the model
194 def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]: 195 """Get common sqlmesh model parameters""" 196 jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros) 197 jinja_macros.global_objs.update( 198 { 199 "this": self.relation_info, 200 "schema": self.table_schema, 201 **model_context.jinja_globals, # type: ignore 202 } 203 ) 204 205 optional_kwargs: t.Dict[str, t.Any] = {} 206 for field in ("description", "owner", "stamp", "storage_format"): 207 field_val = getattr(self, field, None) or self.meta.get(field, None) 208 if field_val: 209 optional_kwargs[field] = field_val 210 211 return { 212 "columns": column_types_to_sqlmesh(self.columns) or None, 213 "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None, 214 "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs}, 215 "jinja_macros": jinja_macros, 216 "path": self.path, 217 "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)], 218 "post": [exp for hook in self.post_hook for exp in d.parse(hook)], 219 **optional_kwargs, 220 }
Get common sqlmesh model parameters
222 def render_config(self: BMC, context: DbtContext) -> BMC: 223 rendered = super().render_config(context) 224 rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql)) 225 rendered = ModelSqlRenderer(context, rendered).enriched_config 226 227 rendered_dependencies = rendered._dependencies 228 for dependency in rendered_dependencies.refs: 229 model = context.models.get(dependency) 230 if model and model.materialized == Materialization.EPHEMERAL: 231 rendered._dependencies = rendered._dependencies.union( 232 model.render_config(context)._dependencies 233 ) 234 rendered._dependencies.refs.discard(dependency) 235 236 return rendered
238 @abstractmethod 239 def to_sqlmesh(self, context: DbtContext) -> Model: 240 """Convert DBT model into sqlmesh Model"""
Convert DBT model into sqlmesh Model
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
265class ModelSqlRenderer(t.Generic[BMC]): 266 def __init__(self, context: DbtContext, config: BMC): 267 self.context = context 268 self.config = config 269 270 self._captured_dependencies: Dependencies = Dependencies() 271 self._rendered_sql: t.Optional[str] = None 272 self._enriched_config: BMC = config.copy() 273 274 self._jinja_globals = create_builtin_globals( 275 jinja_macros=context.jinja_macros, 276 jinja_globals={ 277 **context.jinja_globals, 278 **date_dict(c.EPOCH, c.EPOCH, c.EPOCH), 279 "config": self._config, 280 "ref": self._ref, 281 "var": self._var, 282 "source": self._source, 283 "this": self.config.relation_info, 284 "schema": self.config.table_schema, 285 }, 286 engine_adapter=None, 287 ) 288 289 # Set the adapter separately since it requires jinja globals to passed into it. 290 self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter( 291 self, 292 context.jinja_macros, 293 jinja_globals=self._jinja_globals, 294 dialect=context.engine_adapter.dialect if context.engine_adapter else "", 295 ) 296 297 @property 298 def enriched_config(self) -> BMC: 299 if self._rendered_sql is None: 300 self.render() 301 self._enriched_config._dependencies = self._enriched_config._dependencies.union( 302 self._captured_dependencies 303 ) 304 return self._enriched_config 305 306 def render(self) -> str: 307 if self._rendered_sql is None: 308 registry = self.context.jinja_macros 309 self._rendered_sql = ( 310 registry.build_environment(**self._jinja_globals) 311 .from_string(self.config.all_sql) 312 .render() 313 ) 314 return self._rendered_sql 315 316 def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation: 317 if package_name in self.context.models: 318 relation = BaseRelation.create(**self.context.models[package_name].relation_info) 319 elif package_name in self.context.seeds: 320 relation = BaseRelation.create(**self.context.seeds[package_name].relation_info) 321 else: 322 raise ConfigError( 323 f"Model '{package_name}' was not found for model '{self.config.table_name}'." 324 ) 325 self._captured_dependencies.refs.add(package_name) 326 return relation 327 328 def _var(self, name: str, default: t.Optional[str] = None) -> t.Any: 329 if default is None and name not in self.context.variables: 330 raise ConfigError( 331 f"Variable '{name}' was not found for model '{self.config.table_name}'." 332 ) 333 self._captured_dependencies.variables.add(name) 334 return self.context.variables.get(name, default) 335 336 def _source(self, source_name: str, table_name: str) -> BaseRelation: 337 full_name = ".".join([source_name, table_name]) 338 if full_name not in self.context.sources: 339 raise ConfigError( 340 f"Source '{full_name}' was not found for model '{self.config.table_name}'." 341 ) 342 self._captured_dependencies.sources.add(full_name) 343 return BaseRelation.create(**self.context.sources[full_name].relation_info) 344 345 def _config(self, *args: t.Any, **kwargs: t.Any) -> str: 346 if args and isinstance(args[0], dict): 347 self._enriched_config = self._enriched_config.update_with(args[0]) 348 if kwargs: 349 self._enriched_config = self._enriched_config.update_with(kwargs) 350 return "" 351 352 class TrackingAdapter(ParsetimeAdapter): 353 def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any): 354 super().__init__(*args, **kwargs) 355 self.outer_self = outer_self 356 self.context = outer_self.context 357 358 def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable: 359 macros = ( 360 self.context.jinja_macros.packages.get(package, {}) 361 if package is not None 362 else self.context.jinja_macros.root_macros 363 ) 364 for target_name in macros: 365 if target_name.endswith(f"__{name}"): 366 self.outer_self._captured_dependencies.macros.add( 367 MacroReference(package=package, name=target_name) 368 ) 369 return super().dispatch(name, package=package)
Abstract base class for generic types.
A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::
class Mapping(Generic[KT, VT]): def __getitem__(self, key: KT) -> VT: ... # Etc.
This class can then be used as follows::
def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default
266 def __init__(self, context: DbtContext, config: BMC): 267 self.context = context 268 self.config = config 269 270 self._captured_dependencies: Dependencies = Dependencies() 271 self._rendered_sql: t.Optional[str] = None 272 self._enriched_config: BMC = config.copy() 273 274 self._jinja_globals = create_builtin_globals( 275 jinja_macros=context.jinja_macros, 276 jinja_globals={ 277 **context.jinja_globals, 278 **date_dict(c.EPOCH, c.EPOCH, c.EPOCH), 279 "config": self._config, 280 "ref": self._ref, 281 "var": self._var, 282 "source": self._source, 283 "this": self.config.relation_info, 284 "schema": self.config.table_schema, 285 }, 286 engine_adapter=None, 287 ) 288 289 # Set the adapter separately since it requires jinja globals to passed into it. 290 self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter( 291 self, 292 context.jinja_macros, 293 jinja_globals=self._jinja_globals, 294 dialect=context.engine_adapter.dialect if context.engine_adapter else "", 295 )
352 class TrackingAdapter(ParsetimeAdapter): 353 def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any): 354 super().__init__(*args, **kwargs) 355 self.outer_self = outer_self 356 self.context = outer_self.context 357 358 def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable: 359 macros = ( 360 self.context.jinja_macros.packages.get(package, {}) 361 if package is not None 362 else self.context.jinja_macros.root_macros 363 ) 364 for target_name in macros: 365 if target_name.endswith(f"__{name}"): 366 self.outer_self._captured_dependencies.macros.add( 367 MacroReference(package=package, name=target_name) 368 ) 369 return super().dispatch(name, package=package)
Helper class that provides a standard way to create an ABC using inheritance.
358 def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable: 359 macros = ( 360 self.context.jinja_macros.packages.get(package, {}) 361 if package is not None 362 else self.context.jinja_macros.root_macros 363 ) 364 for target_name in macros: 365 if target_name.endswith(f"__{name}"): 366 self.outer_self._captured_dependencies.macros.add( 367 MacroReference(package=package, name=target_name) 368 ) 369 return super().dispatch(name, package=package)
Returns a dialect-specific version of a macro with the given name.