Edit on GitHub

sqlmesh.dbt.basemodel

  1from __future__ import annotations
  2
  3import typing as t
  4from abc import abstractmethod
  5from enum import Enum
  6from pathlib import Path
  7
  8from dbt.adapters.base import BaseRelation
  9from dbt.contracts.relation import RelationType
 10from pydantic import Field, validator
 11from sqlglot.helper import ensure_list
 12
 13from sqlmesh.core import constants as c
 14from sqlmesh.core import dialect as d
 15from sqlmesh.core.config.base import UpdateStrategy
 16from sqlmesh.core.model import Model
 17from sqlmesh.dbt.adapter import ParsetimeAdapter
 18from sqlmesh.dbt.builtin import create_builtin_globals
 19from sqlmesh.dbt.column import (
 20    ColumnConfig,
 21    column_descriptions_to_sqlmesh,
 22    column_types_to_sqlmesh,
 23    yaml_to_columns,
 24)
 25from sqlmesh.dbt.common import DbtContext, GeneralConfig, SqlStr
 26from sqlmesh.utils import AttributeDict
 27from sqlmesh.utils.conversions import ensure_bool
 28from sqlmesh.utils.date import date_dict
 29from sqlmesh.utils.errors import ConfigError
 30from sqlmesh.utils.jinja import MacroReference, extract_macro_references
 31from sqlmesh.utils.pydantic import PydanticModel
 32
 33BMC = t.TypeVar("BMC", bound="BaseModelConfig")
 34
 35
 36class Dependencies(PydanticModel):
 37    """
 38    DBT dependencies for a model, macro, etc.
 39
 40    Args:
 41        macros: The references to macros
 42        sources: The "source_name.table_name" for source tables used
 43        refs: The table_name for models used
 44        variables: The names of variables used, mapped to a flag that indicates whether their
 45            definition is optional or not.
 46    """
 47
 48    macros: t.Set[MacroReference] = set()
 49    sources: t.Set[str] = set()
 50    refs: t.Set[str] = set()
 51    variables: t.Set[str] = set()
 52
 53    def union(self, other: Dependencies) -> Dependencies:
 54        dependencies = Dependencies()
 55        dependencies.macros = self.macros | other.macros
 56        dependencies.sources = self.sources | other.sources
 57        dependencies.refs = self.refs | other.refs
 58        dependencies.variables = self.variables | other.variables
 59
 60        return dependencies
 61
 62
 63class Materialization(str, Enum):
 64    """DBT model materializations"""
 65
 66    TABLE = "table"
 67    VIEW = "view"
 68    INCREMENTAL = "incremental"
 69    EPHEMERAL = "ephemeral"
 70
 71
 72class BaseModelConfig(GeneralConfig):
 73    """
 74    Args:
 75        owner: The owner of the model.
 76        stamp: An optional arbitrary string sequence used to create new model versions without making
 77            changes to any of the functional components of the definition.
 78        storage_format: The storage format used to store the physical table, only applicable in certain engines.
 79            (eg. 'parquet')
 80        path: The file path of the model
 81        target_schema: The schema for the profile target
 82        database: Database the model is stored in
 83        schema: Custom schema name added to the model schema name
 84        alias: Relation identifier for this model instead of the filename
 85        pre-hook: List of SQL statements to run before the model is built
 86        post-hook: List of SQL statements to run after the model is built
 87        full_refresh: Forces the model to always do a full refresh or never do a full refresh
 88        grants: Set or revoke permissions to the database object for this model
 89        columns: Column information for the model
 90    """
 91
 92    # sqlmesh fields
 93    owner: t.Optional[str] = None
 94    stamp: t.Optional[str] = None
 95    storage_format: t.Optional[str] = None
 96    path: Path = Path()
 97    target_schema: str = ""
 98    _dependencies: Dependencies = Dependencies()
 99    _variables: t.Dict[str, bool] = {}
100
101    # DBT configuration fields
102    database: t.Optional[str] = None
103    schema_: t.Optional[str] = Field(None, alias="schema")
104    alias: t.Optional[str] = None
105    pre_hook: t.List[SqlStr] = Field([], alias="pre-hook")
106    post_hook: t.List[SqlStr] = Field([], alias="post-hook")
107    full_refresh: t.Optional[bool] = None
108    grants: t.Dict[str, t.List[str]] = {}
109    columns: t.Dict[str, ColumnConfig] = {}
110
111    @validator("pre_hook", "post_hook", pre=True)
112    def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]:
113        return [SqlStr(val) for val in ensure_list(v)]
114
115    @validator("full_refresh", pre=True)
116    def _validate_bool(cls, v: str) -> bool:
117        return ensure_bool(v)
118
119    @validator("grants", pre=True)
120    def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]:
121        return {key: ensure_list(value) for key, value in v.items()}
122
123    @validator("columns", pre=True)
124    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
125        if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()):
126            return v
127
128        return yaml_to_columns(v)
129
130    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
131        **GeneralConfig._FIELD_UPDATE_STRATEGY,
132        **{
133            "grants": UpdateStrategy.KEY_EXTEND,
134            "path": UpdateStrategy.IMMUTABLE,
135            "pre-hook": UpdateStrategy.EXTEND,
136            "post-hook": UpdateStrategy.EXTEND,
137            "columns": UpdateStrategy.KEY_EXTEND,
138        },
139    }
140
141    @property
142    def all_sql(self) -> SqlStr:
143        return SqlStr("")
144
145    @property
146    def table_schema(self) -> str:
147        """
148        Get the full schema name
149        """
150        return "_".join(part for part in (self.target_schema, self.schema_) if part)
151
152    @property
153    def table_name(self) -> str:
154        """
155        Get the table name
156        """
157        return self.alias or self.path.stem
158
159    @property
160    def model_name(self) -> str:
161        """
162        Get the sqlmesh model name
163
164        Returns:
165            The sqlmesh model name
166        """
167        return ".".join(
168            part for part in (self.database, self.table_schema, self.table_name) if part
169        )
170
171    @property
172    def model_materialization(self) -> Materialization:
173        return Materialization.TABLE
174
175    @property
176    def relation_info(self) -> AttributeDict[str, t.Any]:
177        if self.model_materialization == Materialization.VIEW:
178            relation_type = RelationType.View
179        elif self.model_materialization == Materialization.EPHEMERAL:
180            relation_type = RelationType.CTE
181        else:
182            relation_type = RelationType.Table
183
184        return AttributeDict(
185            {
186                "database": self.database,
187                "schema": self.table_schema,
188                "identifier": self.table_name,
189                "type": relation_type.value,
190            }
191        )
192
193    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
194        """Get common sqlmesh model parameters"""
195        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
196        jinja_macros.global_objs.update(
197            {
198                "this": self.relation_info,
199                "schema": self.table_schema,
200                **model_context.jinja_globals,  # type: ignore
201            }
202        )
203
204        optional_kwargs: t.Dict[str, t.Any] = {}
205        for field in ("description", "owner", "stamp", "storage_format"):
206            field_val = getattr(self, field, None) or self.meta.get(field, None)
207            if field_val:
208                optional_kwargs[field] = field_val
209
210        return {
211            "columns": column_types_to_sqlmesh(self.columns) or None,
212            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
213            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
214            "jinja_macros": jinja_macros,
215            "path": self.path,
216            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
217            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
218            **optional_kwargs,
219        }
220
221    def render_config(self: BMC, context: DbtContext) -> BMC:
222        rendered = super().render_config(context)
223        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
224        rendered = ModelSqlRenderer(context, rendered).enriched_config
225
226        rendered_dependencies = rendered._dependencies
227        for dependency in rendered_dependencies.refs:
228            model = context.models.get(dependency)
229            if model and model.materialized == Materialization.EPHEMERAL:
230                rendered._dependencies = rendered._dependencies.union(
231                    model.render_config(context)._dependencies
232                )
233                rendered._dependencies.refs.discard(dependency)
234
235        return rendered
236
237    @abstractmethod
238    def to_sqlmesh(self, context: DbtContext) -> Model:
239        """Convert DBT model into sqlmesh Model"""
240
241    def _context_for_dependencies(
242        self, context: DbtContext, dependencies: Dependencies
243    ) -> DbtContext:
244        model_context = context.copy()
245
246        model_context.sources = {
247            name: value for name, value in context.sources.items() if name in dependencies.sources
248        }
249        model_context.seeds = {
250            name: value for name, value in context.seeds.items() if name in dependencies.refs
251        }
252        model_context.models = {
253            name: value for name, value in context.models.items() if name in dependencies.refs
254        }
255        model_context.variables = {
256            name: value
257            for name, value in context.variables.items()
258            if name in dependencies.variables
259        }
260
261        return model_context
262
263
264class ModelSqlRenderer(t.Generic[BMC]):
265    def __init__(self, context: DbtContext, config: BMC):
266        self.context = context
267        self.config = config
268
269        self._captured_dependencies: Dependencies = Dependencies()
270        self._rendered_sql: t.Optional[str] = None
271        self._enriched_config: BMC = config.copy()
272
273        self._jinja_globals = create_builtin_globals(
274            jinja_macros=context.jinja_macros,
275            jinja_globals={
276                **context.jinja_globals,
277                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
278                "config": self._config,
279                "ref": self._ref,
280                "var": self._var,
281                "source": self._source,
282                "this": self.config.relation_info,
283                "schema": self.config.table_schema,
284            },
285            engine_adapter=None,
286        )
287
288        # Set the adapter separately since it requires jinja globals to passed into it.
289        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
290            self,
291            context.jinja_macros,
292            jinja_globals=self._jinja_globals,
293            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
294        )
295
296    @property
297    def enriched_config(self) -> BMC:
298        if self._rendered_sql is None:
299            self.render()
300            self._enriched_config._dependencies = self._enriched_config._dependencies.union(
301                self._captured_dependencies
302            )
303        return self._enriched_config
304
305    def render(self) -> str:
306        if self._rendered_sql is None:
307            registry = self.context.jinja_macros
308            self._rendered_sql = (
309                registry.build_environment(**self._jinja_globals)
310                .from_string(self.config.all_sql)
311                .render()
312            )
313        return self._rendered_sql
314
315    def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation:
316        if package_name in self.context.models:
317            relation = BaseRelation.create(**self.context.models[package_name].relation_info)
318        elif package_name in self.context.seeds:
319            relation = BaseRelation.create(**self.context.seeds[package_name].relation_info)
320        else:
321            raise ConfigError(
322                f"Model '{package_name}' was not found for model '{self.config.table_name}'."
323            )
324        self._captured_dependencies.refs.add(package_name)
325        return relation
326
327    def _var(self, name: str, default: t.Optional[str] = None) -> t.Any:
328        if default is None and name not in self.context.variables:
329            raise ConfigError(
330                f"Variable '{name}' was not found for model '{self.config.table_name}'."
331            )
332        self._captured_dependencies.variables.add(name)
333        return self.context.variables.get(name, default)
334
335    def _source(self, source_name: str, table_name: str) -> BaseRelation:
336        full_name = ".".join([source_name, table_name])
337        if full_name not in self.context.sources:
338            raise ConfigError(
339                f"Source '{full_name}' was not found for model '{self.config.table_name}'."
340            )
341        self._captured_dependencies.sources.add(full_name)
342        return BaseRelation.create(**self.context.sources[full_name].relation_info)
343
344    def _config(self, *args: t.Any, **kwargs: t.Any) -> str:
345        if args and isinstance(args[0], dict):
346            self._enriched_config = self._enriched_config.update_with(args[0])
347        if kwargs:
348            self._enriched_config = self._enriched_config.update_with(kwargs)
349        return ""
350
351    class TrackingAdapter(ParsetimeAdapter):
352        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
353            super().__init__(*args, **kwargs)
354            self.outer_self = outer_self
355            self.context = outer_self.context
356
357        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
358            macros = (
359                self.context.jinja_macros.packages.get(package, {})
360                if package is not None
361                else self.context.jinja_macros.root_macros
362            )
363            for target_name in macros:
364                if target_name.endswith(f"__{name}"):
365                    self.outer_self._captured_dependencies.macros.add(
366                        MacroReference(package=package, name=target_name)
367                    )
368            return super().dispatch(name, package=package)
class Dependencies(sqlmesh.utils.pydantic.PydanticModel):
37class Dependencies(PydanticModel):
38    """
39    DBT dependencies for a model, macro, etc.
40
41    Args:
42        macros: The references to macros
43        sources: The "source_name.table_name" for source tables used
44        refs: The table_name for models used
45        variables: The names of variables used, mapped to a flag that indicates whether their
46            definition is optional or not.
47    """
48
49    macros: t.Set[MacroReference] = set()
50    sources: t.Set[str] = set()
51    refs: t.Set[str] = set()
52    variables: t.Set[str] = set()
53
54    def union(self, other: Dependencies) -> Dependencies:
55        dependencies = Dependencies()
56        dependencies.macros = self.macros | other.macros
57        dependencies.sources = self.sources | other.sources
58        dependencies.refs = self.refs | other.refs
59        dependencies.variables = self.variables | other.variables
60
61        return dependencies

DBT dependencies for a model, macro, etc.

Arguments:
  • macros: The references to macros
  • sources: The "source_name.table_name" for source tables used
  • refs: The table_name for models used
  • variables: The names of variables used, mapped to a flag that indicates whether their definition is optional or not.
54    def union(self, other: Dependencies) -> Dependencies:
55        dependencies = Dependencies()
56        dependencies.macros = self.macros | other.macros
57        dependencies.sources = self.sources | other.sources
58        dependencies.refs = self.refs | other.refs
59        dependencies.variables = self.variables | other.variables
60
61        return dependencies
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class Materialization(builtins.str, enum.Enum):
64class Materialization(str, Enum):
65    """DBT model materializations"""
66
67    TABLE = "table"
68    VIEW = "view"
69    INCREMENTAL = "incremental"
70    EPHEMERAL = "ephemeral"

DBT model materializations

TABLE = <Materialization.TABLE: 'table'>
VIEW = <Materialization.VIEW: 'view'>
INCREMENTAL = <Materialization.INCREMENTAL: 'incremental'>
EPHEMERAL = <Materialization.EPHEMERAL: 'ephemeral'>
Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
removeprefix
removesuffix
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
class BaseModelConfig(sqlmesh.dbt.common.GeneralConfig):
 73class BaseModelConfig(GeneralConfig):
 74    """
 75    Args:
 76        owner: The owner of the model.
 77        stamp: An optional arbitrary string sequence used to create new model versions without making
 78            changes to any of the functional components of the definition.
 79        storage_format: The storage format used to store the physical table, only applicable in certain engines.
 80            (eg. 'parquet')
 81        path: The file path of the model
 82        target_schema: The schema for the profile target
 83        database: Database the model is stored in
 84        schema: Custom schema name added to the model schema name
 85        alias: Relation identifier for this model instead of the filename
 86        pre-hook: List of SQL statements to run before the model is built
 87        post-hook: List of SQL statements to run after the model is built
 88        full_refresh: Forces the model to always do a full refresh or never do a full refresh
 89        grants: Set or revoke permissions to the database object for this model
 90        columns: Column information for the model
 91    """
 92
 93    # sqlmesh fields
 94    owner: t.Optional[str] = None
 95    stamp: t.Optional[str] = None
 96    storage_format: t.Optional[str] = None
 97    path: Path = Path()
 98    target_schema: str = ""
 99    _dependencies: Dependencies = Dependencies()
100    _variables: t.Dict[str, bool] = {}
101
102    # DBT configuration fields
103    database: t.Optional[str] = None
104    schema_: t.Optional[str] = Field(None, alias="schema")
105    alias: t.Optional[str] = None
106    pre_hook: t.List[SqlStr] = Field([], alias="pre-hook")
107    post_hook: t.List[SqlStr] = Field([], alias="post-hook")
108    full_refresh: t.Optional[bool] = None
109    grants: t.Dict[str, t.List[str]] = {}
110    columns: t.Dict[str, ColumnConfig] = {}
111
112    @validator("pre_hook", "post_hook", pre=True)
113    def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]:
114        return [SqlStr(val) for val in ensure_list(v)]
115
116    @validator("full_refresh", pre=True)
117    def _validate_bool(cls, v: str) -> bool:
118        return ensure_bool(v)
119
120    @validator("grants", pre=True)
121    def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]:
122        return {key: ensure_list(value) for key, value in v.items()}
123
124    @validator("columns", pre=True)
125    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
126        if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()):
127            return v
128
129        return yaml_to_columns(v)
130
131    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
132        **GeneralConfig._FIELD_UPDATE_STRATEGY,
133        **{
134            "grants": UpdateStrategy.KEY_EXTEND,
135            "path": UpdateStrategy.IMMUTABLE,
136            "pre-hook": UpdateStrategy.EXTEND,
137            "post-hook": UpdateStrategy.EXTEND,
138            "columns": UpdateStrategy.KEY_EXTEND,
139        },
140    }
141
142    @property
143    def all_sql(self) -> SqlStr:
144        return SqlStr("")
145
146    @property
147    def table_schema(self) -> str:
148        """
149        Get the full schema name
150        """
151        return "_".join(part for part in (self.target_schema, self.schema_) if part)
152
153    @property
154    def table_name(self) -> str:
155        """
156        Get the table name
157        """
158        return self.alias or self.path.stem
159
160    @property
161    def model_name(self) -> str:
162        """
163        Get the sqlmesh model name
164
165        Returns:
166            The sqlmesh model name
167        """
168        return ".".join(
169            part for part in (self.database, self.table_schema, self.table_name) if part
170        )
171
172    @property
173    def model_materialization(self) -> Materialization:
174        return Materialization.TABLE
175
176    @property
177    def relation_info(self) -> AttributeDict[str, t.Any]:
178        if self.model_materialization == Materialization.VIEW:
179            relation_type = RelationType.View
180        elif self.model_materialization == Materialization.EPHEMERAL:
181            relation_type = RelationType.CTE
182        else:
183            relation_type = RelationType.Table
184
185        return AttributeDict(
186            {
187                "database": self.database,
188                "schema": self.table_schema,
189                "identifier": self.table_name,
190                "type": relation_type.value,
191            }
192        )
193
194    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
195        """Get common sqlmesh model parameters"""
196        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
197        jinja_macros.global_objs.update(
198            {
199                "this": self.relation_info,
200                "schema": self.table_schema,
201                **model_context.jinja_globals,  # type: ignore
202            }
203        )
204
205        optional_kwargs: t.Dict[str, t.Any] = {}
206        for field in ("description", "owner", "stamp", "storage_format"):
207            field_val = getattr(self, field, None) or self.meta.get(field, None)
208            if field_val:
209                optional_kwargs[field] = field_val
210
211        return {
212            "columns": column_types_to_sqlmesh(self.columns) or None,
213            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
214            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
215            "jinja_macros": jinja_macros,
216            "path": self.path,
217            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
218            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
219            **optional_kwargs,
220        }
221
222    def render_config(self: BMC, context: DbtContext) -> BMC:
223        rendered = super().render_config(context)
224        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
225        rendered = ModelSqlRenderer(context, rendered).enriched_config
226
227        rendered_dependencies = rendered._dependencies
228        for dependency in rendered_dependencies.refs:
229            model = context.models.get(dependency)
230            if model and model.materialized == Materialization.EPHEMERAL:
231                rendered._dependencies = rendered._dependencies.union(
232                    model.render_config(context)._dependencies
233                )
234                rendered._dependencies.refs.discard(dependency)
235
236        return rendered
237
238    @abstractmethod
239    def to_sqlmesh(self, context: DbtContext) -> Model:
240        """Convert DBT model into sqlmesh Model"""
241
242    def _context_for_dependencies(
243        self, context: DbtContext, dependencies: Dependencies
244    ) -> DbtContext:
245        model_context = context.copy()
246
247        model_context.sources = {
248            name: value for name, value in context.sources.items() if name in dependencies.sources
249        }
250        model_context.seeds = {
251            name: value for name, value in context.seeds.items() if name in dependencies.refs
252        }
253        model_context.models = {
254            name: value for name, value in context.models.items() if name in dependencies.refs
255        }
256        model_context.variables = {
257            name: value
258            for name, value in context.variables.items()
259            if name in dependencies.variables
260        }
261
262        return model_context
Arguments:
  • owner: The owner of the model.
  • stamp: An optional arbitrary string sequence used to create new model versions without making changes to any of the functional components of the definition.
  • storage_format: The storage format used to store the physical table, only applicable in certain engines. (eg. 'parquet')
  • path: The file path of the model
  • target_schema: The schema for the profile target
  • database: Database the model is stored in
  • schema: Custom schema name added to the model schema name
  • alias: Relation identifier for this model instead of the filename
  • pre-hook: List of SQL statements to run before the model is built
  • post-hook: List of SQL statements to run after the model is built
  • full_refresh: Forces the model to always do a full refresh or never do a full refresh
  • grants: Set or revoke permissions to the database object for this model
  • columns: Column information for the model
table_schema: str

Get the full schema name

table_name: str

Get the table name

model_name: str

Get the sqlmesh model name

Returns:

The sqlmesh model name

def sqlmesh_model_kwargs(self, model_context: sqlmesh.dbt.common.DbtContext) -> Dict[str, Any]:
194    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
195        """Get common sqlmesh model parameters"""
196        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
197        jinja_macros.global_objs.update(
198            {
199                "this": self.relation_info,
200                "schema": self.table_schema,
201                **model_context.jinja_globals,  # type: ignore
202            }
203        )
204
205        optional_kwargs: t.Dict[str, t.Any] = {}
206        for field in ("description", "owner", "stamp", "storage_format"):
207            field_val = getattr(self, field, None) or self.meta.get(field, None)
208            if field_val:
209                optional_kwargs[field] = field_val
210
211        return {
212            "columns": column_types_to_sqlmesh(self.columns) or None,
213            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
214            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
215            "jinja_macros": jinja_macros,
216            "path": self.path,
217            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
218            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
219            **optional_kwargs,
220        }

Get common sqlmesh model parameters

def render_config(self: ~BMC, context: sqlmesh.dbt.common.DbtContext) -> ~BMC:
222    def render_config(self: BMC, context: DbtContext) -> BMC:
223        rendered = super().render_config(context)
224        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
225        rendered = ModelSqlRenderer(context, rendered).enriched_config
226
227        rendered_dependencies = rendered._dependencies
228        for dependency in rendered_dependencies.refs:
229            model = context.models.get(dependency)
230            if model and model.materialized == Materialization.EPHEMERAL:
231                rendered._dependencies = rendered._dependencies.union(
232                    model.render_config(context)._dependencies
233                )
234                rendered._dependencies.refs.discard(dependency)
235
236        return rendered
@abstractmethod
def to_sqlmesh( self, context: sqlmesh.dbt.common.DbtContext) -> Annotated[Union[sqlmesh.core.model.definition.SqlModel, sqlmesh.core.model.definition.SeedModel, sqlmesh.core.model.definition.PythonModel], FieldInfo(default=PydanticUndefined, discriminator='source_type', extra={})]:
238    @abstractmethod
239    def to_sqlmesh(self, context: DbtContext) -> Model:
240        """Convert DBT model into sqlmesh Model"""

Convert DBT model into sqlmesh Model

Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.dbt.common.GeneralConfig
replace
sqlmesh.dbt.common.DbtConfig
Config
sqlmesh.core.config.base.BaseConfig
update_with
sqlmesh.utils.pydantic.PydanticModel
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class ModelSqlRenderer(typing.Generic[~BMC]):
265class ModelSqlRenderer(t.Generic[BMC]):
266    def __init__(self, context: DbtContext, config: BMC):
267        self.context = context
268        self.config = config
269
270        self._captured_dependencies: Dependencies = Dependencies()
271        self._rendered_sql: t.Optional[str] = None
272        self._enriched_config: BMC = config.copy()
273
274        self._jinja_globals = create_builtin_globals(
275            jinja_macros=context.jinja_macros,
276            jinja_globals={
277                **context.jinja_globals,
278                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
279                "config": self._config,
280                "ref": self._ref,
281                "var": self._var,
282                "source": self._source,
283                "this": self.config.relation_info,
284                "schema": self.config.table_schema,
285            },
286            engine_adapter=None,
287        )
288
289        # Set the adapter separately since it requires jinja globals to passed into it.
290        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
291            self,
292            context.jinja_macros,
293            jinja_globals=self._jinja_globals,
294            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
295        )
296
297    @property
298    def enriched_config(self) -> BMC:
299        if self._rendered_sql is None:
300            self.render()
301            self._enriched_config._dependencies = self._enriched_config._dependencies.union(
302                self._captured_dependencies
303            )
304        return self._enriched_config
305
306    def render(self) -> str:
307        if self._rendered_sql is None:
308            registry = self.context.jinja_macros
309            self._rendered_sql = (
310                registry.build_environment(**self._jinja_globals)
311                .from_string(self.config.all_sql)
312                .render()
313            )
314        return self._rendered_sql
315
316    def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation:
317        if package_name in self.context.models:
318            relation = BaseRelation.create(**self.context.models[package_name].relation_info)
319        elif package_name in self.context.seeds:
320            relation = BaseRelation.create(**self.context.seeds[package_name].relation_info)
321        else:
322            raise ConfigError(
323                f"Model '{package_name}' was not found for model '{self.config.table_name}'."
324            )
325        self._captured_dependencies.refs.add(package_name)
326        return relation
327
328    def _var(self, name: str, default: t.Optional[str] = None) -> t.Any:
329        if default is None and name not in self.context.variables:
330            raise ConfigError(
331                f"Variable '{name}' was not found for model '{self.config.table_name}'."
332            )
333        self._captured_dependencies.variables.add(name)
334        return self.context.variables.get(name, default)
335
336    def _source(self, source_name: str, table_name: str) -> BaseRelation:
337        full_name = ".".join([source_name, table_name])
338        if full_name not in self.context.sources:
339            raise ConfigError(
340                f"Source '{full_name}' was not found for model '{self.config.table_name}'."
341            )
342        self._captured_dependencies.sources.add(full_name)
343        return BaseRelation.create(**self.context.sources[full_name].relation_info)
344
345    def _config(self, *args: t.Any, **kwargs: t.Any) -> str:
346        if args and isinstance(args[0], dict):
347            self._enriched_config = self._enriched_config.update_with(args[0])
348        if kwargs:
349            self._enriched_config = self._enriched_config.update_with(kwargs)
350        return ""
351
352    class TrackingAdapter(ParsetimeAdapter):
353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
354            super().__init__(*args, **kwargs)
355            self.outer_self = outer_self
356            self.context = outer_self.context
357
358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
359            macros = (
360                self.context.jinja_macros.packages.get(package, {})
361                if package is not None
362                else self.context.jinja_macros.root_macros
363            )
364            for target_name in macros:
365                if target_name.endswith(f"__{name}"):
366                    self.outer_self._captured_dependencies.macros.add(
367                        MacroReference(package=package, name=target_name)
368                    )
369            return super().dispatch(name, package=package)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def __getitem__(self, key: KT) -> VT: ... # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

ModelSqlRenderer(context: sqlmesh.dbt.common.DbtContext, config: ~BMC)
266    def __init__(self, context: DbtContext, config: BMC):
267        self.context = context
268        self.config = config
269
270        self._captured_dependencies: Dependencies = Dependencies()
271        self._rendered_sql: t.Optional[str] = None
272        self._enriched_config: BMC = config.copy()
273
274        self._jinja_globals = create_builtin_globals(
275            jinja_macros=context.jinja_macros,
276            jinja_globals={
277                **context.jinja_globals,
278                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
279                "config": self._config,
280                "ref": self._ref,
281                "var": self._var,
282                "source": self._source,
283                "this": self.config.relation_info,
284                "schema": self.config.table_schema,
285            },
286            engine_adapter=None,
287        )
288
289        # Set the adapter separately since it requires jinja globals to passed into it.
290        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
291            self,
292            context.jinja_macros,
293            jinja_globals=self._jinja_globals,
294            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
295        )
def render(self) -> str:
306    def render(self) -> str:
307        if self._rendered_sql is None:
308            registry = self.context.jinja_macros
309            self._rendered_sql = (
310                registry.build_environment(**self._jinja_globals)
311                .from_string(self.config.all_sql)
312                .render()
313            )
314        return self._rendered_sql
class ModelSqlRenderer.TrackingAdapter(sqlmesh.dbt.adapter.ParsetimeAdapter):
352    class TrackingAdapter(ParsetimeAdapter):
353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
354            super().__init__(*args, **kwargs)
355            self.outer_self = outer_self
356            self.context = outer_self.context
357
358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
359            macros = (
360                self.context.jinja_macros.packages.get(package, {})
361                if package is not None
362                else self.context.jinja_macros.root_macros
363            )
364            for target_name in macros:
365                if target_name.endswith(f"__{name}"):
366                    self.outer_self._captured_dependencies.macros.add(
367                        MacroReference(package=package, name=target_name)
368                    )
369            return super().dispatch(name, package=package)

Helper class that provides a standard way to create an ABC using inheritance.

ModelSqlRenderer.TrackingAdapter( outer_self: sqlmesh.dbt.basemodel.ModelSqlRenderer, *args: Any, **kwargs: Any)
353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
354            super().__init__(*args, **kwargs)
355            self.outer_self = outer_self
356            self.context = outer_self.context
def dispatch(self, name: str, package: Optional[str] = None) -> Callable:
358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
359            macros = (
360                self.context.jinja_macros.packages.get(package, {})
361                if package is not None
362                else self.context.jinja_macros.root_macros
363            )
364            for target_name in macros:
365                if target_name.endswith(f"__{name}"):
366                    self.outer_self._captured_dependencies.macros.add(
367                        MacroReference(package=package, name=target_name)
368                    )
369            return super().dispatch(name, package=package)

Returns a dialect-specific version of a macro with the given name.