Edit on GitHub

sqlmesh.dbt.source

 1from __future__ import annotations
 2
 3import typing as t
 4
 5from dbt.contracts.relation import RelationType
 6from pydantic import Field, validator
 7
 8from sqlmesh.core.config.base import UpdateStrategy
 9from sqlmesh.dbt.column import ColumnConfig, yaml_to_columns
10from sqlmesh.dbt.common import GeneralConfig
11from sqlmesh.utils.conversions import ensure_bool
12
13
14class SourceConfig(GeneralConfig):
15    """
16    Args:
17        config_name: The schema.table_name names declared in source config
18        name: The name of the source or table
19        database: Name of the database where the table is stored. By default, the project's target database is used.
20        schema: The scehma name as stored in the database. If not specified, the source name is used.
21        identifier: The table name as stored in the database. If not specified, the source table name is used
22        loader: Describes the tool that loads the source into the warehouse
23        overrides: Override a source defined in the specified package
24        freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
25        loaded_at_field: Column name or expression that returns a timestamp indicating freshness
26        quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
27        external: Dictionary of metadata properties specific to sources that point to external tables
28        columns: Columns within the source
29    """
30
31    # sqlmesh fields
32    config_name: str = ""
33
34    # DBT configuration fields
35    name: t.Optional[str] = None
36    database: t.Optional[str] = None
37    schema_: t.Optional[str] = Field(None, alias="schema")
38    identifier: t.Optional[str] = None
39    loader: t.Optional[str] = None
40    overrides: t.Optional[str] = None
41    freshness: t.Optional[t.Dict[str, t.Any]] = {}
42    loaded_at_field: t.Optional[str] = None
43    quoting: t.Optional[t.Dict[str, bool]] = {}
44    external: t.Optional[t.Dict[str, t.Any]] = {}
45    columns: t.Dict[str, ColumnConfig] = {}
46
47    @validator("quoting", pre=True)
48    def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]:
49        return {key: ensure_bool(val) for key, val in v.items()}
50
51    @validator("columns", pre=True)
52    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
53        return yaml_to_columns(v)
54
55    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
56        **GeneralConfig._FIELD_UPDATE_STRATEGY,
57        **{"columns": UpdateStrategy.KEY_EXTEND},
58    }
59
60    @property
61    def table_name(self) -> t.Optional[str]:
62        return self.identifier or self.name
63
64    @property
65    def source_name(self) -> str:
66        return ".".join(part for part in (self.schema_, self.table_name) if part)
67
68    @property
69    def relation_info(self) -> t.Dict[str, t.Any]:
70        return {
71            "database": self.database,
72            "schema": self.schema_,
73            "identifier": self.table_name,
74            "type": RelationType.External.value,
75        }
class SourceConfig(sqlmesh.dbt.common.GeneralConfig):
15class SourceConfig(GeneralConfig):
16    """
17    Args:
18        config_name: The schema.table_name names declared in source config
19        name: The name of the source or table
20        database: Name of the database where the table is stored. By default, the project's target database is used.
21        schema: The scehma name as stored in the database. If not specified, the source name is used.
22        identifier: The table name as stored in the database. If not specified, the source table name is used
23        loader: Describes the tool that loads the source into the warehouse
24        overrides: Override a source defined in the specified package
25        freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
26        loaded_at_field: Column name or expression that returns a timestamp indicating freshness
27        quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
28        external: Dictionary of metadata properties specific to sources that point to external tables
29        columns: Columns within the source
30    """
31
32    # sqlmesh fields
33    config_name: str = ""
34
35    # DBT configuration fields
36    name: t.Optional[str] = None
37    database: t.Optional[str] = None
38    schema_: t.Optional[str] = Field(None, alias="schema")
39    identifier: t.Optional[str] = None
40    loader: t.Optional[str] = None
41    overrides: t.Optional[str] = None
42    freshness: t.Optional[t.Dict[str, t.Any]] = {}
43    loaded_at_field: t.Optional[str] = None
44    quoting: t.Optional[t.Dict[str, bool]] = {}
45    external: t.Optional[t.Dict[str, t.Any]] = {}
46    columns: t.Dict[str, ColumnConfig] = {}
47
48    @validator("quoting", pre=True)
49    def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]:
50        return {key: ensure_bool(val) for key, val in v.items()}
51
52    @validator("columns", pre=True)
53    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
54        return yaml_to_columns(v)
55
56    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
57        **GeneralConfig._FIELD_UPDATE_STRATEGY,
58        **{"columns": UpdateStrategy.KEY_EXTEND},
59    }
60
61    @property
62    def table_name(self) -> t.Optional[str]:
63        return self.identifier or self.name
64
65    @property
66    def source_name(self) -> str:
67        return ".".join(part for part in (self.schema_, self.table_name) if part)
68
69    @property
70    def relation_info(self) -> t.Dict[str, t.Any]:
71        return {
72            "database": self.database,
73            "schema": self.schema_,
74            "identifier": self.table_name,
75            "type": RelationType.External.value,
76        }
Arguments:
  • config_name: The schema.table_name names declared in source config
  • name: The name of the source or table
  • database: Name of the database where the table is stored. By default, the project's target database is used.
  • schema: The scehma name as stored in the database. If not specified, the source name is used.
  • identifier: The table name as stored in the database. If not specified, the source table name is used
  • loader: Describes the tool that loads the source into the warehouse
  • overrides: Override a source defined in the specified package
  • freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
  • loaded_at_field: Column name or expression that returns a timestamp indicating freshness
  • quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
  • external: Dictionary of metadata properties specific to sources that point to external tables
  • columns: Columns within the source
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.dbt.common.GeneralConfig
replace
render_config
sqlmesh.dbt.common.DbtConfig
Config
sqlmesh.core.config.base.BaseConfig
update_with
sqlmesh.utils.pydantic.PydanticModel
dict
json
missing_required_fields
extra_fields
all_fields
required_fields