Edit on GitHub

sqlmesh.dbt.project

  1from __future__ import annotations
  2
  3import typing as t
  4from pathlib import Path
  5
  6from sqlmesh.dbt.common import PROJECT_FILENAME, DbtContext, load_yaml
  7from sqlmesh.dbt.package import Package, PackageLoader, ProjectConfig
  8from sqlmesh.dbt.profile import Profile
  9from sqlmesh.utils.errors import ConfigError
 10
 11
 12class Project:
 13    """Configuration for a DBT project"""
 14
 15    def __init__(
 16        self,
 17        context: DbtContext,
 18        profile: Profile,
 19        packages: t.Dict[str, Package],
 20    ):
 21        """
 22        Args:
 23            context: DBT context for the project
 24            profile: The profile associated with the project
 25            packages: The packages in this project. The project should be included
 26                      with the project name as the key
 27        """
 28        self.context = context
 29        self.profile = profile
 30        self.packages = packages
 31
 32    @classmethod
 33    def load(cls, context: DbtContext) -> Project:
 34        """
 35        Loads the configuration for the specified DBT project
 36
 37        Args:
 38            context: DBT context for this project
 39
 40        Returns:
 41            Project instance for the specified DBT project
 42        """
 43        context = context.copy()
 44
 45        project_file_path = Path(context.project_root, PROJECT_FILENAME)
 46        if not project_file_path.exists():
 47            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
 48        project_yaml = load_yaml(project_file_path)
 49
 50        variables = project_yaml.get("vars", {})
 51        context.variables = {
 52            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
 53        }
 54
 55        context.project_name = context.render(project_yaml.get("name", ""))
 56        if not context.project_name:
 57            raise ConfigError(f"{project_file_path.stem} must include project name.")
 58
 59        context.profile_name = (
 60            context.render(project_yaml.get("profile", "")) or context.project_name
 61        )
 62
 63        profile = Profile.load(context)
 64        context.target = (
 65            profile.targets[context.target_name]
 66            if context.target_name
 67            else profile.targets[profile.default_target]
 68        )
 69
 70        packages = {}
 71        root_loader = PackageLoader(context, ProjectConfig())
 72
 73        packages[context.project_name] = root_loader.load()
 74        project_config = root_loader.project_config
 75
 76        packages_dir = Path(
 77            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
 78        )
 79        if not packages_dir.is_absolute():
 80            packages_dir = Path(context.project_root, packages_dir)
 81
 82        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
 83            name = context.render(load_yaml(path).get("name", ""))
 84            if not name:
 85                raise ConfigError(f"{path} must include package name")
 86
 87            package_context = context.copy()
 88            package_context.project_root = path.parent
 89            package_context.variables = {}
 90            packages[name] = PackageLoader(
 91                package_context, cls._overrides_for_package(name, project_config)
 92            ).load()
 93
 94        for name, package in packages.items():
 95            package_vars = variables.get(name)
 96
 97            if isinstance(package_vars, dict):
 98                package.variables.update(package_vars)
 99
100        return Project(context, profile, packages)
101
102    @classmethod
103    def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig:
104        overrides = ProjectConfig()
105
106        source_overrides = {
107            scope[1:]: value
108            for scope, value in config.source_config.items()
109            if scope and scope[0] == name
110        }
111        if source_overrides:
112            overrides.source_config = source_overrides
113
114        seed_overrides = {
115            scope[1:]: value
116            for scope, value in config.seed_config.items()
117            if scope and scope[0] == name
118        }
119        if seed_overrides:
120            overrides.seed_config = seed_overrides
121
122        model_overrides = {
123            scope[1:]: value
124            for scope, value in config.model_config.items()
125            if scope and scope[0] == name
126        }
127        if model_overrides:
128            overrides.model_config = model_overrides
129
130        return overrides
131
132    @property
133    def project_files(self) -> t.Set[Path]:
134        paths = {self.profile.path}
135        for package in self.packages.values():
136            paths.update(package.files)
137
138        return paths
class Project:
 13class Project:
 14    """Configuration for a DBT project"""
 15
 16    def __init__(
 17        self,
 18        context: DbtContext,
 19        profile: Profile,
 20        packages: t.Dict[str, Package],
 21    ):
 22        """
 23        Args:
 24            context: DBT context for the project
 25            profile: The profile associated with the project
 26            packages: The packages in this project. The project should be included
 27                      with the project name as the key
 28        """
 29        self.context = context
 30        self.profile = profile
 31        self.packages = packages
 32
 33    @classmethod
 34    def load(cls, context: DbtContext) -> Project:
 35        """
 36        Loads the configuration for the specified DBT project
 37
 38        Args:
 39            context: DBT context for this project
 40
 41        Returns:
 42            Project instance for the specified DBT project
 43        """
 44        context = context.copy()
 45
 46        project_file_path = Path(context.project_root, PROJECT_FILENAME)
 47        if not project_file_path.exists():
 48            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
 49        project_yaml = load_yaml(project_file_path)
 50
 51        variables = project_yaml.get("vars", {})
 52        context.variables = {
 53            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
 54        }
 55
 56        context.project_name = context.render(project_yaml.get("name", ""))
 57        if not context.project_name:
 58            raise ConfigError(f"{project_file_path.stem} must include project name.")
 59
 60        context.profile_name = (
 61            context.render(project_yaml.get("profile", "")) or context.project_name
 62        )
 63
 64        profile = Profile.load(context)
 65        context.target = (
 66            profile.targets[context.target_name]
 67            if context.target_name
 68            else profile.targets[profile.default_target]
 69        )
 70
 71        packages = {}
 72        root_loader = PackageLoader(context, ProjectConfig())
 73
 74        packages[context.project_name] = root_loader.load()
 75        project_config = root_loader.project_config
 76
 77        packages_dir = Path(
 78            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
 79        )
 80        if not packages_dir.is_absolute():
 81            packages_dir = Path(context.project_root, packages_dir)
 82
 83        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
 84            name = context.render(load_yaml(path).get("name", ""))
 85            if not name:
 86                raise ConfigError(f"{path} must include package name")
 87
 88            package_context = context.copy()
 89            package_context.project_root = path.parent
 90            package_context.variables = {}
 91            packages[name] = PackageLoader(
 92                package_context, cls._overrides_for_package(name, project_config)
 93            ).load()
 94
 95        for name, package in packages.items():
 96            package_vars = variables.get(name)
 97
 98            if isinstance(package_vars, dict):
 99                package.variables.update(package_vars)
100
101        return Project(context, profile, packages)
102
103    @classmethod
104    def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig:
105        overrides = ProjectConfig()
106
107        source_overrides = {
108            scope[1:]: value
109            for scope, value in config.source_config.items()
110            if scope and scope[0] == name
111        }
112        if source_overrides:
113            overrides.source_config = source_overrides
114
115        seed_overrides = {
116            scope[1:]: value
117            for scope, value in config.seed_config.items()
118            if scope and scope[0] == name
119        }
120        if seed_overrides:
121            overrides.seed_config = seed_overrides
122
123        model_overrides = {
124            scope[1:]: value
125            for scope, value in config.model_config.items()
126            if scope and scope[0] == name
127        }
128        if model_overrides:
129            overrides.model_config = model_overrides
130
131        return overrides
132
133    @property
134    def project_files(self) -> t.Set[Path]:
135        paths = {self.profile.path}
136        for package in self.packages.values():
137            paths.update(package.files)
138
139        return paths

Configuration for a DBT project

Project( context: sqlmesh.dbt.common.DbtContext, profile: sqlmesh.dbt.profile.Profile, packages: Dict[str, sqlmesh.dbt.package.Package])
16    def __init__(
17        self,
18        context: DbtContext,
19        profile: Profile,
20        packages: t.Dict[str, Package],
21    ):
22        """
23        Args:
24            context: DBT context for the project
25            profile: The profile associated with the project
26            packages: The packages in this project. The project should be included
27                      with the project name as the key
28        """
29        self.context = context
30        self.profile = profile
31        self.packages = packages
Arguments:
  • context: DBT context for the project
  • profile: The profile associated with the project
  • packages: The packages in this project. The project should be included with the project name as the key
@classmethod
def load( cls, context: sqlmesh.dbt.common.DbtContext) -> sqlmesh.dbt.project.Project:
 33    @classmethod
 34    def load(cls, context: DbtContext) -> Project:
 35        """
 36        Loads the configuration for the specified DBT project
 37
 38        Args:
 39            context: DBT context for this project
 40
 41        Returns:
 42            Project instance for the specified DBT project
 43        """
 44        context = context.copy()
 45
 46        project_file_path = Path(context.project_root, PROJECT_FILENAME)
 47        if not project_file_path.exists():
 48            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
 49        project_yaml = load_yaml(project_file_path)
 50
 51        variables = project_yaml.get("vars", {})
 52        context.variables = {
 53            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
 54        }
 55
 56        context.project_name = context.render(project_yaml.get("name", ""))
 57        if not context.project_name:
 58            raise ConfigError(f"{project_file_path.stem} must include project name.")
 59
 60        context.profile_name = (
 61            context.render(project_yaml.get("profile", "")) or context.project_name
 62        )
 63
 64        profile = Profile.load(context)
 65        context.target = (
 66            profile.targets[context.target_name]
 67            if context.target_name
 68            else profile.targets[profile.default_target]
 69        )
 70
 71        packages = {}
 72        root_loader = PackageLoader(context, ProjectConfig())
 73
 74        packages[context.project_name] = root_loader.load()
 75        project_config = root_loader.project_config
 76
 77        packages_dir = Path(
 78            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
 79        )
 80        if not packages_dir.is_absolute():
 81            packages_dir = Path(context.project_root, packages_dir)
 82
 83        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
 84            name = context.render(load_yaml(path).get("name", ""))
 85            if not name:
 86                raise ConfigError(f"{path} must include package name")
 87
 88            package_context = context.copy()
 89            package_context.project_root = path.parent
 90            package_context.variables = {}
 91            packages[name] = PackageLoader(
 92                package_context, cls._overrides_for_package(name, project_config)
 93            ).load()
 94
 95        for name, package in packages.items():
 96            package_vars = variables.get(name)
 97
 98            if isinstance(package_vars, dict):
 99                package.variables.update(package_vars)
100
101        return Project(context, profile, packages)

Loads the configuration for the specified DBT project

Arguments:
  • context: DBT context for this project
Returns:

Project instance for the specified DBT project