sqlmesh.dbt.project
1from __future__ import annotations 2 3import typing as t 4from pathlib import Path 5 6from sqlmesh.dbt.common import PROJECT_FILENAME, DbtContext, load_yaml 7from sqlmesh.dbt.package import Package, PackageLoader, ProjectConfig 8from sqlmesh.dbt.profile import Profile 9from sqlmesh.utils.errors import ConfigError 10 11 12class Project: 13 """Configuration for a DBT project""" 14 15 def __init__( 16 self, 17 context: DbtContext, 18 profile: Profile, 19 packages: t.Dict[str, Package], 20 ): 21 """ 22 Args: 23 context: DBT context for the project 24 profile: The profile associated with the project 25 packages: The packages in this project. The project should be included 26 with the project name as the key 27 """ 28 self.context = context 29 self.profile = profile 30 self.packages = packages 31 32 @classmethod 33 def load(cls, context: DbtContext) -> Project: 34 """ 35 Loads the configuration for the specified DBT project 36 37 Args: 38 context: DBT context for this project 39 40 Returns: 41 Project instance for the specified DBT project 42 """ 43 context = context.copy() 44 45 project_file_path = Path(context.project_root, PROJECT_FILENAME) 46 if not project_file_path.exists(): 47 raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}") 48 project_yaml = load_yaml(project_file_path) 49 50 variables = project_yaml.get("vars", {}) 51 context.variables = { 52 name: var for name, var in variables.items() if not isinstance(var, t.Dict) 53 } 54 55 context.project_name = context.render(project_yaml.get("name", "")) 56 if not context.project_name: 57 raise ConfigError(f"{project_file_path.stem} must include project name.") 58 59 context.profile_name = ( 60 context.render(project_yaml.get("profile", "")) or context.project_name 61 ) 62 63 profile = Profile.load(context) 64 context.target = ( 65 profile.targets[context.target_name] 66 if context.target_name 67 else profile.targets[profile.default_target] 68 ) 69 70 packages = {} 71 root_loader = PackageLoader(context, ProjectConfig()) 72 73 packages[context.project_name] = root_loader.load() 74 project_config = root_loader.project_config 75 76 packages_dir = Path( 77 context.render(project_yaml.get("packages-install-path", "dbt_packages")) 78 ) 79 if not packages_dir.is_absolute(): 80 packages_dir = Path(context.project_root, packages_dir) 81 82 for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"): 83 name = context.render(load_yaml(path).get("name", "")) 84 if not name: 85 raise ConfigError(f"{path} must include package name") 86 87 package_context = context.copy() 88 package_context.project_root = path.parent 89 package_context.variables = {} 90 packages[name] = PackageLoader( 91 package_context, cls._overrides_for_package(name, project_config) 92 ).load() 93 94 for name, package in packages.items(): 95 package_vars = variables.get(name) 96 97 if isinstance(package_vars, dict): 98 package.variables.update(package_vars) 99 100 return Project(context, profile, packages) 101 102 @classmethod 103 def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig: 104 overrides = ProjectConfig() 105 106 source_overrides = { 107 scope[1:]: value 108 for scope, value in config.source_config.items() 109 if scope and scope[0] == name 110 } 111 if source_overrides: 112 overrides.source_config = source_overrides 113 114 seed_overrides = { 115 scope[1:]: value 116 for scope, value in config.seed_config.items() 117 if scope and scope[0] == name 118 } 119 if seed_overrides: 120 overrides.seed_config = seed_overrides 121 122 model_overrides = { 123 scope[1:]: value 124 for scope, value in config.model_config.items() 125 if scope and scope[0] == name 126 } 127 if model_overrides: 128 overrides.model_config = model_overrides 129 130 return overrides 131 132 @property 133 def project_files(self) -> t.Set[Path]: 134 paths = {self.profile.path} 135 for package in self.packages.values(): 136 paths.update(package.files) 137 138 return paths
class
Project:
13class Project: 14 """Configuration for a DBT project""" 15 16 def __init__( 17 self, 18 context: DbtContext, 19 profile: Profile, 20 packages: t.Dict[str, Package], 21 ): 22 """ 23 Args: 24 context: DBT context for the project 25 profile: The profile associated with the project 26 packages: The packages in this project. The project should be included 27 with the project name as the key 28 """ 29 self.context = context 30 self.profile = profile 31 self.packages = packages 32 33 @classmethod 34 def load(cls, context: DbtContext) -> Project: 35 """ 36 Loads the configuration for the specified DBT project 37 38 Args: 39 context: DBT context for this project 40 41 Returns: 42 Project instance for the specified DBT project 43 """ 44 context = context.copy() 45 46 project_file_path = Path(context.project_root, PROJECT_FILENAME) 47 if not project_file_path.exists(): 48 raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}") 49 project_yaml = load_yaml(project_file_path) 50 51 variables = project_yaml.get("vars", {}) 52 context.variables = { 53 name: var for name, var in variables.items() if not isinstance(var, t.Dict) 54 } 55 56 context.project_name = context.render(project_yaml.get("name", "")) 57 if not context.project_name: 58 raise ConfigError(f"{project_file_path.stem} must include project name.") 59 60 context.profile_name = ( 61 context.render(project_yaml.get("profile", "")) or context.project_name 62 ) 63 64 profile = Profile.load(context) 65 context.target = ( 66 profile.targets[context.target_name] 67 if context.target_name 68 else profile.targets[profile.default_target] 69 ) 70 71 packages = {} 72 root_loader = PackageLoader(context, ProjectConfig()) 73 74 packages[context.project_name] = root_loader.load() 75 project_config = root_loader.project_config 76 77 packages_dir = Path( 78 context.render(project_yaml.get("packages-install-path", "dbt_packages")) 79 ) 80 if not packages_dir.is_absolute(): 81 packages_dir = Path(context.project_root, packages_dir) 82 83 for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"): 84 name = context.render(load_yaml(path).get("name", "")) 85 if not name: 86 raise ConfigError(f"{path} must include package name") 87 88 package_context = context.copy() 89 package_context.project_root = path.parent 90 package_context.variables = {} 91 packages[name] = PackageLoader( 92 package_context, cls._overrides_for_package(name, project_config) 93 ).load() 94 95 for name, package in packages.items(): 96 package_vars = variables.get(name) 97 98 if isinstance(package_vars, dict): 99 package.variables.update(package_vars) 100 101 return Project(context, profile, packages) 102 103 @classmethod 104 def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig: 105 overrides = ProjectConfig() 106 107 source_overrides = { 108 scope[1:]: value 109 for scope, value in config.source_config.items() 110 if scope and scope[0] == name 111 } 112 if source_overrides: 113 overrides.source_config = source_overrides 114 115 seed_overrides = { 116 scope[1:]: value 117 for scope, value in config.seed_config.items() 118 if scope and scope[0] == name 119 } 120 if seed_overrides: 121 overrides.seed_config = seed_overrides 122 123 model_overrides = { 124 scope[1:]: value 125 for scope, value in config.model_config.items() 126 if scope and scope[0] == name 127 } 128 if model_overrides: 129 overrides.model_config = model_overrides 130 131 return overrides 132 133 @property 134 def project_files(self) -> t.Set[Path]: 135 paths = {self.profile.path} 136 for package in self.packages.values(): 137 paths.update(package.files) 138 139 return paths
Configuration for a DBT project
Project( context: sqlmesh.dbt.common.DbtContext, profile: sqlmesh.dbt.profile.Profile, packages: Dict[str, sqlmesh.dbt.package.Package])
16 def __init__( 17 self, 18 context: DbtContext, 19 profile: Profile, 20 packages: t.Dict[str, Package], 21 ): 22 """ 23 Args: 24 context: DBT context for the project 25 profile: The profile associated with the project 26 packages: The packages in this project. The project should be included 27 with the project name as the key 28 """ 29 self.context = context 30 self.profile = profile 31 self.packages = packages
Arguments:
- context: DBT context for the project
- profile: The profile associated with the project
- packages: The packages in this project. The project should be included with the project name as the key
33 @classmethod 34 def load(cls, context: DbtContext) -> Project: 35 """ 36 Loads the configuration for the specified DBT project 37 38 Args: 39 context: DBT context for this project 40 41 Returns: 42 Project instance for the specified DBT project 43 """ 44 context = context.copy() 45 46 project_file_path = Path(context.project_root, PROJECT_FILENAME) 47 if not project_file_path.exists(): 48 raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}") 49 project_yaml = load_yaml(project_file_path) 50 51 variables = project_yaml.get("vars", {}) 52 context.variables = { 53 name: var for name, var in variables.items() if not isinstance(var, t.Dict) 54 } 55 56 context.project_name = context.render(project_yaml.get("name", "")) 57 if not context.project_name: 58 raise ConfigError(f"{project_file_path.stem} must include project name.") 59 60 context.profile_name = ( 61 context.render(project_yaml.get("profile", "")) or context.project_name 62 ) 63 64 profile = Profile.load(context) 65 context.target = ( 66 profile.targets[context.target_name] 67 if context.target_name 68 else profile.targets[profile.default_target] 69 ) 70 71 packages = {} 72 root_loader = PackageLoader(context, ProjectConfig()) 73 74 packages[context.project_name] = root_loader.load() 75 project_config = root_loader.project_config 76 77 packages_dir = Path( 78 context.render(project_yaml.get("packages-install-path", "dbt_packages")) 79 ) 80 if not packages_dir.is_absolute(): 81 packages_dir = Path(context.project_root, packages_dir) 82 83 for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"): 84 name = context.render(load_yaml(path).get("name", "")) 85 if not name: 86 raise ConfigError(f"{path} must include package name") 87 88 package_context = context.copy() 89 package_context.project_root = path.parent 90 package_context.variables = {} 91 packages[name] = PackageLoader( 92 package_context, cls._overrides_for_package(name, project_config) 93 ).load() 94 95 for name, package in packages.items(): 96 package_vars = variables.get(name) 97 98 if isinstance(package_vars, dict): 99 package.variables.update(package_vars) 100 101 return Project(context, profile, packages)
Loads the configuration for the specified DBT project
Arguments:
- context: DBT context for this project
Returns:
Project instance for the specified DBT project