sqlmesh.core.config.loader
1from __future__ import annotations 2 3import os 4import typing as t 5from pathlib import Path 6 7from sqlmesh.core import constants as c 8from sqlmesh.core.config.root import Config 9from sqlmesh.utils import sys_path 10from sqlmesh.utils.errors import ConfigError 11from sqlmesh.utils.metaprogramming import import_python_file 12from sqlmesh.utils.yaml import load as yaml_load 13 14 15def load_config_from_paths( 16 *paths: Path, config_name: str = "config", load_from_env: bool = True 17) -> Config: 18 config: t.Optional[Config] = None 19 20 visited_folders: t.Set[Path] = set() 21 for path in paths: 22 if not path.exists(): 23 continue 24 25 if not path.is_file(): 26 raise ConfigError(f"Path '{path}' must be a file.") 27 28 parent_path = path.parent 29 if parent_path in visited_folders: 30 raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.") 31 visited_folders.add(parent_path) 32 33 extension = path.name.split(".")[-1].lower() 34 if extension in ("yml", "yaml"): 35 next_config = load_config_from_yaml(path) 36 elif extension == "py": 37 next_config = load_config_from_python_module(path, config_name=config_name) 38 else: 39 raise ConfigError( 40 f"Unsupported config file extension '{extension}' in config file '{path}'." 41 ) 42 43 config = config.update_with(next_config) if config is not None else next_config 44 45 if config is None: 46 raise ConfigError( 47 "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`." 48 ) 49 50 if load_from_env: 51 config = config.update_with(load_config_from_env()) 52 53 return config 54 55 56def load_config_from_yaml(path: Path) -> Config: 57 config_dict = yaml_load(path) 58 return Config.parse_obj(config_dict) 59 60 61def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config: 62 with sys_path(module_path.parent): 63 try: 64 config_module = import_python_file(module_path, module_path.parent) 65 except ImportError: 66 raise ConfigError(f"Config module '{module_path}' was not found.") 67 68 try: 69 config_obj = getattr(config_module, config_name) 70 except AttributeError: 71 raise ConfigError(f"Config '{config_name}' was not found.") 72 73 if config_obj is None or not isinstance(config_obj, Config): 74 raise ConfigError( 75 f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'." 76 ) 77 78 return config_obj 79 80 81def load_config_from_env() -> Config: 82 config_dict: t.Dict[str, t.Any] = {} 83 84 for key, value in os.environ.items(): 85 key = key.lower() 86 if key.startswith(f"{c.SQLMESH}__"): 87 segments = key.split("__")[1:] 88 if not segments or not segments[-1]: 89 raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.") 90 91 target_dict = config_dict 92 for config_key in segments[:-1]: 93 if config_key not in target_dict: 94 target_dict[config_key] = {} 95 target_dict = target_dict[config_key] 96 target_dict[segments[-1]] = value 97 98 return Config.parse_obj(config_dict)
def
load_config_from_paths( *paths: pathlib.Path, config_name: str = 'config', load_from_env: bool = True) -> sqlmesh.core.config.root.Config:
16def load_config_from_paths( 17 *paths: Path, config_name: str = "config", load_from_env: bool = True 18) -> Config: 19 config: t.Optional[Config] = None 20 21 visited_folders: t.Set[Path] = set() 22 for path in paths: 23 if not path.exists(): 24 continue 25 26 if not path.is_file(): 27 raise ConfigError(f"Path '{path}' must be a file.") 28 29 parent_path = path.parent 30 if parent_path in visited_folders: 31 raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.") 32 visited_folders.add(parent_path) 33 34 extension = path.name.split(".")[-1].lower() 35 if extension in ("yml", "yaml"): 36 next_config = load_config_from_yaml(path) 37 elif extension == "py": 38 next_config = load_config_from_python_module(path, config_name=config_name) 39 else: 40 raise ConfigError( 41 f"Unsupported config file extension '{extension}' in config file '{path}'." 42 ) 43 44 config = config.update_with(next_config) if config is not None else next_config 45 46 if config is None: 47 raise ConfigError( 48 "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`." 49 ) 50 51 if load_from_env: 52 config = config.update_with(load_config_from_env()) 53 54 return config
def
load_config_from_python_module( module_path: pathlib.Path, config_name: str = 'config') -> sqlmesh.core.config.root.Config:
62def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config: 63 with sys_path(module_path.parent): 64 try: 65 config_module = import_python_file(module_path, module_path.parent) 66 except ImportError: 67 raise ConfigError(f"Config module '{module_path}' was not found.") 68 69 try: 70 config_obj = getattr(config_module, config_name) 71 except AttributeError: 72 raise ConfigError(f"Config '{config_name}' was not found.") 73 74 if config_obj is None or not isinstance(config_obj, Config): 75 raise ConfigError( 76 f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'." 77 ) 78 79 return config_obj
82def load_config_from_env() -> Config: 83 config_dict: t.Dict[str, t.Any] = {} 84 85 for key, value in os.environ.items(): 86 key = key.lower() 87 if key.startswith(f"{c.SQLMESH}__"): 88 segments = key.split("__")[1:] 89 if not segments or not segments[-1]: 90 raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.") 91 92 target_dict = config_dict 93 for config_key in segments[:-1]: 94 if config_key not in target_dict: 95 target_dict[config_key] = {} 96 target_dict = target_dict[config_key] 97 target_dict[segments[-1]] = value 98 99 return Config.parse_obj(config_dict)