Edit on GitHub

sqlmesh.core.config.loader

 1from __future__ import annotations
 2
 3import os
 4import typing as t
 5from pathlib import Path
 6
 7from sqlmesh.core import constants as c
 8from sqlmesh.core.config.root import Config
 9from sqlmesh.utils import sys_path
10from sqlmesh.utils.errors import ConfigError
11from sqlmesh.utils.metaprogramming import import_python_file
12from sqlmesh.utils.yaml import load as yaml_load
13
14
15def load_config_from_paths(
16    *paths: Path, config_name: str = "config", load_from_env: bool = True
17) -> Config:
18    config: t.Optional[Config] = None
19
20    visited_folders: t.Set[Path] = set()
21    for path in paths:
22        if not path.exists():
23            continue
24
25        if not path.is_file():
26            raise ConfigError(f"Path '{path}' must be a file.")
27
28        parent_path = path.parent
29        if parent_path in visited_folders:
30            raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.")
31        visited_folders.add(parent_path)
32
33        extension = path.name.split(".")[-1].lower()
34        if extension in ("yml", "yaml"):
35            next_config = load_config_from_yaml(path)
36        elif extension == "py":
37            next_config = load_config_from_python_module(path, config_name=config_name)
38        else:
39            raise ConfigError(
40                f"Unsupported config file extension '{extension}' in config file '{path}'."
41            )
42
43        config = config.update_with(next_config) if config is not None else next_config
44
45    if config is None:
46        raise ConfigError(
47            "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`."
48        )
49
50    if load_from_env:
51        config = config.update_with(load_config_from_env())
52
53    return config
54
55
56def load_config_from_yaml(path: Path) -> Config:
57    config_dict = yaml_load(path)
58    return Config.parse_obj(config_dict)
59
60
61def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config:
62    with sys_path(module_path.parent):
63        try:
64            config_module = import_python_file(module_path, module_path.parent)
65        except ImportError:
66            raise ConfigError(f"Config module '{module_path}' was not found.")
67
68    try:
69        config_obj = getattr(config_module, config_name)
70    except AttributeError:
71        raise ConfigError(f"Config '{config_name}' was not found.")
72
73    if config_obj is None or not isinstance(config_obj, Config):
74        raise ConfigError(
75            f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'."
76        )
77
78    return config_obj
79
80
81def load_config_from_env() -> Config:
82    config_dict: t.Dict[str, t.Any] = {}
83
84    for key, value in os.environ.items():
85        key = key.lower()
86        if key.startswith(f"{c.SQLMESH}__"):
87            segments = key.split("__")[1:]
88            if not segments or not segments[-1]:
89                raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.")
90
91            target_dict = config_dict
92            for config_key in segments[:-1]:
93                if config_key not in target_dict:
94                    target_dict[config_key] = {}
95                target_dict = target_dict[config_key]
96            target_dict[segments[-1]] = value
97
98    return Config.parse_obj(config_dict)
def load_config_from_paths( *paths: pathlib.Path, config_name: str = 'config', load_from_env: bool = True) -> sqlmesh.core.config.root.Config:
16def load_config_from_paths(
17    *paths: Path, config_name: str = "config", load_from_env: bool = True
18) -> Config:
19    config: t.Optional[Config] = None
20
21    visited_folders: t.Set[Path] = set()
22    for path in paths:
23        if not path.exists():
24            continue
25
26        if not path.is_file():
27            raise ConfigError(f"Path '{path}' must be a file.")
28
29        parent_path = path.parent
30        if parent_path in visited_folders:
31            raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.")
32        visited_folders.add(parent_path)
33
34        extension = path.name.split(".")[-1].lower()
35        if extension in ("yml", "yaml"):
36            next_config = load_config_from_yaml(path)
37        elif extension == "py":
38            next_config = load_config_from_python_module(path, config_name=config_name)
39        else:
40            raise ConfigError(
41                f"Unsupported config file extension '{extension}' in config file '{path}'."
42            )
43
44        config = config.update_with(next_config) if config is not None else next_config
45
46    if config is None:
47        raise ConfigError(
48            "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`."
49        )
50
51    if load_from_env:
52        config = config.update_with(load_config_from_env())
53
54    return config
def load_config_from_yaml(path: pathlib.Path) -> sqlmesh.core.config.root.Config:
57def load_config_from_yaml(path: Path) -> Config:
58    config_dict = yaml_load(path)
59    return Config.parse_obj(config_dict)
def load_config_from_python_module( module_path: pathlib.Path, config_name: str = 'config') -> sqlmesh.core.config.root.Config:
62def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config:
63    with sys_path(module_path.parent):
64        try:
65            config_module = import_python_file(module_path, module_path.parent)
66        except ImportError:
67            raise ConfigError(f"Config module '{module_path}' was not found.")
68
69    try:
70        config_obj = getattr(config_module, config_name)
71    except AttributeError:
72        raise ConfigError(f"Config '{config_name}' was not found.")
73
74    if config_obj is None or not isinstance(config_obj, Config):
75        raise ConfigError(
76            f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'."
77        )
78
79    return config_obj
def load_config_from_env() -> sqlmesh.core.config.root.Config:
82def load_config_from_env() -> Config:
83    config_dict: t.Dict[str, t.Any] = {}
84
85    for key, value in os.environ.items():
86        key = key.lower()
87        if key.startswith(f"{c.SQLMESH}__"):
88            segments = key.split("__")[1:]
89            if not segments or not segments[-1]:
90                raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.")
91
92            target_dict = config_dict
93            for config_key in segments[:-1]:
94                if config_key not in target_dict:
95                    target_dict[config_key] = {}
96                target_dict = target_dict[config_key]
97            target_dict[segments[-1]] = value
98
99    return Config.parse_obj(config_dict)