Edit on GitHub

sqlmesh.utils.metaprogramming

  1from __future__ import annotations
  2
  3import ast
  4import dis
  5import importlib
  6import inspect
  7import linecache
  8import os
  9import re
 10import sys
 11import textwrap
 12import types
 13import typing as t
 14from enum import Enum
 15from pathlib import Path
 16
 17from astor import to_source
 18
 19from sqlmesh.utils import format_exception, unique
 20from sqlmesh.utils.errors import SQLMeshError
 21from sqlmesh.utils.pydantic import PydanticModel
 22
 23IGNORE_DECORATORS = {"hook", "macro", "model"}
 24
 25
 26def _is_relative_to(path: t.Optional[Path | str], other: t.Optional[Path | str]) -> bool:
 27    """path.is_relative_to compatibility, was only supported >= 3.9"""
 28    if path is None or other is None:
 29        return False
 30
 31    if isinstance(path, str):
 32        path = Path(path)
 33    if isinstance(other, str):
 34        other = Path(other)
 35
 36    try:
 37        path.absolute().relative_to(other.absolute())
 38        return True
 39    except ValueError:
 40        return False
 41
 42
 43def _code_globals(code: types.CodeType) -> t.Dict[str, None]:
 44    variables = {
 45        instruction.argval: None
 46        for instruction in dis.get_instructions(code)
 47        if instruction.opname == "LOAD_GLOBAL"
 48    }
 49
 50    for const in code.co_consts:
 51        if isinstance(const, types.CodeType):
 52            variables.update(_code_globals(const))
 53
 54    return variables
 55
 56
 57def func_globals(func: t.Callable) -> t.Dict[str, t.Any]:
 58    """Finds all global references and closures in a function and nested functions.
 59
 60    This function treats closures as global variables, which could cause problems in the future.
 61
 62    Args:
 63        func: The function to introspect
 64
 65    Returns:
 66        A dictionary of all global references.
 67    """
 68    variables = {}
 69
 70    if hasattr(func, "__code__"):
 71        code = func.__code__
 72
 73        for var in list(_code_globals(code)) + decorators(func):
 74            if var in func.__globals__:
 75                ref = func.__globals__[var]
 76                variables[var] = ref
 77
 78        if func.__closure__:
 79            for var, value in zip(code.co_freevars, func.__closure__):
 80                variables[var] = value.cell_contents
 81
 82    return variables
 83
 84
 85class ClassFoundException(Exception):
 86    pass
 87
 88
 89class _ClassFinder(ast.NodeVisitor):
 90    def __init__(self, qualname: str) -> None:
 91        self.stack: t.List[str] = []
 92        self.qualname = qualname
 93
 94    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
 95        self.stack.append(node.name)
 96        self.stack.append("<locals>")
 97        self.generic_visit(node)
 98        self.stack.pop()
 99        self.stack.pop()
100
101    visit_AsyncFunctionDef = visit_FunctionDef  # type: ignore
102
103    def visit_ClassDef(self, node: ast.ClassDef) -> None:
104        self.stack.append(node.name)
105        if self.qualname == ".".join(self.stack):
106            # Return the decorator for the class if present
107            if node.decorator_list:
108                line_number = node.decorator_list[0].lineno
109            else:
110                line_number = node.lineno
111
112            # decrement by one since lines starts with indexing by zero
113            line_number -= 1
114            raise ClassFoundException(line_number)
115        self.generic_visit(node)
116        self.stack.pop()
117
118
119def getsource(obj: t.Any) -> str:
120    """Get the source of a function or class.
121
122    inspect.getsource doesn't find decorators in python < 3.9
123    https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade
124    """
125    path = inspect.getsourcefile(obj)
126    if path:
127        module = inspect.getmodule(obj, path)
128
129        if module:
130            lines = linecache.getlines(path, module.__dict__)
131        else:
132            lines = linecache.getlines(path)
133
134        def join_source(lnum: int) -> str:
135            return "".join(inspect.getblock(lines[lnum:]))
136
137        if inspect.isclass(obj):
138            qualname = obj.__qualname__
139            source = "".join(lines)
140            tree = ast.parse(source)
141            class_finder = _ClassFinder(qualname)
142            try:
143                class_finder.visit(tree)
144            except ClassFoundException as e:
145                return join_source(e.args[0])
146        elif inspect.isfunction(obj):
147            obj = obj.__code__
148            if hasattr(obj, "co_firstlineno"):
149                lnum = obj.co_firstlineno - 1
150                pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)")
151                while lnum > 0:
152                    try:
153                        line = lines[lnum]
154                    except IndexError:
155                        raise OSError("lineno is out of bounds")
156                    if pat.match(line):
157                        break
158                    lnum = lnum - 1
159                return join_source(lnum)
160    raise SQLMeshError(f"Cannot find source for {obj}")
161
162
163def parse_source(func: t.Callable) -> ast.Module:
164    """Parse a function and returns an ast node."""
165    return ast.parse(textwrap.dedent(getsource(func)))
166
167
168def _decorator_name(decorator: ast.expr) -> str:
169    if isinstance(decorator, ast.Call):
170        return decorator.func.id  # type: ignore
171    if isinstance(decorator, ast.Name):
172        return decorator.id
173    return ""
174
175
176def decorators(func: t.Callable) -> t.List[str]:
177    """Finds a list of all the decorators of a callable."""
178    root_node = parse_source(func)
179    decorators = []
180
181    for node in ast.walk(root_node):
182        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
183            for decorator in node.decorator_list:
184                name = _decorator_name(decorator)
185                if name not in IGNORE_DECORATORS:
186                    decorators.append(name)
187    return unique(decorators)
188
189
190def normalize_source(obj: t.Any) -> str:
191    """Rewrites an object's source with formatting and doc strings removed by using Python ast.
192
193    Args:
194        obj: The object to fetch source from and convert to a string.
195
196    Returns:
197        A string representation of the normalized function.
198    """
199    root_node = parse_source(obj)
200
201    for node in ast.walk(root_node):
202        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
203            for decorator in node.decorator_list:
204                if _decorator_name(decorator) in IGNORE_DECORATORS:
205                    node.decorator_list.remove(decorator)
206
207            # remove docstrings
208            body = node.body
209            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str):
210                node.body = body[1:]
211
212            # remove function return type annotation
213            if isinstance(node, ast.FunctionDef):
214                node.returns = None
215        elif isinstance(node, ast.arg):
216            node.annotation = None
217
218    return to_source(root_node).strip()
219
220
221def build_env(
222    obj: t.Any,
223    *,
224    env: t.Dict[str, t.Any],
225    name: str,
226    path: Path,
227) -> None:
228    """Fills in env dictionary with all globals needed to execute the object.
229
230    Recursively traverse classes and functions.
231
232    Args:
233        obj: Any python object.
234        env: Dictionary to store the env.
235        name: Name of the object in the env.
236        path: The module path to serialize. Other modules will not be walked and treated as imports.
237    """
238
239    obj_module = inspect.getmodule(obj)
240
241    if obj_module and obj_module.__name__ == "builtins":
242        return
243
244    def walk(obj: t.Any) -> None:
245        if inspect.isclass(obj):
246            for decorator in decorators(obj):
247                if obj_module and decorator in obj_module.__dict__:
248                    build_env(
249                        obj_module.__dict__[decorator],
250                        env=env,
251                        name=decorator,
252                        path=path,
253                    )
254
255            for base in obj.__bases__:
256                build_env(base, env=env, name=base.__qualname__, path=path)
257
258            for k, v in obj.__dict__.items():
259                if k.startswith("__"):
260                    continue
261                # traverse methods in a class to find global references
262                if isinstance(v, (classmethod, staticmethod)):
263                    v = v.__func__
264                if callable(v):
265                    # if the method is a part of the object, walk it
266                    # else it is a global function and we just store it
267                    if v.__qualname__.startswith(obj.__qualname__):
268                        walk(v)
269                    else:
270                        build_env(v, env=env, name=v.__name__, path=path)
271        elif callable(obj):
272            for k, v in func_globals(obj).items():
273                build_env(v, env=env, name=k, path=path)
274
275    if name not in env:
276        env[name] = obj
277        if obj_module and _is_relative_to(obj_module.__file__, path):
278            walk(obj)
279    elif env[name] != obj:
280        raise SQLMeshError(
281            f"Cannot store {obj} in environment, duplicate definitions found for '{name}'"
282        )
283
284
285class ExecutableKind(str, Enum):
286    """The kind of of executable. The order of the members is used when serializing the python model to text."""
287
288    IMPORT = "import"
289    VALUE = "value"
290    DEFINITION = "definition"
291    STATEMENT = "statement"
292
293    def __lt__(self, other: t.Any) -> bool:
294        if not isinstance(other, ExecutableKind):
295            return NotImplemented
296        values = list(ExecutableKind.__dict__.values())
297        return values.index(self) < values.index(other)
298
299
300class Executable(PydanticModel):
301    payload: t.Any
302    kind: ExecutableKind = ExecutableKind.DEFINITION
303    name: t.Optional[str] = None
304    path: t.Optional[str] = None
305    alias: t.Optional[str] = None
306
307    @property
308    def is_definition(self) -> bool:
309        return self.kind == ExecutableKind.DEFINITION
310
311    @property
312    def is_import(self) -> bool:
313        return self.kind == ExecutableKind.IMPORT
314
315    @property
316    def is_statement(self) -> bool:
317        return self.kind == ExecutableKind.STATEMENT
318
319    @property
320    def is_value(self) -> bool:
321        return self.kind == ExecutableKind.VALUE
322
323
324def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
325    """Serializes a python function into a self contained dictionary.
326
327    Recursively walks a function's globals to store all other references inside of env.
328
329    Args:
330        env: Dictionary to store the env.
331        path: The root path to seralize. Other modules will not be walked and treated as imports.
332    """
333    serialized = {}
334
335    for k, v in env.items():
336        if callable(v):
337            name = v.__name__
338            name = k if name == "<lambda>" else name
339            file_path = Path(inspect.getfile(v))
340
341            if _is_relative_to(file_path, path):
342                serialized[k] = Executable(
343                    name=name,
344                    payload=normalize_source(v),
345                    kind=ExecutableKind.DEFINITION,
346                    path=str(file_path.relative_to(path.absolute())),
347                    alias=k if name != k else None,
348                )
349            else:
350                serialized[k] = Executable(
351                    payload=f"from {v.__module__} import {name}",
352                    kind=ExecutableKind.IMPORT,
353                )
354        elif inspect.ismodule(v):
355            name = v.__name__
356            if _is_relative_to(v.__file__, path):
357                raise SQLMeshError(
358                    f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead."
359                )
360            postfix = "" if name == k else f" as {k}"
361            serialized[k] = Executable(
362                payload=f"import {name}{postfix}",
363                kind=ExecutableKind.IMPORT,
364            )
365        else:
366            serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE)
367
368    return serialized
369
370
371def prepare_env(
372    python_env: t.Dict[str, Executable],
373    env: t.Optional[t.Dict[str, t.Any]] = None,
374) -> t.Dict[str, t.Any]:
375    """Prepare a python env by hydrating and executing functions.
376
377    The Python ENV is stored in a json serializable format.
378    Functions and imports are stored as a special data class.
379
380    Args:
381        python_env: The dictionary containing the serialized python environment.
382        env: The dictionary to execute code in.
383
384    Returns:
385        The prepared environment with hydrated functions.
386    """
387    env = {} if env is None else env
388
389    for name, executable in sorted(
390        python_env.items(), key=lambda item: 0 if item[1].is_import else 1
391    ):
392        if executable.is_value:
393            env[name] = ast.literal_eval(executable.payload)
394        else:
395            exec(executable.payload, env)
396            if executable.alias and executable.name:
397                env[executable.alias] = env[executable.name]
398    return env
399
400
401def print_exception(
402    exception: Exception,
403    python_env: t.Dict[str, Executable],
404    out: t.TextIO = sys.stderr,
405) -> None:
406    """Formats exceptions that occur from evaled code.
407
408    Stack traces generated by evaled code lose code context and are difficult to debug.
409    This intercepts the default stack trace and tries to make it debuggable.
410
411    Args:
412        exception: The exception to print the stack trace for.
413        python_env: The environment containing stringified python code.
414        out: The output stream to write to.
415    """
416    tb: t.List[str] = []
417
418    for error_line in format_exception(exception):
419        match = re.search(f'File "<string>", line (.*), in (.*)', error_line)
420
421        if not match:
422            tb.append(error_line)
423            continue
424
425        line_num = int(match.group(1))
426        func = match.group(2)
427
428        if func not in python_env:
429            tb.append(error_line)
430            continue
431
432        executable = python_env[func]
433        indent = error_line[: match.start()]
434
435        error_line = (
436            f"{indent}File '{executable.path}' (or imported file), line {line_num}, in {func}"
437        )
438
439        code = executable.payload
440        formatted = []
441
442        for i, code_line in enumerate(code.splitlines()):
443            if i < line_num:
444                pad = len(code_line) - len(code_line.lstrip())
445                if i + 1 == line_num:
446                    formatted.append(f"{code_line[:pad]}{code_line[pad:]}")
447                else:
448                    formatted.append(code_line)
449
450        tb.extend(
451            (
452                error_line,
453                textwrap.indent(
454                    os.linesep.join(formatted),
455                    indent + "  ",
456                ),
457                os.linesep,
458            )
459        )
460
461    out.write(os.linesep.join(tb))
462
463
464def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:
465    module_name = str(
466        path.absolute().relative_to(relative_base.absolute()).with_suffix("")
467    ).replace(os.path.sep, ".")
468    # remove the entire module hierarchy in case they were already loaded
469    parts = module_name.split(".")
470    for i in range(len(parts)):
471        sys.modules.pop(".".join(parts[0 : i + 1]), None)
472
473    return importlib.import_module(module_name)
def func_globals(func: Callable) -> Dict[str, Any]:
58def func_globals(func: t.Callable) -> t.Dict[str, t.Any]:
59    """Finds all global references and closures in a function and nested functions.
60
61    This function treats closures as global variables, which could cause problems in the future.
62
63    Args:
64        func: The function to introspect
65
66    Returns:
67        A dictionary of all global references.
68    """
69    variables = {}
70
71    if hasattr(func, "__code__"):
72        code = func.__code__
73
74        for var in list(_code_globals(code)) + decorators(func):
75            if var in func.__globals__:
76                ref = func.__globals__[var]
77                variables[var] = ref
78
79        if func.__closure__:
80            for var, value in zip(code.co_freevars, func.__closure__):
81                variables[var] = value.cell_contents
82
83    return variables

Finds all global references and closures in a function and nested functions.

This function treats closures as global variables, which could cause problems in the future.

Arguments:
  • func: The function to introspect
Returns:

A dictionary of all global references.

class ClassFoundException(builtins.Exception):
86class ClassFoundException(Exception):
87    pass

Common base class for all non-exit exceptions.

Inherited Members
builtins.Exception
Exception
builtins.BaseException
with_traceback
def getsource(obj: Any) -> str:
120def getsource(obj: t.Any) -> str:
121    """Get the source of a function or class.
122
123    inspect.getsource doesn't find decorators in python < 3.9
124    https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade
125    """
126    path = inspect.getsourcefile(obj)
127    if path:
128        module = inspect.getmodule(obj, path)
129
130        if module:
131            lines = linecache.getlines(path, module.__dict__)
132        else:
133            lines = linecache.getlines(path)
134
135        def join_source(lnum: int) -> str:
136            return "".join(inspect.getblock(lines[lnum:]))
137
138        if inspect.isclass(obj):
139            qualname = obj.__qualname__
140            source = "".join(lines)
141            tree = ast.parse(source)
142            class_finder = _ClassFinder(qualname)
143            try:
144                class_finder.visit(tree)
145            except ClassFoundException as e:
146                return join_source(e.args[0])
147        elif inspect.isfunction(obj):
148            obj = obj.__code__
149            if hasattr(obj, "co_firstlineno"):
150                lnum = obj.co_firstlineno - 1
151                pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)")
152                while lnum > 0:
153                    try:
154                        line = lines[lnum]
155                    except IndexError:
156                        raise OSError("lineno is out of bounds")
157                    if pat.match(line):
158                        break
159                    lnum = lnum - 1
160                return join_source(lnum)
161    raise SQLMeshError(f"Cannot find source for {obj}")

Get the source of a function or class.

inspect.getsource doesn't find decorators in python < 3.9 https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade

def parse_source(func: Callable) -> ast.Module:
164def parse_source(func: t.Callable) -> ast.Module:
165    """Parse a function and returns an ast node."""
166    return ast.parse(textwrap.dedent(getsource(func)))

Parse a function and returns an ast node.

def decorators(func: Callable) -> List[str]:
177def decorators(func: t.Callable) -> t.List[str]:
178    """Finds a list of all the decorators of a callable."""
179    root_node = parse_source(func)
180    decorators = []
181
182    for node in ast.walk(root_node):
183        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
184            for decorator in node.decorator_list:
185                name = _decorator_name(decorator)
186                if name not in IGNORE_DECORATORS:
187                    decorators.append(name)
188    return unique(decorators)

Finds a list of all the decorators of a callable.

def normalize_source(obj: Any) -> str:
191def normalize_source(obj: t.Any) -> str:
192    """Rewrites an object's source with formatting and doc strings removed by using Python ast.
193
194    Args:
195        obj: The object to fetch source from and convert to a string.
196
197    Returns:
198        A string representation of the normalized function.
199    """
200    root_node = parse_source(obj)
201
202    for node in ast.walk(root_node):
203        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
204            for decorator in node.decorator_list:
205                if _decorator_name(decorator) in IGNORE_DECORATORS:
206                    node.decorator_list.remove(decorator)
207
208            # remove docstrings
209            body = node.body
210            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str):
211                node.body = body[1:]
212
213            # remove function return type annotation
214            if isinstance(node, ast.FunctionDef):
215                node.returns = None
216        elif isinstance(node, ast.arg):
217            node.annotation = None
218
219    return to_source(root_node).strip()

Rewrites an object's source with formatting and doc strings removed by using Python ast.

Arguments:
  • obj: The object to fetch source from and convert to a string.
Returns:

A string representation of the normalized function.

def build_env(obj: Any, *, env: Dict[str, Any], name: str, path: pathlib.Path) -> None:
222def build_env(
223    obj: t.Any,
224    *,
225    env: t.Dict[str, t.Any],
226    name: str,
227    path: Path,
228) -> None:
229    """Fills in env dictionary with all globals needed to execute the object.
230
231    Recursively traverse classes and functions.
232
233    Args:
234        obj: Any python object.
235        env: Dictionary to store the env.
236        name: Name of the object in the env.
237        path: The module path to serialize. Other modules will not be walked and treated as imports.
238    """
239
240    obj_module = inspect.getmodule(obj)
241
242    if obj_module and obj_module.__name__ == "builtins":
243        return
244
245    def walk(obj: t.Any) -> None:
246        if inspect.isclass(obj):
247            for decorator in decorators(obj):
248                if obj_module and decorator in obj_module.__dict__:
249                    build_env(
250                        obj_module.__dict__[decorator],
251                        env=env,
252                        name=decorator,
253                        path=path,
254                    )
255
256            for base in obj.__bases__:
257                build_env(base, env=env, name=base.__qualname__, path=path)
258
259            for k, v in obj.__dict__.items():
260                if k.startswith("__"):
261                    continue
262                # traverse methods in a class to find global references
263                if isinstance(v, (classmethod, staticmethod)):
264                    v = v.__func__
265                if callable(v):
266                    # if the method is a part of the object, walk it
267                    # else it is a global function and we just store it
268                    if v.__qualname__.startswith(obj.__qualname__):
269                        walk(v)
270                    else:
271                        build_env(v, env=env, name=v.__name__, path=path)
272        elif callable(obj):
273            for k, v in func_globals(obj).items():
274                build_env(v, env=env, name=k, path=path)
275
276    if name not in env:
277        env[name] = obj
278        if obj_module and _is_relative_to(obj_module.__file__, path):
279            walk(obj)
280    elif env[name] != obj:
281        raise SQLMeshError(
282            f"Cannot store {obj} in environment, duplicate definitions found for '{name}'"
283        )

Fills in env dictionary with all globals needed to execute the object.

Recursively traverse classes and functions.

Arguments:
  • obj: Any python object.
  • env: Dictionary to store the env.
  • name: Name of the object in the env.
  • path: The module path to serialize. Other modules will not be walked and treated as imports.
class ExecutableKind(builtins.str, enum.Enum):
286class ExecutableKind(str, Enum):
287    """The kind of of executable. The order of the members is used when serializing the python model to text."""
288
289    IMPORT = "import"
290    VALUE = "value"
291    DEFINITION = "definition"
292    STATEMENT = "statement"
293
294    def __lt__(self, other: t.Any) -> bool:
295        if not isinstance(other, ExecutableKind):
296            return NotImplemented
297        values = list(ExecutableKind.__dict__.values())
298        return values.index(self) < values.index(other)

The kind of of executable. The order of the members is used when serializing the python model to text.

IMPORT = <ExecutableKind.IMPORT: 'import'>
VALUE = <ExecutableKind.VALUE: 'value'>
DEFINITION = <ExecutableKind.DEFINITION: 'definition'>
STATEMENT = <ExecutableKind.STATEMENT: 'statement'>
Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
removeprefix
removesuffix
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
class Executable(sqlmesh.utils.pydantic.PydanticModel):
301class Executable(PydanticModel):
302    payload: t.Any
303    kind: ExecutableKind = ExecutableKind.DEFINITION
304    name: t.Optional[str] = None
305    path: t.Optional[str] = None
306    alias: t.Optional[str] = None
307
308    @property
309    def is_definition(self) -> bool:
310        return self.kind == ExecutableKind.DEFINITION
311
312    @property
313    def is_import(self) -> bool:
314        return self.kind == ExecutableKind.IMPORT
315
316    @property
317    def is_statement(self) -> bool:
318        return self.kind == ExecutableKind.STATEMENT
319
320    @property
321    def is_value(self) -> bool:
322        return self.kind == ExecutableKind.VALUE
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
def serialize_env( env: Dict[str, Any], path: pathlib.Path) -> Dict[str, sqlmesh.utils.metaprogramming.Executable]:
325def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
326    """Serializes a python function into a self contained dictionary.
327
328    Recursively walks a function's globals to store all other references inside of env.
329
330    Args:
331        env: Dictionary to store the env.
332        path: The root path to seralize. Other modules will not be walked and treated as imports.
333    """
334    serialized = {}
335
336    for k, v in env.items():
337        if callable(v):
338            name = v.__name__
339            name = k if name == "<lambda>" else name
340            file_path = Path(inspect.getfile(v))
341
342            if _is_relative_to(file_path, path):
343                serialized[k] = Executable(
344                    name=name,
345                    payload=normalize_source(v),
346                    kind=ExecutableKind.DEFINITION,
347                    path=str(file_path.relative_to(path.absolute())),
348                    alias=k if name != k else None,
349                )
350            else:
351                serialized[k] = Executable(
352                    payload=f"from {v.__module__} import {name}",
353                    kind=ExecutableKind.IMPORT,
354                )
355        elif inspect.ismodule(v):
356            name = v.__name__
357            if _is_relative_to(v.__file__, path):
358                raise SQLMeshError(
359                    f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead."
360                )
361            postfix = "" if name == k else f" as {k}"
362            serialized[k] = Executable(
363                payload=f"import {name}{postfix}",
364                kind=ExecutableKind.IMPORT,
365            )
366        else:
367            serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE)
368
369    return serialized

Serializes a python function into a self contained dictionary.

Recursively walks a function's globals to store all other references inside of env.

Arguments:
  • env: Dictionary to store the env.
  • path: The root path to seralize. Other modules will not be walked and treated as imports.
def prepare_env( python_env: Dict[str, sqlmesh.utils.metaprogramming.Executable], env: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
372def prepare_env(
373    python_env: t.Dict[str, Executable],
374    env: t.Optional[t.Dict[str, t.Any]] = None,
375) -> t.Dict[str, t.Any]:
376    """Prepare a python env by hydrating and executing functions.
377
378    The Python ENV is stored in a json serializable format.
379    Functions and imports are stored as a special data class.
380
381    Args:
382        python_env: The dictionary containing the serialized python environment.
383        env: The dictionary to execute code in.
384
385    Returns:
386        The prepared environment with hydrated functions.
387    """
388    env = {} if env is None else env
389
390    for name, executable in sorted(
391        python_env.items(), key=lambda item: 0 if item[1].is_import else 1
392    ):
393        if executable.is_value:
394            env[name] = ast.literal_eval(executable.payload)
395        else:
396            exec(executable.payload, env)
397            if executable.alias and executable.name:
398                env[executable.alias] = env[executable.name]
399    return env

Prepare a python env by hydrating and executing functions.

The Python ENV is stored in a json serializable format. Functions and imports are stored as a special data class.

Arguments:
  • python_env: The dictionary containing the serialized python environment.
  • env: The dictionary to execute code in.
Returns:

The prepared environment with hydrated functions.

def import_python_file( path: pathlib.Path, relative_base: pathlib.Path = PosixPath('.')) -> module:
465def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:
466    module_name = str(
467        path.absolute().relative_to(relative_base.absolute()).with_suffix("")
468    ).replace(os.path.sep, ".")
469    # remove the entire module hierarchy in case they were already loaded
470    parts = module_name.split(".")
471    for i in range(len(parts)):
472        sys.modules.pop(".".join(parts[0 : i + 1]), None)
473
474    return importlib.import_module(module_name)