sqlmesh.utils.metaprogramming
1from __future__ import annotations 2 3import ast 4import dis 5import importlib 6import inspect 7import linecache 8import os 9import re 10import sys 11import textwrap 12import types 13import typing as t 14from enum import Enum 15from pathlib import Path 16 17from astor import to_source 18 19from sqlmesh.utils import format_exception, unique 20from sqlmesh.utils.errors import SQLMeshError 21from sqlmesh.utils.pydantic import PydanticModel 22 23IGNORE_DECORATORS = {"hook", "macro", "model"} 24 25 26def _is_relative_to(path: t.Optional[Path | str], other: t.Optional[Path | str]) -> bool: 27 """path.is_relative_to compatibility, was only supported >= 3.9""" 28 if path is None or other is None: 29 return False 30 31 if isinstance(path, str): 32 path = Path(path) 33 if isinstance(other, str): 34 other = Path(other) 35 36 try: 37 path.absolute().relative_to(other.absolute()) 38 return True 39 except ValueError: 40 return False 41 42 43def _code_globals(code: types.CodeType) -> t.Dict[str, None]: 44 variables = { 45 instruction.argval: None 46 for instruction in dis.get_instructions(code) 47 if instruction.opname == "LOAD_GLOBAL" 48 } 49 50 for const in code.co_consts: 51 if isinstance(const, types.CodeType): 52 variables.update(_code_globals(const)) 53 54 return variables 55 56 57def func_globals(func: t.Callable) -> t.Dict[str, t.Any]: 58 """Finds all global references and closures in a function and nested functions. 59 60 This function treats closures as global variables, which could cause problems in the future. 61 62 Args: 63 func: The function to introspect 64 65 Returns: 66 A dictionary of all global references. 67 """ 68 variables = {} 69 70 if hasattr(func, "__code__"): 71 code = func.__code__ 72 73 for var in list(_code_globals(code)) + decorators(func): 74 if var in func.__globals__: 75 ref = func.__globals__[var] 76 variables[var] = ref 77 78 if func.__closure__: 79 for var, value in zip(code.co_freevars, func.__closure__): 80 variables[var] = value.cell_contents 81 82 return variables 83 84 85class ClassFoundException(Exception): 86 pass 87 88 89class _ClassFinder(ast.NodeVisitor): 90 def __init__(self, qualname: str) -> None: 91 self.stack: t.List[str] = [] 92 self.qualname = qualname 93 94 def visit_FunctionDef(self, node: ast.FunctionDef) -> None: 95 self.stack.append(node.name) 96 self.stack.append("<locals>") 97 self.generic_visit(node) 98 self.stack.pop() 99 self.stack.pop() 100 101 visit_AsyncFunctionDef = visit_FunctionDef # type: ignore 102 103 def visit_ClassDef(self, node: ast.ClassDef) -> None: 104 self.stack.append(node.name) 105 if self.qualname == ".".join(self.stack): 106 # Return the decorator for the class if present 107 if node.decorator_list: 108 line_number = node.decorator_list[0].lineno 109 else: 110 line_number = node.lineno 111 112 # decrement by one since lines starts with indexing by zero 113 line_number -= 1 114 raise ClassFoundException(line_number) 115 self.generic_visit(node) 116 self.stack.pop() 117 118 119def getsource(obj: t.Any) -> str: 120 """Get the source of a function or class. 121 122 inspect.getsource doesn't find decorators in python < 3.9 123 https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade 124 """ 125 path = inspect.getsourcefile(obj) 126 if path: 127 module = inspect.getmodule(obj, path) 128 129 if module: 130 lines = linecache.getlines(path, module.__dict__) 131 else: 132 lines = linecache.getlines(path) 133 134 def join_source(lnum: int) -> str: 135 return "".join(inspect.getblock(lines[lnum:])) 136 137 if inspect.isclass(obj): 138 qualname = obj.__qualname__ 139 source = "".join(lines) 140 tree = ast.parse(source) 141 class_finder = _ClassFinder(qualname) 142 try: 143 class_finder.visit(tree) 144 except ClassFoundException as e: 145 return join_source(e.args[0]) 146 elif inspect.isfunction(obj): 147 obj = obj.__code__ 148 if hasattr(obj, "co_firstlineno"): 149 lnum = obj.co_firstlineno - 1 150 pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)") 151 while lnum > 0: 152 try: 153 line = lines[lnum] 154 except IndexError: 155 raise OSError("lineno is out of bounds") 156 if pat.match(line): 157 break 158 lnum = lnum - 1 159 return join_source(lnum) 160 raise SQLMeshError(f"Cannot find source for {obj}") 161 162 163def parse_source(func: t.Callable) -> ast.Module: 164 """Parse a function and returns an ast node.""" 165 return ast.parse(textwrap.dedent(getsource(func))) 166 167 168def _decorator_name(decorator: ast.expr) -> str: 169 if isinstance(decorator, ast.Call): 170 return decorator.func.id # type: ignore 171 if isinstance(decorator, ast.Name): 172 return decorator.id 173 return "" 174 175 176def decorators(func: t.Callable) -> t.List[str]: 177 """Finds a list of all the decorators of a callable.""" 178 root_node = parse_source(func) 179 decorators = [] 180 181 for node in ast.walk(root_node): 182 if isinstance(node, (ast.FunctionDef, ast.ClassDef)): 183 for decorator in node.decorator_list: 184 name = _decorator_name(decorator) 185 if name not in IGNORE_DECORATORS: 186 decorators.append(name) 187 return unique(decorators) 188 189 190def normalize_source(obj: t.Any) -> str: 191 """Rewrites an object's source with formatting and doc strings removed by using Python ast. 192 193 Args: 194 obj: The object to fetch source from and convert to a string. 195 196 Returns: 197 A string representation of the normalized function. 198 """ 199 root_node = parse_source(obj) 200 201 for node in ast.walk(root_node): 202 if isinstance(node, (ast.FunctionDef, ast.ClassDef)): 203 for decorator in node.decorator_list: 204 if _decorator_name(decorator) in IGNORE_DECORATORS: 205 node.decorator_list.remove(decorator) 206 207 # remove docstrings 208 body = node.body 209 if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str): 210 node.body = body[1:] 211 212 # remove function return type annotation 213 if isinstance(node, ast.FunctionDef): 214 node.returns = None 215 elif isinstance(node, ast.arg): 216 node.annotation = None 217 218 return to_source(root_node).strip() 219 220 221def build_env( 222 obj: t.Any, 223 *, 224 env: t.Dict[str, t.Any], 225 name: str, 226 path: Path, 227) -> None: 228 """Fills in env dictionary with all globals needed to execute the object. 229 230 Recursively traverse classes and functions. 231 232 Args: 233 obj: Any python object. 234 env: Dictionary to store the env. 235 name: Name of the object in the env. 236 path: The module path to serialize. Other modules will not be walked and treated as imports. 237 """ 238 239 obj_module = inspect.getmodule(obj) 240 241 if obj_module and obj_module.__name__ == "builtins": 242 return 243 244 def walk(obj: t.Any) -> None: 245 if inspect.isclass(obj): 246 for decorator in decorators(obj): 247 if obj_module and decorator in obj_module.__dict__: 248 build_env( 249 obj_module.__dict__[decorator], 250 env=env, 251 name=decorator, 252 path=path, 253 ) 254 255 for base in obj.__bases__: 256 build_env(base, env=env, name=base.__qualname__, path=path) 257 258 for k, v in obj.__dict__.items(): 259 if k.startswith("__"): 260 continue 261 # traverse methods in a class to find global references 262 if isinstance(v, (classmethod, staticmethod)): 263 v = v.__func__ 264 if callable(v): 265 # if the method is a part of the object, walk it 266 # else it is a global function and we just store it 267 if v.__qualname__.startswith(obj.__qualname__): 268 walk(v) 269 else: 270 build_env(v, env=env, name=v.__name__, path=path) 271 elif callable(obj): 272 for k, v in func_globals(obj).items(): 273 build_env(v, env=env, name=k, path=path) 274 275 if name not in env: 276 env[name] = obj 277 if obj_module and _is_relative_to(obj_module.__file__, path): 278 walk(obj) 279 elif env[name] != obj: 280 raise SQLMeshError( 281 f"Cannot store {obj} in environment, duplicate definitions found for '{name}'" 282 ) 283 284 285class ExecutableKind(str, Enum): 286 """The kind of of executable. The order of the members is used when serializing the python model to text.""" 287 288 IMPORT = "import" 289 VALUE = "value" 290 DEFINITION = "definition" 291 STATEMENT = "statement" 292 293 def __lt__(self, other: t.Any) -> bool: 294 if not isinstance(other, ExecutableKind): 295 return NotImplemented 296 values = list(ExecutableKind.__dict__.values()) 297 return values.index(self) < values.index(other) 298 299 300class Executable(PydanticModel): 301 payload: t.Any 302 kind: ExecutableKind = ExecutableKind.DEFINITION 303 name: t.Optional[str] = None 304 path: t.Optional[str] = None 305 alias: t.Optional[str] = None 306 307 @property 308 def is_definition(self) -> bool: 309 return self.kind == ExecutableKind.DEFINITION 310 311 @property 312 def is_import(self) -> bool: 313 return self.kind == ExecutableKind.IMPORT 314 315 @property 316 def is_statement(self) -> bool: 317 return self.kind == ExecutableKind.STATEMENT 318 319 @property 320 def is_value(self) -> bool: 321 return self.kind == ExecutableKind.VALUE 322 323 324def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]: 325 """Serializes a python function into a self contained dictionary. 326 327 Recursively walks a function's globals to store all other references inside of env. 328 329 Args: 330 env: Dictionary to store the env. 331 path: The root path to seralize. Other modules will not be walked and treated as imports. 332 """ 333 serialized = {} 334 335 for k, v in env.items(): 336 if callable(v): 337 name = v.__name__ 338 name = k if name == "<lambda>" else name 339 file_path = Path(inspect.getfile(v)) 340 341 if _is_relative_to(file_path, path): 342 serialized[k] = Executable( 343 name=name, 344 payload=normalize_source(v), 345 kind=ExecutableKind.DEFINITION, 346 path=str(file_path.relative_to(path.absolute())), 347 alias=k if name != k else None, 348 ) 349 else: 350 serialized[k] = Executable( 351 payload=f"from {v.__module__} import {name}", 352 kind=ExecutableKind.IMPORT, 353 ) 354 elif inspect.ismodule(v): 355 name = v.__name__ 356 if _is_relative_to(v.__file__, path): 357 raise SQLMeshError( 358 f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead." 359 ) 360 postfix = "" if name == k else f" as {k}" 361 serialized[k] = Executable( 362 payload=f"import {name}{postfix}", 363 kind=ExecutableKind.IMPORT, 364 ) 365 else: 366 serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE) 367 368 return serialized 369 370 371def prepare_env( 372 python_env: t.Dict[str, Executable], 373 env: t.Optional[t.Dict[str, t.Any]] = None, 374) -> t.Dict[str, t.Any]: 375 """Prepare a python env by hydrating and executing functions. 376 377 The Python ENV is stored in a json serializable format. 378 Functions and imports are stored as a special data class. 379 380 Args: 381 python_env: The dictionary containing the serialized python environment. 382 env: The dictionary to execute code in. 383 384 Returns: 385 The prepared environment with hydrated functions. 386 """ 387 env = {} if env is None else env 388 389 for name, executable in sorted( 390 python_env.items(), key=lambda item: 0 if item[1].is_import else 1 391 ): 392 if executable.is_value: 393 env[name] = ast.literal_eval(executable.payload) 394 else: 395 exec(executable.payload, env) 396 if executable.alias and executable.name: 397 env[executable.alias] = env[executable.name] 398 return env 399 400 401def print_exception( 402 exception: Exception, 403 python_env: t.Dict[str, Executable], 404 out: t.TextIO = sys.stderr, 405) -> None: 406 """Formats exceptions that occur from evaled code. 407 408 Stack traces generated by evaled code lose code context and are difficult to debug. 409 This intercepts the default stack trace and tries to make it debuggable. 410 411 Args: 412 exception: The exception to print the stack trace for. 413 python_env: The environment containing stringified python code. 414 out: The output stream to write to. 415 """ 416 tb: t.List[str] = [] 417 418 for error_line in format_exception(exception): 419 match = re.search(f'File "<string>", line (.*), in (.*)', error_line) 420 421 if not match: 422 tb.append(error_line) 423 continue 424 425 line_num = int(match.group(1)) 426 func = match.group(2) 427 428 if func not in python_env: 429 tb.append(error_line) 430 continue 431 432 executable = python_env[func] 433 indent = error_line[: match.start()] 434 435 error_line = ( 436 f"{indent}File '{executable.path}' (or imported file), line {line_num}, in {func}" 437 ) 438 439 code = executable.payload 440 formatted = [] 441 442 for i, code_line in enumerate(code.splitlines()): 443 if i < line_num: 444 pad = len(code_line) - len(code_line.lstrip()) 445 if i + 1 == line_num: 446 formatted.append(f"{code_line[:pad]}{code_line[pad:]}") 447 else: 448 formatted.append(code_line) 449 450 tb.extend( 451 ( 452 error_line, 453 textwrap.indent( 454 os.linesep.join(formatted), 455 indent + " ", 456 ), 457 os.linesep, 458 ) 459 ) 460 461 out.write(os.linesep.join(tb)) 462 463 464def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType: 465 module_name = str( 466 path.absolute().relative_to(relative_base.absolute()).with_suffix("") 467 ).replace(os.path.sep, ".") 468 # remove the entire module hierarchy in case they were already loaded 469 parts = module_name.split(".") 470 for i in range(len(parts)): 471 sys.modules.pop(".".join(parts[0 : i + 1]), None) 472 473 return importlib.import_module(module_name)
58def func_globals(func: t.Callable) -> t.Dict[str, t.Any]: 59 """Finds all global references and closures in a function and nested functions. 60 61 This function treats closures as global variables, which could cause problems in the future. 62 63 Args: 64 func: The function to introspect 65 66 Returns: 67 A dictionary of all global references. 68 """ 69 variables = {} 70 71 if hasattr(func, "__code__"): 72 code = func.__code__ 73 74 for var in list(_code_globals(code)) + decorators(func): 75 if var in func.__globals__: 76 ref = func.__globals__[var] 77 variables[var] = ref 78 79 if func.__closure__: 80 for var, value in zip(code.co_freevars, func.__closure__): 81 variables[var] = value.cell_contents 82 83 return variables
Finds all global references and closures in a function and nested functions.
This function treats closures as global variables, which could cause problems in the future.
Arguments:
- func: The function to introspect
Returns:
A dictionary of all global references.
Common base class for all non-exit exceptions.
Inherited Members
- builtins.Exception
- Exception
- builtins.BaseException
- with_traceback
120def getsource(obj: t.Any) -> str: 121 """Get the source of a function or class. 122 123 inspect.getsource doesn't find decorators in python < 3.9 124 https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade 125 """ 126 path = inspect.getsourcefile(obj) 127 if path: 128 module = inspect.getmodule(obj, path) 129 130 if module: 131 lines = linecache.getlines(path, module.__dict__) 132 else: 133 lines = linecache.getlines(path) 134 135 def join_source(lnum: int) -> str: 136 return "".join(inspect.getblock(lines[lnum:])) 137 138 if inspect.isclass(obj): 139 qualname = obj.__qualname__ 140 source = "".join(lines) 141 tree = ast.parse(source) 142 class_finder = _ClassFinder(qualname) 143 try: 144 class_finder.visit(tree) 145 except ClassFoundException as e: 146 return join_source(e.args[0]) 147 elif inspect.isfunction(obj): 148 obj = obj.__code__ 149 if hasattr(obj, "co_firstlineno"): 150 lnum = obj.co_firstlineno - 1 151 pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)") 152 while lnum > 0: 153 try: 154 line = lines[lnum] 155 except IndexError: 156 raise OSError("lineno is out of bounds") 157 if pat.match(line): 158 break 159 lnum = lnum - 1 160 return join_source(lnum) 161 raise SQLMeshError(f"Cannot find source for {obj}")
Get the source of a function or class.
inspect.getsource doesn't find decorators in python < 3.9 https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade
164def parse_source(func: t.Callable) -> ast.Module: 165 """Parse a function and returns an ast node.""" 166 return ast.parse(textwrap.dedent(getsource(func)))
Parse a function and returns an ast node.
177def decorators(func: t.Callable) -> t.List[str]: 178 """Finds a list of all the decorators of a callable.""" 179 root_node = parse_source(func) 180 decorators = [] 181 182 for node in ast.walk(root_node): 183 if isinstance(node, (ast.FunctionDef, ast.ClassDef)): 184 for decorator in node.decorator_list: 185 name = _decorator_name(decorator) 186 if name not in IGNORE_DECORATORS: 187 decorators.append(name) 188 return unique(decorators)
Finds a list of all the decorators of a callable.
191def normalize_source(obj: t.Any) -> str: 192 """Rewrites an object's source with formatting and doc strings removed by using Python ast. 193 194 Args: 195 obj: The object to fetch source from and convert to a string. 196 197 Returns: 198 A string representation of the normalized function. 199 """ 200 root_node = parse_source(obj) 201 202 for node in ast.walk(root_node): 203 if isinstance(node, (ast.FunctionDef, ast.ClassDef)): 204 for decorator in node.decorator_list: 205 if _decorator_name(decorator) in IGNORE_DECORATORS: 206 node.decorator_list.remove(decorator) 207 208 # remove docstrings 209 body = node.body 210 if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str): 211 node.body = body[1:] 212 213 # remove function return type annotation 214 if isinstance(node, ast.FunctionDef): 215 node.returns = None 216 elif isinstance(node, ast.arg): 217 node.annotation = None 218 219 return to_source(root_node).strip()
Rewrites an object's source with formatting and doc strings removed by using Python ast.
Arguments:
- obj: The object to fetch source from and convert to a string.
Returns:
A string representation of the normalized function.
222def build_env( 223 obj: t.Any, 224 *, 225 env: t.Dict[str, t.Any], 226 name: str, 227 path: Path, 228) -> None: 229 """Fills in env dictionary with all globals needed to execute the object. 230 231 Recursively traverse classes and functions. 232 233 Args: 234 obj: Any python object. 235 env: Dictionary to store the env. 236 name: Name of the object in the env. 237 path: The module path to serialize. Other modules will not be walked and treated as imports. 238 """ 239 240 obj_module = inspect.getmodule(obj) 241 242 if obj_module and obj_module.__name__ == "builtins": 243 return 244 245 def walk(obj: t.Any) -> None: 246 if inspect.isclass(obj): 247 for decorator in decorators(obj): 248 if obj_module and decorator in obj_module.__dict__: 249 build_env( 250 obj_module.__dict__[decorator], 251 env=env, 252 name=decorator, 253 path=path, 254 ) 255 256 for base in obj.__bases__: 257 build_env(base, env=env, name=base.__qualname__, path=path) 258 259 for k, v in obj.__dict__.items(): 260 if k.startswith("__"): 261 continue 262 # traverse methods in a class to find global references 263 if isinstance(v, (classmethod, staticmethod)): 264 v = v.__func__ 265 if callable(v): 266 # if the method is a part of the object, walk it 267 # else it is a global function and we just store it 268 if v.__qualname__.startswith(obj.__qualname__): 269 walk(v) 270 else: 271 build_env(v, env=env, name=v.__name__, path=path) 272 elif callable(obj): 273 for k, v in func_globals(obj).items(): 274 build_env(v, env=env, name=k, path=path) 275 276 if name not in env: 277 env[name] = obj 278 if obj_module and _is_relative_to(obj_module.__file__, path): 279 walk(obj) 280 elif env[name] != obj: 281 raise SQLMeshError( 282 f"Cannot store {obj} in environment, duplicate definitions found for '{name}'" 283 )
Fills in env dictionary with all globals needed to execute the object.
Recursively traverse classes and functions.
Arguments:
- obj: Any python object.
- env: Dictionary to store the env.
- name: Name of the object in the env.
- path: The module path to serialize. Other modules will not be walked and treated as imports.
286class ExecutableKind(str, Enum): 287 """The kind of of executable. The order of the members is used when serializing the python model to text.""" 288 289 IMPORT = "import" 290 VALUE = "value" 291 DEFINITION = "definition" 292 STATEMENT = "statement" 293 294 def __lt__(self, other: t.Any) -> bool: 295 if not isinstance(other, ExecutableKind): 296 return NotImplemented 297 values = list(ExecutableKind.__dict__.values()) 298 return values.index(self) < values.index(other)
The kind of of executable. The order of the members is used when serializing the python model to text.
Inherited Members
- enum.Enum
- name
- value
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
301class Executable(PydanticModel): 302 payload: t.Any 303 kind: ExecutableKind = ExecutableKind.DEFINITION 304 name: t.Optional[str] = None 305 path: t.Optional[str] = None 306 alias: t.Optional[str] = None 307 308 @property 309 def is_definition(self) -> bool: 310 return self.kind == ExecutableKind.DEFINITION 311 312 @property 313 def is_import(self) -> bool: 314 return self.kind == ExecutableKind.IMPORT 315 316 @property 317 def is_statement(self) -> bool: 318 return self.kind == ExecutableKind.STATEMENT 319 320 @property 321 def is_value(self) -> bool: 322 return self.kind == ExecutableKind.VALUE
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
325def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]: 326 """Serializes a python function into a self contained dictionary. 327 328 Recursively walks a function's globals to store all other references inside of env. 329 330 Args: 331 env: Dictionary to store the env. 332 path: The root path to seralize. Other modules will not be walked and treated as imports. 333 """ 334 serialized = {} 335 336 for k, v in env.items(): 337 if callable(v): 338 name = v.__name__ 339 name = k if name == "<lambda>" else name 340 file_path = Path(inspect.getfile(v)) 341 342 if _is_relative_to(file_path, path): 343 serialized[k] = Executable( 344 name=name, 345 payload=normalize_source(v), 346 kind=ExecutableKind.DEFINITION, 347 path=str(file_path.relative_to(path.absolute())), 348 alias=k if name != k else None, 349 ) 350 else: 351 serialized[k] = Executable( 352 payload=f"from {v.__module__} import {name}", 353 kind=ExecutableKind.IMPORT, 354 ) 355 elif inspect.ismodule(v): 356 name = v.__name__ 357 if _is_relative_to(v.__file__, path): 358 raise SQLMeshError( 359 f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead." 360 ) 361 postfix = "" if name == k else f" as {k}" 362 serialized[k] = Executable( 363 payload=f"import {name}{postfix}", 364 kind=ExecutableKind.IMPORT, 365 ) 366 else: 367 serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE) 368 369 return serialized
Serializes a python function into a self contained dictionary.
Recursively walks a function's globals to store all other references inside of env.
Arguments:
- env: Dictionary to store the env.
- path: The root path to seralize. Other modules will not be walked and treated as imports.
372def prepare_env( 373 python_env: t.Dict[str, Executable], 374 env: t.Optional[t.Dict[str, t.Any]] = None, 375) -> t.Dict[str, t.Any]: 376 """Prepare a python env by hydrating and executing functions. 377 378 The Python ENV is stored in a json serializable format. 379 Functions and imports are stored as a special data class. 380 381 Args: 382 python_env: The dictionary containing the serialized python environment. 383 env: The dictionary to execute code in. 384 385 Returns: 386 The prepared environment with hydrated functions. 387 """ 388 env = {} if env is None else env 389 390 for name, executable in sorted( 391 python_env.items(), key=lambda item: 0 if item[1].is_import else 1 392 ): 393 if executable.is_value: 394 env[name] = ast.literal_eval(executable.payload) 395 else: 396 exec(executable.payload, env) 397 if executable.alias and executable.name: 398 env[executable.alias] = env[executable.name] 399 return env
Prepare a python env by hydrating and executing functions.
The Python ENV is stored in a json serializable format. Functions and imports are stored as a special data class.
Arguments:
- python_env: The dictionary containing the serialized python environment.
- env: The dictionary to execute code in.
Returns:
The prepared environment with hydrated functions.
402def print_exception( 403 exception: Exception, 404 python_env: t.Dict[str, Executable], 405 out: t.TextIO = sys.stderr, 406) -> None: 407 """Formats exceptions that occur from evaled code. 408 409 Stack traces generated by evaled code lose code context and are difficult to debug. 410 This intercepts the default stack trace and tries to make it debuggable. 411 412 Args: 413 exception: The exception to print the stack trace for. 414 python_env: The environment containing stringified python code. 415 out: The output stream to write to. 416 """ 417 tb: t.List[str] = [] 418 419 for error_line in format_exception(exception): 420 match = re.search(f'File "<string>", line (.*), in (.*)', error_line) 421 422 if not match: 423 tb.append(error_line) 424 continue 425 426 line_num = int(match.group(1)) 427 func = match.group(2) 428 429 if func not in python_env: 430 tb.append(error_line) 431 continue 432 433 executable = python_env[func] 434 indent = error_line[: match.start()] 435 436 error_line = ( 437 f"{indent}File '{executable.path}' (or imported file), line {line_num}, in {func}" 438 ) 439 440 code = executable.payload 441 formatted = [] 442 443 for i, code_line in enumerate(code.splitlines()): 444 if i < line_num: 445 pad = len(code_line) - len(code_line.lstrip()) 446 if i + 1 == line_num: 447 formatted.append(f"{code_line[:pad]}{code_line[pad:]}") 448 else: 449 formatted.append(code_line) 450 451 tb.extend( 452 ( 453 error_line, 454 textwrap.indent( 455 os.linesep.join(formatted), 456 indent + " ", 457 ), 458 os.linesep, 459 ) 460 ) 461 462 out.write(os.linesep.join(tb))
Formats exceptions that occur from evaled code.
Stack traces generated by evaled code lose code context and are difficult to debug. This intercepts the default stack trace and tries to make it debuggable.
Arguments:
- exception: The exception to print the stack trace for.
- python_env: The environment containing stringified python code.
- out: The output stream to write to.
465def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType: 466 module_name = str( 467 path.absolute().relative_to(relative_base.absolute()).with_suffix("") 468 ).replace(os.path.sep, ".") 469 # remove the entire module hierarchy in case they were already loaded 470 parts = module_name.split(".") 471 for i in range(len(parts)): 472 sys.modules.pop(".".join(parts[0 : i + 1]), None) 473 474 return importlib.import_module(module_name)