Edit on GitHub

sqlmesh.core.macros

  1from __future__ import annotations
  2
  3import typing as t
  4from functools import reduce
  5from string import Template
  6
  7import sqlglot
  8from jinja2 import Environment
  9from sqlglot import Generator, exp
 10from sqlglot.executor.env import ENV
 11from sqlglot.executor.python import Python
 12from sqlglot.helper import csv, ensure_collection
 13
 14from sqlmesh.core.dialect import (
 15    Jinja,
 16    MacroDef,
 17    MacroFunc,
 18    MacroSQL,
 19    MacroStrReplace,
 20    MacroVar,
 21)
 22from sqlmesh.utils import UniqueKeyDict, registry_decorator
 23from sqlmesh.utils.errors import MacroEvalError, SQLMeshError
 24from sqlmesh.utils.jinja import JinjaMacroRegistry
 25from sqlmesh.utils.metaprogramming import Executable, prepare_env, print_exception
 26
 27
 28class MacroStrTemplate(Template):
 29    delimiter = "@"
 30
 31
 32EXPRESSIONS_NAME_MAP = {}
 33
 34for klass in sqlglot.Parser.EXPRESSION_PARSERS:
 35    name = klass if isinstance(klass, str) else klass.__name__  # type: ignore
 36    EXPRESSIONS_NAME_MAP[name.lower()] = name
 37
 38
 39def _macro_sql(sql: str, into: t.Optional[str] = None) -> str:
 40    args = [_macro_str_replace(sql)]
 41    if into in EXPRESSIONS_NAME_MAP:
 42        args.append(f"into=exp.{EXPRESSIONS_NAME_MAP[into]}")
 43    return f"self.parse_one({', '.join(args)})"
 44
 45
 46def _macro_func_sql(self: Generator, e: exp.Expression) -> str:
 47    func = e.this
 48
 49    if isinstance(func, exp.Anonymous):
 50        return f"""self.send({csv("'" + func.name + "'", self.expressions(func))})"""
 51    return self.sql(func)
 52
 53
 54def _macro_str_replace(text: str) -> str:
 55    """Stringifies python code for variable replacement
 56    Args:
 57        text: text string
 58    Returns:
 59        Stringified python code to execute variable replacement
 60    """
 61    return f"self.template({text}, locals())"
 62
 63
 64class MacroDialect(Python):
 65    class Generator(Python.Generator):
 66        TRANSFORMS = {
 67            **Python.Generator.TRANSFORMS,  # type: ignore
 68            exp.Column: lambda self, e: f"exp.to_column('{self.sql(e, 'this')}')",
 69            exp.Lambda: lambda self, e: f"lambda {self.expressions(e)}: {self.sql(e, 'this')}",
 70            MacroFunc: _macro_func_sql,
 71            MacroSQL: lambda self, e: _macro_sql(self.sql(e, "this"), e.args.get("into")),
 72            MacroStrReplace: lambda self, e: _macro_str_replace(self.sql(e, "this")),
 73        }
 74
 75
 76class MacroEvaluator:
 77    """The class responsible for evaluating SQLMesh Macros/SQL.
 78
 79    SQLMesh supports special preprocessed SQL prefixed with `@`. Although it provides similar power to
 80    traditional methods like string templating, there is semantic understanding of SQL which prevents
 81    common errors like leading/trailing commas, syntax errors, etc.
 82
 83    SQLMesh SQL allows for macro variables and macro functions. Macro variables take the form of @variable. These are used for variable substitution.
 84
 85    SELECT * FROM foo WHERE ds BETWEEN @start_date AND @end_date
 86
 87    Macro variables can be defined with a special macro function.
 88
 89    @DEF(start_date, '2021-01-01')
 90
 91    Args:
 92        dialect: Dialect of the SQL to evaluate.
 93        python_env: Serialized Python environment.
 94    """
 95
 96    def __init__(
 97        self,
 98        dialect: str = "",
 99        python_env: t.Optional[t.Dict[str, Executable]] = None,
100        jinja_env: t.Optional[Environment] = None,
101    ):
102        self.dialect = dialect
103        self.generator = MacroDialect().generator()
104        self.locals: t.Dict[str, t.Any] = {}
105        self.env = {**ENV, "self": self}
106        self.python_env = python_env or {}
107        self._jinja_env: t.Optional[Environment] = jinja_env
108        self.macros = {normalize_macro_name(k): v.func for k, v in macro.get_registry().items()}
109        prepare_env(self.python_env, self.env)
110        for k, v in self.python_env.items():
111            if v.is_definition:
112                self.macros[normalize_macro_name(k)] = self.env[v.name or k]
113
114    def send(
115        self, name: str, *args: t.Any
116    ) -> t.Union[None, exp.Expression, t.List[exp.Expression]]:
117        func = self.macros.get(normalize_macro_name(name))
118
119        if not callable(func):
120            raise SQLMeshError(f"Macro '{name}' does not exist.")
121
122        try:
123            return func(self, *args)
124        except Exception as e:
125            print_exception(e, self.python_env)
126            raise MacroEvalError(f"Error trying to eval macro.") from e
127
128    def transform(self, query: exp.Expression) -> exp.Expression | t.List[exp.Expression] | None:
129        def _transform_node(node: exp.Expression) -> exp.Expression:
130            if isinstance(node, MacroVar):
131                return exp.convert(_norm_env_value(self.locals[node.name]))
132            elif node.is_string:
133                node.set("this", self.jinja_env.from_string(node.this).render())
134                return node
135            else:
136                return node
137
138        query = query.transform(_transform_node)
139
140        def evaluate_macros(
141            node: exp.Expression,
142        ) -> exp.Expression | t.List[exp.Expression] | None:
143            exp.replace_children(
144                node, lambda n: n if isinstance(n, exp.Lambda) else evaluate_macros(n)
145            )
146            if isinstance(node, MacroFunc):
147                return self.evaluate(node)
148            return node
149
150        transformed = evaluate_macros(query)
151
152        if isinstance(transformed, list):
153            return [self.parse_one(node.sql(dialect=self.dialect)) for node in transformed]
154        elif isinstance(transformed, Jinja):
155            return transformed
156        elif isinstance(transformed, exp.Expression):
157            return self.parse_one(transformed.sql(dialect=self.dialect))
158
159        return transformed
160
161    def template(self, text: t.Any, local_variables: t.Dict[str, t.Any]) -> str:
162        """Substitute @vars with locals.
163
164        Args:
165            text: The string to do substitition on.
166            local_variables: Local variables in the context so that lambdas can be used.
167
168        Returns:
169           The rendered string.
170        """
171        return MacroStrTemplate(str(text)).safe_substitute(self.locals, **local_variables)
172
173    def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
174        if isinstance(node, MacroDef):
175            self.locals[node.name] = node.expression
176            return node
177
178        if isinstance(node, MacroSQL) or not node.find(exp.Column, exp.Table):
179            result = self.eval_expression(node)
180        else:
181            func = node.this
182            result = self.send(func.name, *func.expressions)
183
184        if result is None:
185            return None
186
187        if isinstance(result, list):
188            return [exp.convert(item) for item in result if item is not None]
189        return exp.convert(result)
190
191    def eval_expression(self, node: exp.Expression) -> t.Any:
192        """Converts a SQLGlot expression into executable Python code and evals it.
193
194        Args:
195            node: expression
196        Returns:
197            The return value of the evaled Python Code.
198        """
199        code = node.sql()
200        try:
201            code = self.generator.generate(node)
202            return eval(code, self.env, self.locals)
203        except Exception as e:
204            print_exception(e, self.python_env)
205            raise MacroEvalError(
206                f"Error trying to eval macro.\n\nGenerated code: {code}\n\nOriginal sql: {node}"
207            ) from e
208
209    def parse_one(
210        self, sql: str, into: t.Optional[exp.IntoType] = None, **opts: t.Any
211    ) -> exp.Expression:
212        """Parses the given SQL string and returns a syntax tree for the first
213        parsed SQL statement.
214
215        Args:
216            sql (str): the SQL code string to parse.
217            into (Expression): the Expression to parse into
218            **opts: other options
219
220        Returns:
221            Expression: the syntax tree for the first parsed statement
222        """
223        return sqlglot.maybe_parse(sql, dialect=self.dialect, into=into, **opts)
224
225    @property
226    def jinja_env(self) -> Environment:
227        if not self._jinja_env:
228            jinja_env_methods = {**self.locals, **self.env}
229            del jinja_env_methods["self"]
230            self._jinja_env = JinjaMacroRegistry().build_environment(**jinja_env_methods)
231        return self._jinja_env
232
233
234class macro(registry_decorator):
235    """Specifies a function is a macro and registers it the global MACROS registry.
236
237    Registered macros can be referenced in SQL statements to make queries more dynamic/cleaner.
238
239    Example:
240        from typing import t
241        from sqlglot import exp
242        from sqlmesh.core.macros import MacroEvaluator, macro
243
244        @macro()
245        def add_one(evaluator: MacroEvaluator, column: str) -> exp.Add:
246            return evaluator.parse_one(f"{column} + 1")
247
248    Args:
249        name: A custom name for the macro, the default is the name of the function.
250    """
251
252    registry_name = "macros"
253
254
255ExecutableOrMacro = t.Union[Executable, macro]
256MacroRegistry = UniqueKeyDict[str, ExecutableOrMacro]
257
258
259def _norm_var_arg_lambda(
260    evaluator: MacroEvaluator, func: exp.Lambda, *items: t.Any
261) -> t.Tuple[t.Iterable, t.Callable]:
262    """
263    Converts sql literal array and lambda into actual python iterable + callable.
264
265    In order to support expressions like @EACH([a, b, c], x -> @SQL('@x')), the lambda var x
266    needs be passed to the local state.
267
268    Args:
269        evaluator: MacroEvaluator that invoked the macro
270        func: Lambda SQLGlot expression.
271        items: Array or items of SQLGlot expressions.
272    """
273
274    def substitute(
275        node: exp.Expression, arg_index: t.Dict[str, int], *items: exp.Expression
276    ) -> exp.Expression | t.List[exp.Expression] | None:
277        if (
278            isinstance(node, exp.Identifier)
279            and node.name in arg_index
280            and not isinstance(node.parent, exp.Column)
281        ):
282            return items[arg_index[node.name]].copy()
283        if isinstance(node, MacroFunc):
284            local_copy = evaluator.locals.copy()
285            for k, v in arg_index.items():
286                evaluator.locals[k] = items[v]
287            result = evaluator.transform(node)
288            evaluator.locals = local_copy
289            return result
290        return node
291
292    if len(items) == 1:
293        item = items[0]
294        expressions = item.expressions if isinstance(item, (exp.Array, exp.Tuple)) else item
295    else:
296        expressions = items
297
298    if not callable(func):
299        arg_index = {expression.name: i for i, expression in enumerate(func.expressions)}
300        body = func.this
301        return expressions, lambda *x: body.transform(substitute, arg_index, *x)
302
303    return expressions, func
304
305
306def _norm_env_value(value: t.Any) -> t.Any:
307    if isinstance(value, list):
308        return tuple(value)
309    if isinstance(value, exp.Array):
310        return exp.Tuple(expressions=value.expressions)
311    return value
312
313
314@macro()
315def each(
316    evaluator: MacroEvaluator,
317    *args: t.Any,
318) -> t.List[t.Any]:
319    """Iterates through items calling func on each.
320
321    If a func call on item returns None, it will be excluded from the list.
322
323    Args:
324        evaluator: MacroEvaluator that invoked the macro
325        args: The last argument should be a lambda of the form x -> x +1. The first argument can be
326            an Array or var args can be used.
327
328    Returns:
329        A list of items that is the result of func
330    """
331    *items, func = args
332    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
333    return [item for item in map(func, ensure_collection(items)) if item is not None]
334
335
336@macro("REDUCE")
337def reduce_(evaluator: MacroEvaluator, *args: t.Any) -> t.Any:
338    """Iterates through items applying provided function that takes two arguments
339    cumulatively to the items of iterable items, from left to right, so as to reduce
340    the iterable to a single item.
341
342    Example:
343        >>> from sqlglot import parse_one
344        >>> from sqlmesh.core.macros import MacroEvaluator, reduce_
345        >>> sql = "@SQL(@REDUCE([100, 200, 300, 400], (x, y) -> x + y))"
346        >>> MacroEvaluator().transform(parse_one(sql)).sql()
347        '1000'
348
349    Args:
350        evaluator: MacroEvaluator that invoked the macro
351        args: The last argument should be a lambda of the form (x, y) -> x + y. The first argument can be
352            an Array or var args can be used.
353    Returns:
354        A single item that is the result of applying func cumulatively to items
355    """
356    *items, func = args
357    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
358    return reduce(func, ensure_collection(items))
359
360
361@macro("FILTER")
362def filter_(evaluator: MacroEvaluator, *args: t.Any) -> t.List[t.Any]:
363    """Iterates through items, applying provided function to each item and removing
364    all items where the function returns False
365
366    Example:
367        >>> from sqlglot import parse_one
368        >>> from sqlmesh.core.macros import MacroEvaluator, filter_
369        >>> sql = "@REDUCE(@FILTER([1, 2, 3], x -> x > 1), (x, y) -> x + y)"
370        >>> MacroEvaluator().transform(parse_one(sql)).sql()
371        '5'
372
373    Args:
374        evaluator: MacroEvaluator that invoked the macro
375        args: The last argument should be a lambda of the form x -> x > 1. The first argument can be
376            an Array or var args can be used.
377    Returns:
378        The items for which the func returned True
379    """
380    *items, func = args
381    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
382    return list(filter(func, items))
383
384
385@macro("WITH")
386def with_(
387    evaluator: MacroEvaluator,
388    condition: exp.Condition,
389    expression: exp.With,
390) -> t.Optional[exp.With]:
391    """Inserts WITH expression when the condition is True
392
393    Example:
394        >>> from sqlglot import parse_one
395        >>> from sqlmesh.core.macros import MacroEvaluator, with_
396        >>> sql = "@WITH(True) all_cities as (select * from city) select all_cities"
397        >>> MacroEvaluator().transform(parse_one(sql)).sql()
398        'WITH all_cities AS (SELECT * FROM city) SELECT all_cities'
399
400    Args:
401        evaluator: MacroEvaluator that invoked the macro
402        condition: Condition expression
403        expression: With expression
404    Returns:
405        With expression if the conditional is True; otherwise None
406    """
407    return expression if evaluator.eval_expression(condition) else None
408
409
410@macro()
411def join(
412    evaluator: MacroEvaluator,
413    condition: exp.Condition,
414    expression: exp.Join,
415) -> t.Optional[exp.Join]:
416    """Inserts JOIN expression when the condition is True
417
418    Example:
419        >>> from sqlglot import parse_one
420        >>> from sqlmesh.core.macros import MacroEvaluator, join
421        >>> sql = "select * from city @JOIN(True) country on city.country = country.name"
422        >>> MacroEvaluator().transform(parse_one(sql)).sql()
423        'SELECT * FROM city JOIN country ON city.country = country.name'
424
425        >>> sql = "select * from city left outer @JOIN(True) country on city.country = country.name"
426        >>> MacroEvaluator().transform(parse_one(sql)).sql()
427        'SELECT * FROM city LEFT OUTER JOIN country ON city.country = country.name'
428
429    Args:
430        evaluator: MacroEvaluator that invoked the macro
431        condition: Condition expression
432        expression: Join expression
433    Returns:
434        Join expression if the conditional is True; otherwise None
435    """
436    return expression if evaluator.eval_expression(condition) else None
437
438
439@macro()
440def where(
441    evaluator: MacroEvaluator,
442    condition: exp.Condition,
443    expression: exp.Where,
444) -> t.Optional[exp.Where]:
445    """Inserts WHERE expression when the condition is True
446
447    Example:
448        >>> from sqlglot import parse_one
449        >>> from sqlmesh.core.macros import MacroEvaluator, where
450        >>> sql = "select * from city @WHERE(True) population > 100 and country = 'Mexico'"
451        >>> MacroEvaluator().transform(parse_one(sql)).sql()
452        "SELECT * FROM city WHERE population > 100 AND country = 'Mexico'"
453
454    Args:
455        evaluator: MacroEvaluator that invoked the macro
456        condition: Condition expression
457        expression: Where expression
458    Returns:
459        Where expression if condition is True; otherwise None
460    """
461    return expression if evaluator.eval_expression(condition) else None
462
463
464@macro()
465def group_by(
466    evaluator: MacroEvaluator,
467    condition: exp.Condition,
468    expression: exp.Group,
469) -> t.Optional[exp.Group]:
470    """Inserts GROUP BY expression when the condition is True
471
472    Example:
473        >>> from sqlglot import parse_one
474        >>> from sqlmesh.core.macros import MacroEvaluator, group_by
475        >>> sql = "select * from city @GROUP_BY(True) country, population"
476        >>> MacroEvaluator().transform(parse_one(sql)).sql()
477        'SELECT * FROM city GROUP BY country, population'
478
479    Args:
480        evaluator: MacroEvaluator that invoked the macro
481        condition: Condition expression
482        expression: Group expression
483    Returns:
484        Group expression if the condition is True; otherwise None
485    """
486    return expression if evaluator.eval_expression(condition) else None
487
488
489@macro()
490def having(
491    evaluator: MacroEvaluator,
492    condition: exp.Condition,
493    expression: exp.Having,
494) -> t.Optional[exp.Having]:
495    """Inserts HAVING expression when the condition is True
496
497    Example:
498        >>> from sqlglot import parse_one
499        >>> from sqlmesh.core.macros import MacroEvaluator, having
500        >>> sql = "select * from city group by country @HAVING(True) population > 100 and country = 'Mexico'"
501        >>> MacroEvaluator().transform(parse_one(sql)).sql()
502        "SELECT * FROM city GROUP BY country HAVING population > 100 AND country = 'Mexico'"
503
504    Args:
505        evaluator: MacroEvaluator that invoked the macro
506        condition: Condition expression
507        expression: Having expression
508    Returns:
509        Having expression if the condition is True; otherwise None
510    """
511    return expression if evaluator.eval_expression(condition) else None
512
513
514@macro()
515def order_by(
516    evaluator: MacroEvaluator,
517    condition: exp.Condition,
518    expression: exp.Order,
519) -> t.Optional[exp.Order]:
520    """Inserts ORDER BY expression when the condition is True
521
522    Example:
523        >>> from sqlglot import parse_one
524        >>> from sqlmesh.core.macros import MacroEvaluator, order_by
525        >>> sql = "select * from city @ORDER_BY(True) population, name DESC"
526        >>> MacroEvaluator().transform(parse_one(sql)).sql()
527        'SELECT * FROM city ORDER BY population, name DESC'
528
529    Args:
530        evaluator: MacroEvaluator that invoked the macro
531        condition: Condition expression
532        expression: Order expression
533    Returns:
534        Order expression if the condition is True; otherwise None
535    """
536    return expression if evaluator.eval_expression(condition) else None
537
538
539def normalize_macro_name(name: str) -> str:
540    """Prefix macro name with @ and upcase"""
541    return f"@{name.upper()}"
class MacroStrTemplate(string.Template):
29class MacroStrTemplate(Template):
30    delimiter = "@"

A string class for supporting $-substitutions.

Inherited Members
string.Template
Template
flags
substitute
safe_substitute
class MacroDialect(sqlglot.executor.python.Python):
65class MacroDialect(Python):
66    class Generator(Python.Generator):
67        TRANSFORMS = {
68            **Python.Generator.TRANSFORMS,  # type: ignore
69            exp.Column: lambda self, e: f"exp.to_column('{self.sql(e, 'this')}')",
70            exp.Lambda: lambda self, e: f"lambda {self.expressions(e)}: {self.sql(e, 'this')}",
71            MacroFunc: _macro_func_sql,
72            MacroSQL: lambda self, e: _macro_sql(self.sql(e, "this"), e.args.get("into")),
73            MacroStrReplace: lambda self, e: _macro_str_replace(self.sql(e, "this")),
74        }
MacroDialect()
Inherited Members
sqlglot.executor.python.Python
Tokenizer
sqlglot.dialects.dialect.Dialect
get_or_raise
format_time
parse
parse_into
generate
transpile
tokenize
parser
generator
class MacroDialect.Generator(sqlglot.executor.python.Python.Generator):
66    class Generator(Python.Generator):
67        TRANSFORMS = {
68            **Python.Generator.TRANSFORMS,  # type: ignore
69            exp.Column: lambda self, e: f"exp.to_column('{self.sql(e, 'this')}')",
70            exp.Lambda: lambda self, e: f"lambda {self.expressions(e)}: {self.sql(e, 'this')}",
71            MacroFunc: _macro_func_sql,
72            MacroSQL: lambda self, e: _macro_sql(self.sql(e, "this"), e.args.get("into")),
73            MacroStrReplace: lambda self, e: _macro_str_replace(self.sql(e, "this")),
74        }

Generator interprets the given syntax tree and produces a SQL string as an output.

Arguments:
  • time_mapping (dict): the dictionary of custom time mappings in which the key represents a python time format and the output the target time format
  • time_trie (trie): a trie of the time_mapping keys
  • pretty (bool): if set to True the returned string will be formatted. Default: False.
  • quote_start (str): specifies which starting character to use to delimit quotes. Default: '.
  • quote_end (str): specifies which ending character to use to delimit quotes. Default: '.
  • identifier_start (str): specifies which starting character to use to delimit identifiers. Default: ".
  • identifier_end (str): specifies which ending character to use to delimit identifiers. Default: ".
  • identify (bool | str): 'always': always quote, 'safe': quote identifiers if they don't contain an upcase, True defaults to always.
  • normalize (bool): if set to True all identifiers will lower cased
  • string_escape (str): specifies a string escape character. Default: '.
  • identifier_escape (str): specifies an identifier escape character. Default: ".
  • pad (int): determines padding in a formatted string. Default: 2.
  • indent (int): determines the size of indentation in a formatted string. Default: 4.
  • unnest_column_only (bool): if true unnest table aliases are considered only as column aliases
  • normalize_functions (str): normalize function names, "upper", "lower", or None Default: "upper"
  • alias_post_tablesample (bool): if the table alias comes after tablesample Default: False
  • unsupported_level (ErrorLevel): determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
  • null_ordering (str): Indicates the default null ordering method to use if not explicitly set. Options are "nulls_are_small", "nulls_are_large", "nulls_are_last". Default: "nulls_are_small"
  • max_unsupported (int): Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
  • leading_comma (bool): if the the comma is leading or trailing in select statements Default: False
  • max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
  • comments: Whether or not to preserve comments in the output SQL code. Default: True
Inherited Members
sqlglot.generator.Generator
Generator
generate
unsupported
sep
seg
pad_comment
maybe_comment
wrap
no_identify
normalize_func
indent
sql
uncache_sql
cache_sql
characterset_sql
column_sql
columndef_sql
columnconstraint_sql
autoincrementcolumnconstraint_sql
compresscolumnconstraint_sql
generatedasidentitycolumnconstraint_sql
notnullcolumnconstraint_sql
primarykeycolumnconstraint_sql
uniquecolumnconstraint_sql
create_sql
describe_sql
prepend_ctes
with_sql
cte_sql
tablealias_sql
bitstring_sql
hexstring_sql
datatype_sql
directory_sql
delete_sql
drop_sql
except_sql
except_op
fetch_sql
filter_sql
hint_sql
index_sql
identifier_sql
national_sql
partition_sql
properties_sql
root_properties
properties
with_properties
locate_properties
property_sql
likeproperty_sql
fallbackproperty_sql
journalproperty_sql
freespaceproperty_sql
afterjournalproperty_sql
checksumproperty_sql
mergeblockratioproperty_sql
datablocksizeproperty_sql
blockcompressionproperty_sql
isolatedloadingproperty_sql
lockingproperty_sql
withdataproperty_sql
insert_sql
intersect_sql
intersect_op
introducer_sql
pseudotype_sql
returning_sql
rowformatdelimitedproperty_sql
table_sql
tablesample_sql
pivot_sql
tuple_sql
update_sql
values_sql
var_sql
into_sql
from_sql
group_sql
having_sql
join_sql
lambda_sql
lateral_sql
limit_sql
offset_sql
setitem_sql
set_sql
pragma_sql
lock_sql
literal_sql
loaddata_sql
null_sql
boolean_sql
order_sql
cluster_sql
distribute_sql
sort_sql
ordered_sql
matchrecognize_sql
query_modifiers
select_sql
schema_sql
star_sql
structkwarg_sql
parameter_sql
sessionparameter_sql
placeholder_sql
subquery_sql
qualify_sql
union_sql
union_op
unnest_sql
where_sql
window_sql
partition_by_sql
window_spec_sql
withingroup_sql
between_sql
bracket_sql
all_sql
any_sql
exists_sql
case_sql
constraint_sql
extract_sql
trim_sql
concat_sql
check_sql
foreignkey_sql
primarykey_sql
unique_sql
if_sql
jsonkeyvalue_sql
jsonobject_sql
in_sql
in_unnest_op
interval_sql
return_sql
reference_sql
anonymous_sql
paren_sql
neg_sql
not_sql
alias_sql
aliases_sql
attimezone_sql
add_sql
and_sql
connector_sql
bitwiseand_sql
bitwiseleftshift_sql
bitwisenot_sql
bitwiseor_sql
bitwiserightshift_sql
bitwisexor_sql
cast_sql
currentdate_sql
collate_sql
command_sql
comment_sql
transaction_sql
commit_sql
rollback_sql
altercolumn_sql
renametable_sql
altertable_sql
droppartition_sql
addconstraint_sql
distinct_sql
ignorenulls_sql
respectnulls_sql
intdiv_sql
dpipe_sql
div_sql
overlaps_sql
distance_sql
dot_sql
eq_sql
escape_sql
glob_sql
gt_sql
gte_sql
ilike_sql
is_sql
like_sql
similarto_sql
lt_sql
lte_sql
mod_sql
mul_sql
neq_sql
nullsafeeq_sql
nullsafeneq_sql
or_sql
slice_sql
sub_sql
trycast_sql
use_sql
binary
function_fallback_sql
func
format_args
text_width
format_time
expressions
op_expressions
naked_property
set_operation
tag_sql
token_sql
userdefinedfunction_sql
joinhint_sql
kwarg_sql
when_sql
merge_sql
tochar_sql
class MacroEvaluator:
 77class MacroEvaluator:
 78    """The class responsible for evaluating SQLMesh Macros/SQL.
 79
 80    SQLMesh supports special preprocessed SQL prefixed with `@`. Although it provides similar power to
 81    traditional methods like string templating, there is semantic understanding of SQL which prevents
 82    common errors like leading/trailing commas, syntax errors, etc.
 83
 84    SQLMesh SQL allows for macro variables and macro functions. Macro variables take the form of @variable. These are used for variable substitution.
 85
 86    SELECT * FROM foo WHERE ds BETWEEN @start_date AND @end_date
 87
 88    Macro variables can be defined with a special macro function.
 89
 90    @DEF(start_date, '2021-01-01')
 91
 92    Args:
 93        dialect: Dialect of the SQL to evaluate.
 94        python_env: Serialized Python environment.
 95    """
 96
 97    def __init__(
 98        self,
 99        dialect: str = "",
100        python_env: t.Optional[t.Dict[str, Executable]] = None,
101        jinja_env: t.Optional[Environment] = None,
102    ):
103        self.dialect = dialect
104        self.generator = MacroDialect().generator()
105        self.locals: t.Dict[str, t.Any] = {}
106        self.env = {**ENV, "self": self}
107        self.python_env = python_env or {}
108        self._jinja_env: t.Optional[Environment] = jinja_env
109        self.macros = {normalize_macro_name(k): v.func for k, v in macro.get_registry().items()}
110        prepare_env(self.python_env, self.env)
111        for k, v in self.python_env.items():
112            if v.is_definition:
113                self.macros[normalize_macro_name(k)] = self.env[v.name or k]
114
115    def send(
116        self, name: str, *args: t.Any
117    ) -> t.Union[None, exp.Expression, t.List[exp.Expression]]:
118        func = self.macros.get(normalize_macro_name(name))
119
120        if not callable(func):
121            raise SQLMeshError(f"Macro '{name}' does not exist.")
122
123        try:
124            return func(self, *args)
125        except Exception as e:
126            print_exception(e, self.python_env)
127            raise MacroEvalError(f"Error trying to eval macro.") from e
128
129    def transform(self, query: exp.Expression) -> exp.Expression | t.List[exp.Expression] | None:
130        def _transform_node(node: exp.Expression) -> exp.Expression:
131            if isinstance(node, MacroVar):
132                return exp.convert(_norm_env_value(self.locals[node.name]))
133            elif node.is_string:
134                node.set("this", self.jinja_env.from_string(node.this).render())
135                return node
136            else:
137                return node
138
139        query = query.transform(_transform_node)
140
141        def evaluate_macros(
142            node: exp.Expression,
143        ) -> exp.Expression | t.List[exp.Expression] | None:
144            exp.replace_children(
145                node, lambda n: n if isinstance(n, exp.Lambda) else evaluate_macros(n)
146            )
147            if isinstance(node, MacroFunc):
148                return self.evaluate(node)
149            return node
150
151        transformed = evaluate_macros(query)
152
153        if isinstance(transformed, list):
154            return [self.parse_one(node.sql(dialect=self.dialect)) for node in transformed]
155        elif isinstance(transformed, Jinja):
156            return transformed
157        elif isinstance(transformed, exp.Expression):
158            return self.parse_one(transformed.sql(dialect=self.dialect))
159
160        return transformed
161
162    def template(self, text: t.Any, local_variables: t.Dict[str, t.Any]) -> str:
163        """Substitute @vars with locals.
164
165        Args:
166            text: The string to do substitition on.
167            local_variables: Local variables in the context so that lambdas can be used.
168
169        Returns:
170           The rendered string.
171        """
172        return MacroStrTemplate(str(text)).safe_substitute(self.locals, **local_variables)
173
174    def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
175        if isinstance(node, MacroDef):
176            self.locals[node.name] = node.expression
177            return node
178
179        if isinstance(node, MacroSQL) or not node.find(exp.Column, exp.Table):
180            result = self.eval_expression(node)
181        else:
182            func = node.this
183            result = self.send(func.name, *func.expressions)
184
185        if result is None:
186            return None
187
188        if isinstance(result, list):
189            return [exp.convert(item) for item in result if item is not None]
190        return exp.convert(result)
191
192    def eval_expression(self, node: exp.Expression) -> t.Any:
193        """Converts a SQLGlot expression into executable Python code and evals it.
194
195        Args:
196            node: expression
197        Returns:
198            The return value of the evaled Python Code.
199        """
200        code = node.sql()
201        try:
202            code = self.generator.generate(node)
203            return eval(code, self.env, self.locals)
204        except Exception as e:
205            print_exception(e, self.python_env)
206            raise MacroEvalError(
207                f"Error trying to eval macro.\n\nGenerated code: {code}\n\nOriginal sql: {node}"
208            ) from e
209
210    def parse_one(
211        self, sql: str, into: t.Optional[exp.IntoType] = None, **opts: t.Any
212    ) -> exp.Expression:
213        """Parses the given SQL string and returns a syntax tree for the first
214        parsed SQL statement.
215
216        Args:
217            sql (str): the SQL code string to parse.
218            into (Expression): the Expression to parse into
219            **opts: other options
220
221        Returns:
222            Expression: the syntax tree for the first parsed statement
223        """
224        return sqlglot.maybe_parse(sql, dialect=self.dialect, into=into, **opts)
225
226    @property
227    def jinja_env(self) -> Environment:
228        if not self._jinja_env:
229            jinja_env_methods = {**self.locals, **self.env}
230            del jinja_env_methods["self"]
231            self._jinja_env = JinjaMacroRegistry().build_environment(**jinja_env_methods)
232        return self._jinja_env

The class responsible for evaluating SQLMesh Macros/SQL.

SQLMesh supports special preprocessed SQL prefixed with @. Although it provides similar power to traditional methods like string templating, there is semantic understanding of SQL which prevents common errors like leading/trailing commas, syntax errors, etc.

SQLMesh SQL allows for macro variables and macro functions. Macro variables take the form of @variable. These are used for variable substitution.

SELECT * FROM foo WHERE ds BETWEEN @start_date AND @end_date

Macro variables can be defined with a special macro function.

@DEF(start_date, '2021-01-01')

Arguments:
  • dialect: Dialect of the SQL to evaluate.
  • python_env: Serialized Python environment.
MacroEvaluator( dialect: str = '', python_env: Optional[Dict[str, sqlmesh.utils.metaprogramming.Executable]] = None, jinja_env: Optional[jinja2.environment.Environment] = None)
 97    def __init__(
 98        self,
 99        dialect: str = "",
100        python_env: t.Optional[t.Dict[str, Executable]] = None,
101        jinja_env: t.Optional[Environment] = None,
102    ):
103        self.dialect = dialect
104        self.generator = MacroDialect().generator()
105        self.locals: t.Dict[str, t.Any] = {}
106        self.env = {**ENV, "self": self}
107        self.python_env = python_env or {}
108        self._jinja_env: t.Optional[Environment] = jinja_env
109        self.macros = {normalize_macro_name(k): v.func for k, v in macro.get_registry().items()}
110        prepare_env(self.python_env, self.env)
111        for k, v in self.python_env.items():
112            if v.is_definition:
113                self.macros[normalize_macro_name(k)] = self.env[v.name or k]
def send( self, name: str, *args: Any) -> Union[NoneType, sqlglot.expressions.Expression, List[sqlglot.expressions.Expression]]:
115    def send(
116        self, name: str, *args: t.Any
117    ) -> t.Union[None, exp.Expression, t.List[exp.Expression]]:
118        func = self.macros.get(normalize_macro_name(name))
119
120        if not callable(func):
121            raise SQLMeshError(f"Macro '{name}' does not exist.")
122
123        try:
124            return func(self, *args)
125        except Exception as e:
126            print_exception(e, self.python_env)
127            raise MacroEvalError(f"Error trying to eval macro.") from e
def transform( self, query: sqlglot.expressions.Expression) -> Union[sqlglot.expressions.Expression, List[sqlglot.expressions.Expression], NoneType]:
129    def transform(self, query: exp.Expression) -> exp.Expression | t.List[exp.Expression] | None:
130        def _transform_node(node: exp.Expression) -> exp.Expression:
131            if isinstance(node, MacroVar):
132                return exp.convert(_norm_env_value(self.locals[node.name]))
133            elif node.is_string:
134                node.set("this", self.jinja_env.from_string(node.this).render())
135                return node
136            else:
137                return node
138
139        query = query.transform(_transform_node)
140
141        def evaluate_macros(
142            node: exp.Expression,
143        ) -> exp.Expression | t.List[exp.Expression] | None:
144            exp.replace_children(
145                node, lambda n: n if isinstance(n, exp.Lambda) else evaluate_macros(n)
146            )
147            if isinstance(node, MacroFunc):
148                return self.evaluate(node)
149            return node
150
151        transformed = evaluate_macros(query)
152
153        if isinstance(transformed, list):
154            return [self.parse_one(node.sql(dialect=self.dialect)) for node in transformed]
155        elif isinstance(transformed, Jinja):
156            return transformed
157        elif isinstance(transformed, exp.Expression):
158            return self.parse_one(transformed.sql(dialect=self.dialect))
159
160        return transformed
def template(self, text: Any, local_variables: Dict[str, Any]) -> str:
162    def template(self, text: t.Any, local_variables: t.Dict[str, t.Any]) -> str:
163        """Substitute @vars with locals.
164
165        Args:
166            text: The string to do substitition on.
167            local_variables: Local variables in the context so that lambdas can be used.
168
169        Returns:
170           The rendered string.
171        """
172        return MacroStrTemplate(str(text)).safe_substitute(self.locals, **local_variables)

Substitute @vars with locals.

Arguments:
  • text: The string to do substitition on.
  • local_variables: Local variables in the context so that lambdas can be used.
Returns:

The rendered string.

def evaluate( self, node: sqlmesh.core.dialect.MacroFunc) -> Union[sqlglot.expressions.Expression, List[sqlglot.expressions.Expression], NoneType]:
174    def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
175        if isinstance(node, MacroDef):
176            self.locals[node.name] = node.expression
177            return node
178
179        if isinstance(node, MacroSQL) or not node.find(exp.Column, exp.Table):
180            result = self.eval_expression(node)
181        else:
182            func = node.this
183            result = self.send(func.name, *func.expressions)
184
185        if result is None:
186            return None
187
188        if isinstance(result, list):
189            return [exp.convert(item) for item in result if item is not None]
190        return exp.convert(result)
def eval_expression(self, node: sqlglot.expressions.Expression) -> Any:
192    def eval_expression(self, node: exp.Expression) -> t.Any:
193        """Converts a SQLGlot expression into executable Python code and evals it.
194
195        Args:
196            node: expression
197        Returns:
198            The return value of the evaled Python Code.
199        """
200        code = node.sql()
201        try:
202            code = self.generator.generate(node)
203            return eval(code, self.env, self.locals)
204        except Exception as e:
205            print_exception(e, self.python_env)
206            raise MacroEvalError(
207                f"Error trying to eval macro.\n\nGenerated code: {code}\n\nOriginal sql: {node}"
208            ) from e

Converts a SQLGlot expression into executable Python code and evals it.

Arguments:
  • node: expression
Returns:

The return value of the evaled Python Code.

def parse_one( self, sql: str, into: Union[str, Type[sqlglot.expressions.Expression], Collection[Union[str, Type[sqlglot.expressions.Expression]]], NoneType] = None, **opts: Any) -> sqlglot.expressions.Expression:
210    def parse_one(
211        self, sql: str, into: t.Optional[exp.IntoType] = None, **opts: t.Any
212    ) -> exp.Expression:
213        """Parses the given SQL string and returns a syntax tree for the first
214        parsed SQL statement.
215
216        Args:
217            sql (str): the SQL code string to parse.
218            into (Expression): the Expression to parse into
219            **opts: other options
220
221        Returns:
222            Expression: the syntax tree for the first parsed statement
223        """
224        return sqlglot.maybe_parse(sql, dialect=self.dialect, into=into, **opts)

Parses the given SQL string and returns a syntax tree for the first parsed SQL statement.

Arguments:
  • sql (str): the SQL code string to parse.
  • into (Expression): the Expression to parse into
  • **opts: other options
Returns:

Expression: the syntax tree for the first parsed statement

class macro(sqlmesh.utils.registry_decorator):
235class macro(registry_decorator):
236    """Specifies a function is a macro and registers it the global MACROS registry.
237
238    Registered macros can be referenced in SQL statements to make queries more dynamic/cleaner.
239
240    Example:
241        from typing import t
242        from sqlglot import exp
243        from sqlmesh.core.macros import MacroEvaluator, macro
244
245        @macro()
246        def add_one(evaluator: MacroEvaluator, column: str) -> exp.Add:
247            return evaluator.parse_one(f"{column} + 1")
248
249    Args:
250        name: A custom name for the macro, the default is the name of the function.
251    """
252
253    registry_name = "macros"

Specifies a function is a macro and registers it the global MACROS registry.

Registered macros can be referenced in SQL statements to make queries more dynamic/cleaner.

Example:

from typing import t from sqlglot import exp from sqlmesh.core.macros import MacroEvaluator, macro

@macro() def add_one(evaluator: MacroEvaluator, column: str) -> exp.Add: return evaluator.parse_one(f"{column} + 1")

Arguments:
  • name: A custom name for the macro, the default is the name of the function.
@macro()
def each(evaluator: sqlmesh.core.macros.MacroEvaluator, *args: Any) -> List[Any]:
315@macro()
316def each(
317    evaluator: MacroEvaluator,
318    *args: t.Any,
319) -> t.List[t.Any]:
320    """Iterates through items calling func on each.
321
322    If a func call on item returns None, it will be excluded from the list.
323
324    Args:
325        evaluator: MacroEvaluator that invoked the macro
326        args: The last argument should be a lambda of the form x -> x +1. The first argument can be
327            an Array or var args can be used.
328
329    Returns:
330        A list of items that is the result of func
331    """
332    *items, func = args
333    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
334    return [item for item in map(func, ensure_collection(items)) if item is not None]

Iterates through items calling func on each.

If a func call on item returns None, it will be excluded from the list.

Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • args: The last argument should be a lambda of the form x -> x +1. The first argument can be an Array or var args can be used.
Returns:

A list of items that is the result of func

@macro('REDUCE')
def reduce_(evaluator: sqlmesh.core.macros.MacroEvaluator, *args: Any) -> Any:
337@macro("REDUCE")
338def reduce_(evaluator: MacroEvaluator, *args: t.Any) -> t.Any:
339    """Iterates through items applying provided function that takes two arguments
340    cumulatively to the items of iterable items, from left to right, so as to reduce
341    the iterable to a single item.
342
343    Example:
344        >>> from sqlglot import parse_one
345        >>> from sqlmesh.core.macros import MacroEvaluator, reduce_
346        >>> sql = "@SQL(@REDUCE([100, 200, 300, 400], (x, y) -> x + y))"
347        >>> MacroEvaluator().transform(parse_one(sql)).sql()
348        '1000'
349
350    Args:
351        evaluator: MacroEvaluator that invoked the macro
352        args: The last argument should be a lambda of the form (x, y) -> x + y. The first argument can be
353            an Array or var args can be used.
354    Returns:
355        A single item that is the result of applying func cumulatively to items
356    """
357    *items, func = args
358    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
359    return reduce(func, ensure_collection(items))

Iterates through items applying provided function that takes two arguments cumulatively to the items of iterable items, from left to right, so as to reduce the iterable to a single item.

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, reduce_
>>> sql = "@SQL(@REDUCE([100, 200, 300, 400], (x, y) -> x + y))"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'1000'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • args: The last argument should be a lambda of the form (x, y) -> x + y. The first argument can be an Array or var args can be used.
Returns:

A single item that is the result of applying func cumulatively to items

@macro('FILTER')
def filter_(evaluator: sqlmesh.core.macros.MacroEvaluator, *args: Any) -> List[Any]:
362@macro("FILTER")
363def filter_(evaluator: MacroEvaluator, *args: t.Any) -> t.List[t.Any]:
364    """Iterates through items, applying provided function to each item and removing
365    all items where the function returns False
366
367    Example:
368        >>> from sqlglot import parse_one
369        >>> from sqlmesh.core.macros import MacroEvaluator, filter_
370        >>> sql = "@REDUCE(@FILTER([1, 2, 3], x -> x > 1), (x, y) -> x + y)"
371        >>> MacroEvaluator().transform(parse_one(sql)).sql()
372        '5'
373
374    Args:
375        evaluator: MacroEvaluator that invoked the macro
376        args: The last argument should be a lambda of the form x -> x > 1. The first argument can be
377            an Array or var args can be used.
378    Returns:
379        The items for which the func returned True
380    """
381    *items, func = args
382    items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
383    return list(filter(func, items))

Iterates through items, applying provided function to each item and removing all items where the function returns False

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, filter_
>>> sql = "@REDUCE(@FILTER([1, 2, 3], x -> x > 1), (x, y) -> x + y)"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'5'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • args: The last argument should be a lambda of the form x -> x > 1. The first argument can be an Array or var args can be used.
Returns:

The items for which the func returned True

@macro('WITH')
def with_( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.With) -> Optional[sqlglot.expressions.With]:
386@macro("WITH")
387def with_(
388    evaluator: MacroEvaluator,
389    condition: exp.Condition,
390    expression: exp.With,
391) -> t.Optional[exp.With]:
392    """Inserts WITH expression when the condition is True
393
394    Example:
395        >>> from sqlglot import parse_one
396        >>> from sqlmesh.core.macros import MacroEvaluator, with_
397        >>> sql = "@WITH(True) all_cities as (select * from city) select all_cities"
398        >>> MacroEvaluator().transform(parse_one(sql)).sql()
399        'WITH all_cities AS (SELECT * FROM city) SELECT all_cities'
400
401    Args:
402        evaluator: MacroEvaluator that invoked the macro
403        condition: Condition expression
404        expression: With expression
405    Returns:
406        With expression if the conditional is True; otherwise None
407    """
408    return expression if evaluator.eval_expression(condition) else None

Inserts WITH expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, with_
>>> sql = "@WITH(True) all_cities as (select * from city) select all_cities"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'WITH all_cities AS (SELECT * FROM city) SELECT all_cities'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: With expression
Returns:

With expression if the conditional is True; otherwise None

@macro()
def join( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.Join) -> Optional[sqlglot.expressions.Join]:
411@macro()
412def join(
413    evaluator: MacroEvaluator,
414    condition: exp.Condition,
415    expression: exp.Join,
416) -> t.Optional[exp.Join]:
417    """Inserts JOIN expression when the condition is True
418
419    Example:
420        >>> from sqlglot import parse_one
421        >>> from sqlmesh.core.macros import MacroEvaluator, join
422        >>> sql = "select * from city @JOIN(True) country on city.country = country.name"
423        >>> MacroEvaluator().transform(parse_one(sql)).sql()
424        'SELECT * FROM city JOIN country ON city.country = country.name'
425
426        >>> sql = "select * from city left outer @JOIN(True) country on city.country = country.name"
427        >>> MacroEvaluator().transform(parse_one(sql)).sql()
428        'SELECT * FROM city LEFT OUTER JOIN country ON city.country = country.name'
429
430    Args:
431        evaluator: MacroEvaluator that invoked the macro
432        condition: Condition expression
433        expression: Join expression
434    Returns:
435        Join expression if the conditional is True; otherwise None
436    """
437    return expression if evaluator.eval_expression(condition) else None

Inserts JOIN expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, join
>>> sql = "select * from city @JOIN(True) country on city.country = country.name"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'SELECT * FROM city JOIN country ON city.country = country.name'
>>> sql = "select * from city left outer @JOIN(True) country on city.country = country.name"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'SELECT * FROM city LEFT OUTER JOIN country ON city.country = country.name'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: Join expression
Returns:

Join expression if the conditional is True; otherwise None

@macro()
def where( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.Where) -> Optional[sqlglot.expressions.Where]:
440@macro()
441def where(
442    evaluator: MacroEvaluator,
443    condition: exp.Condition,
444    expression: exp.Where,
445) -> t.Optional[exp.Where]:
446    """Inserts WHERE expression when the condition is True
447
448    Example:
449        >>> from sqlglot import parse_one
450        >>> from sqlmesh.core.macros import MacroEvaluator, where
451        >>> sql = "select * from city @WHERE(True) population > 100 and country = 'Mexico'"
452        >>> MacroEvaluator().transform(parse_one(sql)).sql()
453        "SELECT * FROM city WHERE population > 100 AND country = 'Mexico'"
454
455    Args:
456        evaluator: MacroEvaluator that invoked the macro
457        condition: Condition expression
458        expression: Where expression
459    Returns:
460        Where expression if condition is True; otherwise None
461    """
462    return expression if evaluator.eval_expression(condition) else None

Inserts WHERE expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, where
>>> sql = "select * from city @WHERE(True) population > 100 and country = 'Mexico'"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
"SELECT * FROM city WHERE population > 100 AND country = 'Mexico'"
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: Where expression
Returns:

Where expression if condition is True; otherwise None

@macro()
def group_by( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.Group) -> Optional[sqlglot.expressions.Group]:
465@macro()
466def group_by(
467    evaluator: MacroEvaluator,
468    condition: exp.Condition,
469    expression: exp.Group,
470) -> t.Optional[exp.Group]:
471    """Inserts GROUP BY expression when the condition is True
472
473    Example:
474        >>> from sqlglot import parse_one
475        >>> from sqlmesh.core.macros import MacroEvaluator, group_by
476        >>> sql = "select * from city @GROUP_BY(True) country, population"
477        >>> MacroEvaluator().transform(parse_one(sql)).sql()
478        'SELECT * FROM city GROUP BY country, population'
479
480    Args:
481        evaluator: MacroEvaluator that invoked the macro
482        condition: Condition expression
483        expression: Group expression
484    Returns:
485        Group expression if the condition is True; otherwise None
486    """
487    return expression if evaluator.eval_expression(condition) else None

Inserts GROUP BY expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, group_by
>>> sql = "select * from city @GROUP_BY(True) country, population"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'SELECT * FROM city GROUP BY country, population'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: Group expression
Returns:

Group expression if the condition is True; otherwise None

@macro()
def having( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.Having) -> Optional[sqlglot.expressions.Having]:
490@macro()
491def having(
492    evaluator: MacroEvaluator,
493    condition: exp.Condition,
494    expression: exp.Having,
495) -> t.Optional[exp.Having]:
496    """Inserts HAVING expression when the condition is True
497
498    Example:
499        >>> from sqlglot import parse_one
500        >>> from sqlmesh.core.macros import MacroEvaluator, having
501        >>> sql = "select * from city group by country @HAVING(True) population > 100 and country = 'Mexico'"
502        >>> MacroEvaluator().transform(parse_one(sql)).sql()
503        "SELECT * FROM city GROUP BY country HAVING population > 100 AND country = 'Mexico'"
504
505    Args:
506        evaluator: MacroEvaluator that invoked the macro
507        condition: Condition expression
508        expression: Having expression
509    Returns:
510        Having expression if the condition is True; otherwise None
511    """
512    return expression if evaluator.eval_expression(condition) else None

Inserts HAVING expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, having
>>> sql = "select * from city group by country @HAVING(True) population > 100 and country = 'Mexico'"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
"SELECT * FROM city GROUP BY country HAVING population > 100 AND country = 'Mexico'"
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: Having expression
Returns:

Having expression if the condition is True; otherwise None

@macro()
def order_by( evaluator: sqlmesh.core.macros.MacroEvaluator, condition: sqlglot.expressions.Condition, expression: sqlglot.expressions.Order) -> Optional[sqlglot.expressions.Order]:
515@macro()
516def order_by(
517    evaluator: MacroEvaluator,
518    condition: exp.Condition,
519    expression: exp.Order,
520) -> t.Optional[exp.Order]:
521    """Inserts ORDER BY expression when the condition is True
522
523    Example:
524        >>> from sqlglot import parse_one
525        >>> from sqlmesh.core.macros import MacroEvaluator, order_by
526        >>> sql = "select * from city @ORDER_BY(True) population, name DESC"
527        >>> MacroEvaluator().transform(parse_one(sql)).sql()
528        'SELECT * FROM city ORDER BY population, name DESC'
529
530    Args:
531        evaluator: MacroEvaluator that invoked the macro
532        condition: Condition expression
533        expression: Order expression
534    Returns:
535        Order expression if the condition is True; otherwise None
536    """
537    return expression if evaluator.eval_expression(condition) else None

Inserts ORDER BY expression when the condition is True

Example:
>>> from sqlglot import parse_one
>>> from sqlmesh.core.macros import MacroEvaluator, order_by
>>> sql = "select * from city @ORDER_BY(True) population, name DESC"
>>> MacroEvaluator().transform(parse_one(sql)).sql()
'SELECT * FROM city ORDER BY population, name DESC'
Arguments:
  • evaluator: MacroEvaluator that invoked the macro
  • condition: Condition expression
  • expression: Order expression
Returns:

Order expression if the condition is True; otherwise None

def normalize_macro_name(name: str) -> str:
540def normalize_macro_name(name: str) -> str:
541    """Prefix macro name with @ and upcase"""
542    return f"@{name.upper()}"

Prefix macro name with @ and upcase