Edit on GitHub

SQLMesh is a DataOps framework that brings the best practices of DevOps to data teams. It enables data scientists, analysts, and engineers to efficiently run and deploy data transformations written in SQL or Python.

For more infromation, check out the website and documentation.

Geting Started

Install SQLMesh through pypi by running:

pip install sqlmesh

Follow the tutorial to learn how to use SQLMesh.

Join our community

We'd love to join you on your data journey. Connect with us in the following ways:

Contribution

Contributions in the form of issues or pull requests are greatly appreciated. Read more about how to develop for SQLMesh.

  1"""
  2.. include:: ../README.md
  3"""
  4from __future__ import annotations
  5
  6import logging
  7import os
  8import sys
  9from enum import Enum
 10
 11from sqlmesh.core.dialect import extend_sqlglot
 12
 13extend_sqlglot()
 14
 15from sqlmesh.core.config import Config
 16from sqlmesh.core.context import Context, ExecutionContext
 17from sqlmesh.core.engine_adapter import EngineAdapter
 18from sqlmesh.core.hooks import hook
 19from sqlmesh.core.macros import macro
 20from sqlmesh.core.model import Model, model
 21from sqlmesh.core.snapshot import Snapshot
 22
 23try:
 24    from sqlmesh._version import __version__, __version_tuple__  # type: ignore
 25except ImportError:
 26    pass
 27
 28
 29class RuntimeEnv(str, Enum):
 30    """Enum defining what environment SQLMesh is running in."""
 31
 32    TERMINAL = "terminal"
 33    DATABRICKS = "databricks"
 34    GOOGLE_COLAB = "google_colab"  # Not currently officially supported
 35    JUPYTER = "jupyter"
 36
 37    @classmethod
 38    def get(cls) -> RuntimeEnv:
 39        """Get the console class to use based on the environment that the code is running in
 40        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
 41
 42        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
 43        """
 44        try:
 45            shell = get_ipython()  # type: ignore
 46            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
 47                return RuntimeEnv.DATABRICKS
 48            if "google.colab" in str(shell.__class__):  # type: ignore
 49                return RuntimeEnv.GOOGLE_COLAB
 50            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
 51                return RuntimeEnv.JUPYTER
 52        except NameError:
 53            pass
 54
 55        return RuntimeEnv.TERMINAL
 56
 57    @property
 58    def is_terminal(self) -> bool:
 59        return self == RuntimeEnv.TERMINAL
 60
 61    @property
 62    def is_databricks(self) -> bool:
 63        return self == RuntimeEnv.DATABRICKS
 64
 65    @property
 66    def is_jupyter(self) -> bool:
 67        return self == RuntimeEnv.JUPYTER
 68
 69    @property
 70    def is_google_colab(self) -> bool:
 71        return self == RuntimeEnv.GOOGLE_COLAB
 72
 73    @property
 74    def is_notebook(self) -> bool:
 75        return not self.is_terminal
 76
 77
 78runtime_env = RuntimeEnv.get()
 79
 80
 81if runtime_env.is_notebook:
 82    try:
 83        from sqlmesh.magics import register_magics
 84
 85        register_magics()
 86    except ImportError:
 87        pass
 88
 89
 90# SO: https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
 91class CustomFormatter(logging.Formatter):
 92    """Custom logging formatter."""
 93
 94    grey = "\x1b[38;20m"
 95    yellow = "\x1b[33;20m"
 96    red = "\x1b[31;20m"
 97    bold_red = "\x1b[31;1m"
 98    reset = "\x1b[0m"
 99    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
100
101    FORMATS = {
102        logging.DEBUG: grey + log_format + reset,
103        logging.INFO: grey + log_format + reset,
104        logging.WARNING: yellow + log_format + reset,
105        logging.ERROR: red + log_format + reset,
106        logging.CRITICAL: bold_red + log_format + reset,
107    }
108
109    def format(self, record: logging.LogRecord) -> str:
110        log_fmt = self.FORMATS.get(record.levelno)
111        formatter = logging.Formatter(log_fmt)
112        return formatter.format(record)
113
114
115def enable_logging(level: int = logging.INFO) -> None:
116    """Enable logging to send to stdout and color different levels"""
117    logger = logging.getLogger()
118    logger.setLevel(level)
119    if not logger.hasHandlers():
120        handler = logging.StreamHandler(sys.stdout)
121        handler.setLevel(level)
122        handler.setFormatter(CustomFormatter())
123        logger.addHandler(handler)
class RuntimeEnv(builtins.str, enum.Enum):
30class RuntimeEnv(str, Enum):
31    """Enum defining what environment SQLMesh is running in."""
32
33    TERMINAL = "terminal"
34    DATABRICKS = "databricks"
35    GOOGLE_COLAB = "google_colab"  # Not currently officially supported
36    JUPYTER = "jupyter"
37
38    @classmethod
39    def get(cls) -> RuntimeEnv:
40        """Get the console class to use based on the environment that the code is running in
41        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
42
43        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
44        """
45        try:
46            shell = get_ipython()  # type: ignore
47            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
48                return RuntimeEnv.DATABRICKS
49            if "google.colab" in str(shell.__class__):  # type: ignore
50                return RuntimeEnv.GOOGLE_COLAB
51            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
52                return RuntimeEnv.JUPYTER
53        except NameError:
54            pass
55
56        return RuntimeEnv.TERMINAL
57
58    @property
59    def is_terminal(self) -> bool:
60        return self == RuntimeEnv.TERMINAL
61
62    @property
63    def is_databricks(self) -> bool:
64        return self == RuntimeEnv.DATABRICKS
65
66    @property
67    def is_jupyter(self) -> bool:
68        return self == RuntimeEnv.JUPYTER
69
70    @property
71    def is_google_colab(self) -> bool:
72        return self == RuntimeEnv.GOOGLE_COLAB
73
74    @property
75    def is_notebook(self) -> bool:
76        return not self.is_terminal

Enum defining what environment SQLMesh is running in.

TERMINAL = <RuntimeEnv.TERMINAL: 'terminal'>
DATABRICKS = <RuntimeEnv.DATABRICKS: 'databricks'>
GOOGLE_COLAB = <RuntimeEnv.GOOGLE_COLAB: 'google_colab'>
JUPYTER = <RuntimeEnv.JUPYTER: 'jupyter'>
@classmethod
def get(cls) -> sqlmesh.RuntimeEnv:
38    @classmethod
39    def get(cls) -> RuntimeEnv:
40        """Get the console class to use based on the environment that the code is running in
41        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
42
43        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
44        """
45        try:
46            shell = get_ipython()  # type: ignore
47            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
48                return RuntimeEnv.DATABRICKS
49            if "google.colab" in str(shell.__class__):  # type: ignore
50                return RuntimeEnv.GOOGLE_COLAB
51            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
52                return RuntimeEnv.JUPYTER
53        except NameError:
54            pass
55
56        return RuntimeEnv.TERMINAL

Get the console class to use based on the environment that the code is running in Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528

Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.

Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
removeprefix
removesuffix
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
runtime_env = <RuntimeEnv.TERMINAL: 'terminal'>
class CustomFormatter(logging.Formatter):
 92class CustomFormatter(logging.Formatter):
 93    """Custom logging formatter."""
 94
 95    grey = "\x1b[38;20m"
 96    yellow = "\x1b[33;20m"
 97    red = "\x1b[31;20m"
 98    bold_red = "\x1b[31;1m"
 99    reset = "\x1b[0m"
100    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
101
102    FORMATS = {
103        logging.DEBUG: grey + log_format + reset,
104        logging.INFO: grey + log_format + reset,
105        logging.WARNING: yellow + log_format + reset,
106        logging.ERROR: red + log_format + reset,
107        logging.CRITICAL: bold_red + log_format + reset,
108    }
109
110    def format(self, record: logging.LogRecord) -> str:
111        log_fmt = self.FORMATS.get(record.levelno)
112        formatter = logging.Formatter(log_fmt)
113        return formatter.format(record)

Custom logging formatter.

def format(self, record: logging.LogRecord) -> str:
110    def format(self, record: logging.LogRecord) -> str:
111        log_fmt = self.FORMATS.get(record.levelno)
112        formatter = logging.Formatter(log_fmt)
113        return formatter.format(record)

Format the specified record as text.

The record's attribute dictionary is used as the operand to a string formatting operation which yields the returned string. Before formatting the dictionary, a couple of preparatory steps are carried out. The message attribute of the record is computed using LogRecord.getMessage(). If the formatting string uses the time (as determined by a call to usesTime(), formatTime() is called to format the event time. If there is exception information, it is formatted using formatException() and appended to the message.

Inherited Members
logging.Formatter
Formatter
converter
formatTime
formatException
usesTime
formatMessage
formatStack
def enable_logging(level: int = 20) -> None:
116def enable_logging(level: int = logging.INFO) -> None:
117    """Enable logging to send to stdout and color different levels"""
118    logger = logging.getLogger()
119    logger.setLevel(level)
120    if not logger.hasHandlers():
121        handler = logging.StreamHandler(sys.stdout)
122        handler.setLevel(level)
123        handler.setFormatter(CustomFormatter())
124        logger.addHandler(handler)

Enable logging to send to stdout and color different levels