SQLMesh is a DataOps framework that brings the best practices of DevOps to data teams. It enables data scientists, analysts, and engineers to efficiently run and deploy data transformations written in SQL or Python.
For more infromation, check out the website and documentation.
Geting Started
Install SQLMesh through pypi by running:
pip install sqlmesh
Follow the tutorial to learn how to use SQLMesh.
Join our community
We'd love to join you on your data journey. Connect with us in the following ways:
- Join the Tobiko Slack community to ask questions, or just to say hi!
- File an issue on our GitHub.
- Send us an email at hello@tobikodata.com with your questions or feedback.
Contribution
Contributions in the form of issues or pull requests are greatly appreciated. Read more about how to develop for SQLMesh.
1""" 2.. include:: ../README.md 3""" 4from __future__ import annotations 5 6import logging 7import os 8import sys 9from enum import Enum 10 11from sqlmesh.core.dialect import extend_sqlglot 12 13extend_sqlglot() 14 15from sqlmesh.core.config import Config 16from sqlmesh.core.context import Context, ExecutionContext 17from sqlmesh.core.engine_adapter import EngineAdapter 18from sqlmesh.core.hooks import hook 19from sqlmesh.core.macros import macro 20from sqlmesh.core.model import Model, model 21from sqlmesh.core.snapshot import Snapshot 22 23try: 24 from sqlmesh._version import __version__, __version_tuple__ # type: ignore 25except ImportError: 26 pass 27 28 29class RuntimeEnv(str, Enum): 30 """Enum defining what environment SQLMesh is running in.""" 31 32 TERMINAL = "terminal" 33 DATABRICKS = "databricks" 34 GOOGLE_COLAB = "google_colab" # Not currently officially supported 35 JUPYTER = "jupyter" 36 37 @classmethod 38 def get(cls) -> RuntimeEnv: 39 """Get the console class to use based on the environment that the code is running in 40 Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528 41 42 Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter. 43 """ 44 try: 45 shell = get_ipython() # type: ignore 46 if os.getenv("DATABRICKS_RUNTIME_VERSION"): 47 return RuntimeEnv.DATABRICKS 48 if "google.colab" in str(shell.__class__): # type: ignore 49 return RuntimeEnv.GOOGLE_COLAB 50 if shell.__class__.__name__ == "ZMQInteractiveShell": # type: ignore 51 return RuntimeEnv.JUPYTER 52 except NameError: 53 pass 54 55 return RuntimeEnv.TERMINAL 56 57 @property 58 def is_terminal(self) -> bool: 59 return self == RuntimeEnv.TERMINAL 60 61 @property 62 def is_databricks(self) -> bool: 63 return self == RuntimeEnv.DATABRICKS 64 65 @property 66 def is_jupyter(self) -> bool: 67 return self == RuntimeEnv.JUPYTER 68 69 @property 70 def is_google_colab(self) -> bool: 71 return self == RuntimeEnv.GOOGLE_COLAB 72 73 @property 74 def is_notebook(self) -> bool: 75 return not self.is_terminal 76 77 78runtime_env = RuntimeEnv.get() 79 80 81if runtime_env.is_notebook: 82 try: 83 from sqlmesh.magics import register_magics 84 85 register_magics() 86 except ImportError: 87 pass 88 89 90# SO: https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output 91class CustomFormatter(logging.Formatter): 92 """Custom logging formatter.""" 93 94 grey = "\x1b[38;20m" 95 yellow = "\x1b[33;20m" 96 red = "\x1b[31;20m" 97 bold_red = "\x1b[31;1m" 98 reset = "\x1b[0m" 99 log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" 100 101 FORMATS = { 102 logging.DEBUG: grey + log_format + reset, 103 logging.INFO: grey + log_format + reset, 104 logging.WARNING: yellow + log_format + reset, 105 logging.ERROR: red + log_format + reset, 106 logging.CRITICAL: bold_red + log_format + reset, 107 } 108 109 def format(self, record: logging.LogRecord) -> str: 110 log_fmt = self.FORMATS.get(record.levelno) 111 formatter = logging.Formatter(log_fmt) 112 return formatter.format(record) 113 114 115def enable_logging(level: int = logging.INFO) -> None: 116 """Enable logging to send to stdout and color different levels""" 117 logger = logging.getLogger() 118 logger.setLevel(level) 119 if not logger.hasHandlers(): 120 handler = logging.StreamHandler(sys.stdout) 121 handler.setLevel(level) 122 handler.setFormatter(CustomFormatter()) 123 logger.addHandler(handler)
30class RuntimeEnv(str, Enum): 31 """Enum defining what environment SQLMesh is running in.""" 32 33 TERMINAL = "terminal" 34 DATABRICKS = "databricks" 35 GOOGLE_COLAB = "google_colab" # Not currently officially supported 36 JUPYTER = "jupyter" 37 38 @classmethod 39 def get(cls) -> RuntimeEnv: 40 """Get the console class to use based on the environment that the code is running in 41 Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528 42 43 Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter. 44 """ 45 try: 46 shell = get_ipython() # type: ignore 47 if os.getenv("DATABRICKS_RUNTIME_VERSION"): 48 return RuntimeEnv.DATABRICKS 49 if "google.colab" in str(shell.__class__): # type: ignore 50 return RuntimeEnv.GOOGLE_COLAB 51 if shell.__class__.__name__ == "ZMQInteractiveShell": # type: ignore 52 return RuntimeEnv.JUPYTER 53 except NameError: 54 pass 55 56 return RuntimeEnv.TERMINAL 57 58 @property 59 def is_terminal(self) -> bool: 60 return self == RuntimeEnv.TERMINAL 61 62 @property 63 def is_databricks(self) -> bool: 64 return self == RuntimeEnv.DATABRICKS 65 66 @property 67 def is_jupyter(self) -> bool: 68 return self == RuntimeEnv.JUPYTER 69 70 @property 71 def is_google_colab(self) -> bool: 72 return self == RuntimeEnv.GOOGLE_COLAB 73 74 @property 75 def is_notebook(self) -> bool: 76 return not self.is_terminal
Enum defining what environment SQLMesh is running in.
38 @classmethod 39 def get(cls) -> RuntimeEnv: 40 """Get the console class to use based on the environment that the code is running in 41 Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528 42 43 Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter. 44 """ 45 try: 46 shell = get_ipython() # type: ignore 47 if os.getenv("DATABRICKS_RUNTIME_VERSION"): 48 return RuntimeEnv.DATABRICKS 49 if "google.colab" in str(shell.__class__): # type: ignore 50 return RuntimeEnv.GOOGLE_COLAB 51 if shell.__class__.__name__ == "ZMQInteractiveShell": # type: ignore 52 return RuntimeEnv.JUPYTER 53 except NameError: 54 pass 55 56 return RuntimeEnv.TERMINAL
Get the console class to use based on the environment that the code is running in Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
Inherited Members
- enum.Enum
- name
- value
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
92class CustomFormatter(logging.Formatter): 93 """Custom logging formatter.""" 94 95 grey = "\x1b[38;20m" 96 yellow = "\x1b[33;20m" 97 red = "\x1b[31;20m" 98 bold_red = "\x1b[31;1m" 99 reset = "\x1b[0m" 100 log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" 101 102 FORMATS = { 103 logging.DEBUG: grey + log_format + reset, 104 logging.INFO: grey + log_format + reset, 105 logging.WARNING: yellow + log_format + reset, 106 logging.ERROR: red + log_format + reset, 107 logging.CRITICAL: bold_red + log_format + reset, 108 } 109 110 def format(self, record: logging.LogRecord) -> str: 111 log_fmt = self.FORMATS.get(record.levelno) 112 formatter = logging.Formatter(log_fmt) 113 return formatter.format(record)
Custom logging formatter.
110 def format(self, record: logging.LogRecord) -> str: 111 log_fmt = self.FORMATS.get(record.levelno) 112 formatter = logging.Formatter(log_fmt) 113 return formatter.format(record)
Format the specified record as text.
The record's attribute dictionary is used as the operand to a string formatting operation which yields the returned string. Before formatting the dictionary, a couple of preparatory steps are carried out. The message attribute of the record is computed using LogRecord.getMessage(). If the formatting string uses the time (as determined by a call to usesTime(), formatTime() is called to format the event time. If there is exception information, it is formatted using formatException() and appended to the message.
Inherited Members
- logging.Formatter
- Formatter
- converter
- formatTime
- formatException
- usesTime
- formatMessage
- formatStack
116def enable_logging(level: int = logging.INFO) -> None: 117 """Enable logging to send to stdout and color different levels""" 118 logger = logging.getLogger() 119 logger.setLevel(level) 120 if not logger.hasHandlers(): 121 handler = logging.StreamHandler(sys.stdout) 122 handler.setLevel(level) 123 handler.setFormatter(CustomFormatter()) 124 logger.addHandler(handler)
Enable logging to send to stdout and color different levels