sqlmesh.cli.example_project
1import typing as t 2from enum import Enum 3from pathlib import Path 4 5import click 6 7DEFAULT_CONFIG = """connections: 8 local: 9 type: duckdb 10 database: db.duckdb 11 12default_connection: local 13""" 14 15 16DEFAULT_AIRFLOW_CONFIG = """scheduler: 17 type: airflow 18 airflow_url: http://localhost:8080/ 19 username: airflow 20 password: airflow 21""" 22 23DEFAULT_DBT_CONFIG = """from pathlib import Path 24 25from sqlmesh.dbt.loader import sqlmesh_config 26 27config = sqlmesh_config(Path(__file__).parent) 28""" 29 30EXAMPLE_SCHEMA_NAME = "sqlmesh_example" 31EXAMPLE_FULL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_full_model" 32EXAMPLE_INCREMENTAL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_incremental_model" 33 34 35EXAMPLE_FULL_MODEL_DEF = f"""MODEL ( 36 name {EXAMPLE_FULL_MODEL_NAME}, 37 kind FULL, 38 cron '@daily', 39 audits [assert_positive_order_ids], 40); 41 42SELECT 43 item_id, 44 count(distinct id) AS num_orders, 45FROM 46 {EXAMPLE_INCREMENTAL_MODEL_NAME} 47GROUP BY item_id 48""" 49 50EXAMPLE_INCREMENTAL_MODEL_DEF = f"""MODEL ( 51 name {EXAMPLE_INCREMENTAL_MODEL_NAME}, 52 kind INCREMENTAL_BY_TIME_RANGE ( 53 time_column ds 54 ), 55 start '2020-01-01', 56 cron '@daily', 57); 58 59SELECT 60 id, 61 item_id, 62 ds, 63FROM 64 (VALUES 65 (1, 1, '2020-01-01'), 66 (1, 2, '2020-01-01'), 67 (2, 1, '2020-01-01'), 68 (3, 3, '2020-01-03'), 69 (4, 1, '2020-01-04'), 70 (5, 1, '2020-01-05'), 71 (6, 1, '2020-01-06'), 72 (7, 1, '2020-01-07') 73 ) AS t (id, item_id, ds) 74WHERE 75 ds between @start_ds and @end_ds 76""" 77 78EXAMPLE_AUDIT = f"""AUDIT ( 79 name assert_positive_order_ids, 80); 81 82SELECT * 83FROM @this_model 84WHERE 85 item_id < 0 86""" 87 88 89EXAMPLE_TEST = f"""test_example_full_model: 90 model: {EXAMPLE_FULL_MODEL_NAME} 91 inputs: 92 {EXAMPLE_INCREMENTAL_MODEL_NAME}: 93 rows: 94 - id: 1 95 item_id: 1 96 ds: '2020-01-01' 97 - id: 2 98 item_id: 1 99 ds: '2020-01-02' 100 - id: 3 101 item_id: 2 102 ds: '2020-01-03' 103 outputs: 104 query: 105 rows: 106 - item_id: 1 107 num_orders: 2 108 - item_id: 2 109 num_orders: 1 110""" 111 112 113class ProjectTemplate(Enum): 114 AIRFLOW = "airflow" 115 DBT = "dbt" 116 DEFAULT = "default" 117 118 119DEFAULT_CONFIGS = { 120 ProjectTemplate.AIRFLOW: DEFAULT_AIRFLOW_CONFIG, 121 ProjectTemplate.DBT: DEFAULT_DBT_CONFIG, 122 ProjectTemplate.DEFAULT: DEFAULT_CONFIG, 123} 124 125 126def init_example_project( 127 path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT 128) -> None: 129 root_path = Path(path) 130 config_extension = "py" if template == ProjectTemplate.DBT else "yaml" 131 config_path = root_path / f"config.{config_extension}" 132 audits_path = root_path / "audits" 133 macros_path = root_path / "macros" 134 models_path = root_path / "models" 135 tests_path = root_path / "tests" 136 137 if config_path.exists(): 138 raise click.ClickException(f"Found an existing config in '{config_path}'") 139 140 _create_config(config_path, template) 141 if template == ProjectTemplate.DBT: 142 return 143 144 _create_folders([audits_path, macros_path, models_path, tests_path]) 145 _create_macros(macros_path) 146 _create_audits(audits_path) 147 _create_models(models_path) 148 _create_tests(tests_path) 149 150 151def _create_folders(target_folders: t.Sequence[Path]) -> None: 152 for folder_path in target_folders: 153 folder_path.mkdir(exist_ok=True) 154 (folder_path / ".gitkeep").touch() 155 156 157def _create_config(config_path: Path, template: ProjectTemplate) -> None: 158 _write_file( 159 config_path, 160 DEFAULT_CONFIGS[template], 161 ) 162 163 164def _create_macros(macros_path: Path) -> None: 165 (macros_path / "__init__.py").touch() 166 167 168def _create_audits(audits_path: Path) -> None: 169 _write_file(audits_path / "example_full_model.sql", EXAMPLE_AUDIT) 170 171 172def _create_models(models_path: Path) -> None: 173 for model_name, model_def in [ 174 (EXAMPLE_FULL_MODEL_NAME, EXAMPLE_FULL_MODEL_DEF), 175 (EXAMPLE_INCREMENTAL_MODEL_NAME, EXAMPLE_INCREMENTAL_MODEL_DEF), 176 ]: 177 _write_file(models_path / f"{model_name.split('.')[-1]}.sql", model_def) 178 179 180def _create_tests(tests_path: Path) -> None: 181 _write_file(tests_path / "test_example_full_model.yaml", EXAMPLE_TEST) 182 183 184def _write_file(path: Path, payload: str) -> None: 185 with open(path, "w", encoding="utf-8") as fd: 186 fd.write(payload)
class
ProjectTemplate(enum.Enum):
An enumeration.
AIRFLOW = <ProjectTemplate.AIRFLOW: 'airflow'>
DBT = <ProjectTemplate.DBT: 'dbt'>
DEFAULT = <ProjectTemplate.DEFAULT: 'default'>
Inherited Members
- enum.Enum
- name
- value
def
init_example_project( path: Union[str, pathlib.Path], template: sqlmesh.cli.example_project.ProjectTemplate = <ProjectTemplate.DEFAULT: 'default'>) -> None:
127def init_example_project( 128 path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT 129) -> None: 130 root_path = Path(path) 131 config_extension = "py" if template == ProjectTemplate.DBT else "yaml" 132 config_path = root_path / f"config.{config_extension}" 133 audits_path = root_path / "audits" 134 macros_path = root_path / "macros" 135 models_path = root_path / "models" 136 tests_path = root_path / "tests" 137 138 if config_path.exists(): 139 raise click.ClickException(f"Found an existing config in '{config_path}'") 140 141 _create_config(config_path, template) 142 if template == ProjectTemplate.DBT: 143 return 144 145 _create_folders([audits_path, macros_path, models_path, tests_path]) 146 _create_macros(macros_path) 147 _create_audits(audits_path) 148 _create_models(models_path) 149 _create_tests(tests_path)