Edit on GitHub

sqlmesh.cli.example_project

  1import typing as t
  2from enum import Enum
  3from pathlib import Path
  4
  5import click
  6
  7DEFAULT_CONFIG = """connections:
  8    local:
  9        type: duckdb
 10        database: db.duckdb
 11
 12default_connection: local
 13"""
 14
 15
 16DEFAULT_AIRFLOW_CONFIG = """scheduler:
 17    type: airflow
 18    airflow_url: http://localhost:8080/
 19    username: airflow
 20    password: airflow
 21"""
 22
 23DEFAULT_DBT_CONFIG = """from pathlib import Path
 24
 25from sqlmesh.dbt.loader import sqlmesh_config
 26
 27config = sqlmesh_config(Path(__file__).parent)
 28"""
 29
 30EXAMPLE_SCHEMA_NAME = "sqlmesh_example"
 31EXAMPLE_FULL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_full_model"
 32EXAMPLE_INCREMENTAL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_incremental_model"
 33
 34
 35EXAMPLE_FULL_MODEL_DEF = f"""MODEL (
 36  name {EXAMPLE_FULL_MODEL_NAME},
 37  kind FULL,
 38  cron '@daily',
 39  audits [assert_positive_order_ids],
 40);
 41
 42SELECT
 43  item_id,
 44  count(distinct id) AS num_orders,
 45FROM
 46    {EXAMPLE_INCREMENTAL_MODEL_NAME}
 47GROUP BY item_id
 48"""
 49
 50EXAMPLE_INCREMENTAL_MODEL_DEF = f"""MODEL (
 51    name {EXAMPLE_INCREMENTAL_MODEL_NAME},
 52    kind INCREMENTAL_BY_TIME_RANGE (
 53        time_column ds
 54    ),
 55    start '2020-01-01',
 56    cron '@daily',
 57);
 58
 59SELECT
 60    id,
 61    item_id,
 62    ds,
 63FROM
 64    (VALUES
 65        (1, 1, '2020-01-01'),
 66        (1, 2, '2020-01-01'),
 67        (2, 1, '2020-01-01'),
 68        (3, 3, '2020-01-03'),
 69        (4, 1, '2020-01-04'),
 70        (5, 1, '2020-01-05'),
 71        (6, 1, '2020-01-06'),
 72        (7, 1, '2020-01-07')
 73    ) AS t (id, item_id, ds)
 74WHERE
 75    ds between @start_ds and @end_ds
 76"""
 77
 78EXAMPLE_AUDIT = f"""AUDIT (
 79  name assert_positive_order_ids,
 80);
 81
 82SELECT *
 83FROM @this_model
 84WHERE
 85  item_id < 0
 86"""
 87
 88
 89EXAMPLE_TEST = f"""test_example_full_model:
 90  model: {EXAMPLE_FULL_MODEL_NAME}
 91  inputs:
 92    {EXAMPLE_INCREMENTAL_MODEL_NAME}:
 93        rows:
 94        - id: 1
 95          item_id: 1
 96          ds: '2020-01-01'
 97        - id: 2
 98          item_id: 1
 99          ds: '2020-01-02'
100        - id: 3
101          item_id: 2
102          ds: '2020-01-03'
103  outputs:
104    query:
105      rows:
106      - item_id: 1
107        num_orders: 2
108      - item_id: 2
109        num_orders: 1
110"""
111
112
113class ProjectTemplate(Enum):
114    AIRFLOW = "airflow"
115    DBT = "dbt"
116    DEFAULT = "default"
117
118
119DEFAULT_CONFIGS = {
120    ProjectTemplate.AIRFLOW: DEFAULT_AIRFLOW_CONFIG,
121    ProjectTemplate.DBT: DEFAULT_DBT_CONFIG,
122    ProjectTemplate.DEFAULT: DEFAULT_CONFIG,
123}
124
125
126def init_example_project(
127    path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT
128) -> None:
129    root_path = Path(path)
130    config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
131    config_path = root_path / f"config.{config_extension}"
132    audits_path = root_path / "audits"
133    macros_path = root_path / "macros"
134    models_path = root_path / "models"
135    tests_path = root_path / "tests"
136
137    if config_path.exists():
138        raise click.ClickException(f"Found an existing config in '{config_path}'")
139
140    _create_config(config_path, template)
141    if template == ProjectTemplate.DBT:
142        return
143
144    _create_folders([audits_path, macros_path, models_path, tests_path])
145    _create_macros(macros_path)
146    _create_audits(audits_path)
147    _create_models(models_path)
148    _create_tests(tests_path)
149
150
151def _create_folders(target_folders: t.Sequence[Path]) -> None:
152    for folder_path in target_folders:
153        folder_path.mkdir(exist_ok=True)
154        (folder_path / ".gitkeep").touch()
155
156
157def _create_config(config_path: Path, template: ProjectTemplate) -> None:
158    _write_file(
159        config_path,
160        DEFAULT_CONFIGS[template],
161    )
162
163
164def _create_macros(macros_path: Path) -> None:
165    (macros_path / "__init__.py").touch()
166
167
168def _create_audits(audits_path: Path) -> None:
169    _write_file(audits_path / "example_full_model.sql", EXAMPLE_AUDIT)
170
171
172def _create_models(models_path: Path) -> None:
173    for model_name, model_def in [
174        (EXAMPLE_FULL_MODEL_NAME, EXAMPLE_FULL_MODEL_DEF),
175        (EXAMPLE_INCREMENTAL_MODEL_NAME, EXAMPLE_INCREMENTAL_MODEL_DEF),
176    ]:
177        _write_file(models_path / f"{model_name.split('.')[-1]}.sql", model_def)
178
179
180def _create_tests(tests_path: Path) -> None:
181    _write_file(tests_path / "test_example_full_model.yaml", EXAMPLE_TEST)
182
183
184def _write_file(path: Path, payload: str) -> None:
185    with open(path, "w", encoding="utf-8") as fd:
186        fd.write(payload)
class ProjectTemplate(enum.Enum):
114class ProjectTemplate(Enum):
115    AIRFLOW = "airflow"
116    DBT = "dbt"
117    DEFAULT = "default"

An enumeration.

AIRFLOW = <ProjectTemplate.AIRFLOW: 'airflow'>
DBT = <ProjectTemplate.DBT: 'dbt'>
DEFAULT = <ProjectTemplate.DEFAULT: 'default'>
Inherited Members
enum.Enum
name
value
def init_example_project( path: Union[str, pathlib.Path], template: sqlmesh.cli.example_project.ProjectTemplate = <ProjectTemplate.DEFAULT: 'default'>) -> None:
127def init_example_project(
128    path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT
129) -> None:
130    root_path = Path(path)
131    config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
132    config_path = root_path / f"config.{config_extension}"
133    audits_path = root_path / "audits"
134    macros_path = root_path / "macros"
135    models_path = root_path / "models"
136    tests_path = root_path / "tests"
137
138    if config_path.exists():
139        raise click.ClickException(f"Found an existing config in '{config_path}'")
140
141    _create_config(config_path, template)
142    if template == ProjectTemplate.DBT:
143        return
144
145    _create_folders([audits_path, macros_path, models_path, tests_path])
146    _create_macros(macros_path)
147    _create_audits(audits_path)
148    _create_models(models_path)
149    _create_tests(tests_path)