Edit on GitHub

sqlmesh.engines.commands

  1import typing as t
  2from enum import Enum
  3
  4from sqlmesh.core.environment import Environment
  5from sqlmesh.core.snapshot import (
  6    Snapshot,
  7    SnapshotEvaluator,
  8    SnapshotId,
  9    SnapshotTableInfo,
 10)
 11from sqlmesh.utils.date import TimeLike
 12from sqlmesh.utils.pydantic import PydanticModel
 13
 14COMMAND_PAYLOAD_FILE_NAME = "payload.json"
 15
 16
 17class CommandType(str, Enum):
 18    EVALUATE = "evaluate"
 19    PROMOTE = "promote"
 20    DEMOTE = "demote"
 21    CLEANUP = "cleanup"
 22    CREATE_TABLES = "create_tables"
 23    MIGRATE_TABLES = "migrate_tables"
 24
 25    # This makes it easy to integrate with argparse
 26    def __str__(self) -> str:
 27        return self.value
 28
 29
 30class EvaluateCommandPayload(PydanticModel):
 31    snapshot: Snapshot
 32    parent_snapshots: t.Dict[str, Snapshot]
 33    start: TimeLike
 34    end: TimeLike
 35    latest: TimeLike
 36    is_dev: bool
 37
 38
 39class PromoteCommandPayload(PydanticModel):
 40    snapshots: t.List[SnapshotTableInfo]
 41    environment: str
 42    is_dev: bool
 43
 44
 45class DemoteCommandPayload(PydanticModel):
 46    snapshots: t.List[SnapshotTableInfo]
 47    environment: str
 48
 49
 50class CleanupCommandPayload(PydanticModel):
 51    environments: t.List[Environment]
 52    snapshots: t.List[SnapshotTableInfo]
 53
 54
 55class CreateTablesCommandPayload(PydanticModel):
 56    target_snapshot_ids: t.List[SnapshotId]
 57    snapshots: t.List[Snapshot]
 58
 59
 60class MigrateTablesCommandPayload(PydanticModel):
 61    snapshots: t.List[SnapshotTableInfo]
 62
 63
 64def evaluate(
 65    evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload]
 66) -> None:
 67    if isinstance(command_payload, str):
 68        command_payload = EvaluateCommandPayload.parse_raw(command_payload)
 69
 70    parent_snapshots = command_payload.parent_snapshots
 71    parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot
 72
 73    evaluator.evaluate(
 74        command_payload.snapshot,
 75        command_payload.start,
 76        command_payload.end,
 77        command_payload.latest,
 78        snapshots=parent_snapshots,
 79        is_dev=command_payload.is_dev,
 80    )
 81    evaluator.audit(
 82        snapshot=command_payload.snapshot,
 83        start=command_payload.start,
 84        end=command_payload.end,
 85        latest=command_payload.latest,
 86        snapshots=parent_snapshots,
 87        is_dev=command_payload.is_dev,
 88    )
 89
 90
 91def promote(
 92    evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload]
 93) -> None:
 94    if isinstance(command_payload, str):
 95        command_payload = PromoteCommandPayload.parse_raw(command_payload)
 96    evaluator.promote(
 97        command_payload.snapshots,
 98        command_payload.environment,
 99        is_dev=command_payload.is_dev,
100    )
101
102
103def demote(
104    evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload]
105) -> None:
106    if isinstance(command_payload, str):
107        command_payload = DemoteCommandPayload.parse_raw(command_payload)
108    evaluator.demote(command_payload.snapshots, command_payload.environment)
109
110
111def cleanup(
112    evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload]
113) -> None:
114    if isinstance(command_payload, str):
115        command_payload = CleanupCommandPayload.parse_raw(command_payload)
116    for environment in command_payload.environments:
117        evaluator.demote(environment.snapshots, environment.name)
118    evaluator.cleanup(command_payload.snapshots)
119
120
121def create_tables(
122    evaluator: SnapshotEvaluator,
123    command_payload: t.Union[str, CreateTablesCommandPayload],
124) -> None:
125    if isinstance(command_payload, str):
126        command_payload = CreateTablesCommandPayload.parse_raw(command_payload)
127
128    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
129    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
130    evaluator.create(target_snapshots, snapshots_by_id)
131
132
133def migrate_tables(
134    evaluator: SnapshotEvaluator,
135    command_payload: t.Union[str, MigrateTablesCommandPayload],
136) -> None:
137    if isinstance(command_payload, str):
138        command_payload = MigrateTablesCommandPayload.parse_raw(command_payload)
139    evaluator.migrate(command_payload.snapshots)
140
141
142COMMAND_HANDLERS: t.Dict[CommandType, t.Callable[[SnapshotEvaluator, str], None]] = {
143    CommandType.EVALUATE: evaluate,
144    CommandType.PROMOTE: promote,
145    CommandType.DEMOTE: demote,
146    CommandType.CLEANUP: cleanup,
147    CommandType.CREATE_TABLES: create_tables,
148    CommandType.MIGRATE_TABLES: migrate_tables,
149}
class CommandType(builtins.str, enum.Enum):
18class CommandType(str, Enum):
19    EVALUATE = "evaluate"
20    PROMOTE = "promote"
21    DEMOTE = "demote"
22    CLEANUP = "cleanup"
23    CREATE_TABLES = "create_tables"
24    MIGRATE_TABLES = "migrate_tables"
25
26    # This makes it easy to integrate with argparse
27    def __str__(self) -> str:
28        return self.value

An enumeration.

EVALUATE = <CommandType.EVALUATE: 'evaluate'>
PROMOTE = <CommandType.PROMOTE: 'promote'>
DEMOTE = <CommandType.DEMOTE: 'demote'>
CLEANUP = <CommandType.CLEANUP: 'cleanup'>
CREATE_TABLES = <CommandType.CREATE_TABLES: 'create_tables'>
MIGRATE_TABLES = <CommandType.MIGRATE_TABLES: 'migrate_tables'>
Inherited Members
enum.Enum
name
value
builtins.str
encode
replace
split
rsplit
join
capitalize
casefold
title
center
count
expandtabs
find
partition
index
ljust
lower
lstrip
rfind
rindex
rjust
rstrip
rpartition
splitlines
strip
swapcase
translate
upper
startswith
endswith
removeprefix
removesuffix
isascii
islower
isupper
istitle
isspace
isdecimal
isdigit
isnumeric
isalpha
isalnum
isidentifier
isprintable
zfill
format
format_map
maketrans
class EvaluateCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
31class EvaluateCommandPayload(PydanticModel):
32    snapshot: Snapshot
33    parent_snapshots: t.Dict[str, Snapshot]
34    start: TimeLike
35    end: TimeLike
36    latest: TimeLike
37    is_dev: bool
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class PromoteCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
40class PromoteCommandPayload(PydanticModel):
41    snapshots: t.List[SnapshotTableInfo]
42    environment: str
43    is_dev: bool
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class DemoteCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
46class DemoteCommandPayload(PydanticModel):
47    snapshots: t.List[SnapshotTableInfo]
48    environment: str
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class CleanupCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
51class CleanupCommandPayload(PydanticModel):
52    environments: t.List[Environment]
53    snapshots: t.List[SnapshotTableInfo]
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class CreateTablesCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
56class CreateTablesCommandPayload(PydanticModel):
57    target_snapshot_ids: t.List[SnapshotId]
58    snapshots: t.List[Snapshot]
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
class MigrateTablesCommandPayload(sqlmesh.utils.pydantic.PydanticModel):
61class MigrateTablesCommandPayload(PydanticModel):
62    snapshots: t.List[SnapshotTableInfo]
Inherited Members
pydantic.main.BaseModel
BaseModel
parse_obj
parse_raw
parse_file
from_orm
construct
copy
schema
schema_json
validate
update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
Config
dict
json
missing_required_fields
extra_fields
all_fields
required_fields
def evaluate( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.EvaluateCommandPayload]) -> None:
65def evaluate(
66    evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload]
67) -> None:
68    if isinstance(command_payload, str):
69        command_payload = EvaluateCommandPayload.parse_raw(command_payload)
70
71    parent_snapshots = command_payload.parent_snapshots
72    parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot
73
74    evaluator.evaluate(
75        command_payload.snapshot,
76        command_payload.start,
77        command_payload.end,
78        command_payload.latest,
79        snapshots=parent_snapshots,
80        is_dev=command_payload.is_dev,
81    )
82    evaluator.audit(
83        snapshot=command_payload.snapshot,
84        start=command_payload.start,
85        end=command_payload.end,
86        latest=command_payload.latest,
87        snapshots=parent_snapshots,
88        is_dev=command_payload.is_dev,
89    )
def promote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.PromoteCommandPayload]) -> None:
 92def promote(
 93    evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload]
 94) -> None:
 95    if isinstance(command_payload, str):
 96        command_payload = PromoteCommandPayload.parse_raw(command_payload)
 97    evaluator.promote(
 98        command_payload.snapshots,
 99        command_payload.environment,
100        is_dev=command_payload.is_dev,
101    )
def demote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.DemoteCommandPayload]) -> None:
104def demote(
105    evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload]
106) -> None:
107    if isinstance(command_payload, str):
108        command_payload = DemoteCommandPayload.parse_raw(command_payload)
109    evaluator.demote(command_payload.snapshots, command_payload.environment)
def cleanup( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CleanupCommandPayload]) -> None:
112def cleanup(
113    evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload]
114) -> None:
115    if isinstance(command_payload, str):
116        command_payload = CleanupCommandPayload.parse_raw(command_payload)
117    for environment in command_payload.environments:
118        evaluator.demote(environment.snapshots, environment.name)
119    evaluator.cleanup(command_payload.snapshots)
def create_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CreateTablesCommandPayload]) -> None:
122def create_tables(
123    evaluator: SnapshotEvaluator,
124    command_payload: t.Union[str, CreateTablesCommandPayload],
125) -> None:
126    if isinstance(command_payload, str):
127        command_payload = CreateTablesCommandPayload.parse_raw(command_payload)
128
129    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
130    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
131    evaluator.create(target_snapshots, snapshots_by_id)
def migrate_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.MigrateTablesCommandPayload]) -> None:
134def migrate_tables(
135    evaluator: SnapshotEvaluator,
136    command_payload: t.Union[str, MigrateTablesCommandPayload],
137) -> None:
138    if isinstance(command_payload, str):
139        command_payload = MigrateTablesCommandPayload.parse_raw(command_payload)
140    evaluator.migrate(command_payload.snapshots)