sqlmesh.engines.commands
1import typing as t 2from enum import Enum 3 4from sqlmesh.core.environment import Environment 5from sqlmesh.core.snapshot import ( 6 Snapshot, 7 SnapshotEvaluator, 8 SnapshotId, 9 SnapshotTableInfo, 10) 11from sqlmesh.utils.date import TimeLike 12from sqlmesh.utils.pydantic import PydanticModel 13 14COMMAND_PAYLOAD_FILE_NAME = "payload.json" 15 16 17class CommandType(str, Enum): 18 EVALUATE = "evaluate" 19 PROMOTE = "promote" 20 DEMOTE = "demote" 21 CLEANUP = "cleanup" 22 CREATE_TABLES = "create_tables" 23 MIGRATE_TABLES = "migrate_tables" 24 25 # This makes it easy to integrate with argparse 26 def __str__(self) -> str: 27 return self.value 28 29 30class EvaluateCommandPayload(PydanticModel): 31 snapshot: Snapshot 32 parent_snapshots: t.Dict[str, Snapshot] 33 start: TimeLike 34 end: TimeLike 35 latest: TimeLike 36 is_dev: bool 37 38 39class PromoteCommandPayload(PydanticModel): 40 snapshots: t.List[SnapshotTableInfo] 41 environment: str 42 is_dev: bool 43 44 45class DemoteCommandPayload(PydanticModel): 46 snapshots: t.List[SnapshotTableInfo] 47 environment: str 48 49 50class CleanupCommandPayload(PydanticModel): 51 environments: t.List[Environment] 52 snapshots: t.List[SnapshotTableInfo] 53 54 55class CreateTablesCommandPayload(PydanticModel): 56 target_snapshot_ids: t.List[SnapshotId] 57 snapshots: t.List[Snapshot] 58 59 60class MigrateTablesCommandPayload(PydanticModel): 61 snapshots: t.List[SnapshotTableInfo] 62 63 64def evaluate( 65 evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload] 66) -> None: 67 if isinstance(command_payload, str): 68 command_payload = EvaluateCommandPayload.parse_raw(command_payload) 69 70 parent_snapshots = command_payload.parent_snapshots 71 parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot 72 73 evaluator.evaluate( 74 command_payload.snapshot, 75 command_payload.start, 76 command_payload.end, 77 command_payload.latest, 78 snapshots=parent_snapshots, 79 is_dev=command_payload.is_dev, 80 ) 81 evaluator.audit( 82 snapshot=command_payload.snapshot, 83 start=command_payload.start, 84 end=command_payload.end, 85 latest=command_payload.latest, 86 snapshots=parent_snapshots, 87 is_dev=command_payload.is_dev, 88 ) 89 90 91def promote( 92 evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload] 93) -> None: 94 if isinstance(command_payload, str): 95 command_payload = PromoteCommandPayload.parse_raw(command_payload) 96 evaluator.promote( 97 command_payload.snapshots, 98 command_payload.environment, 99 is_dev=command_payload.is_dev, 100 ) 101 102 103def demote( 104 evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload] 105) -> None: 106 if isinstance(command_payload, str): 107 command_payload = DemoteCommandPayload.parse_raw(command_payload) 108 evaluator.demote(command_payload.snapshots, command_payload.environment) 109 110 111def cleanup( 112 evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload] 113) -> None: 114 if isinstance(command_payload, str): 115 command_payload = CleanupCommandPayload.parse_raw(command_payload) 116 for environment in command_payload.environments: 117 evaluator.demote(environment.snapshots, environment.name) 118 evaluator.cleanup(command_payload.snapshots) 119 120 121def create_tables( 122 evaluator: SnapshotEvaluator, 123 command_payload: t.Union[str, CreateTablesCommandPayload], 124) -> None: 125 if isinstance(command_payload, str): 126 command_payload = CreateTablesCommandPayload.parse_raw(command_payload) 127 128 snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots} 129 target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids] 130 evaluator.create(target_snapshots, snapshots_by_id) 131 132 133def migrate_tables( 134 evaluator: SnapshotEvaluator, 135 command_payload: t.Union[str, MigrateTablesCommandPayload], 136) -> None: 137 if isinstance(command_payload, str): 138 command_payload = MigrateTablesCommandPayload.parse_raw(command_payload) 139 evaluator.migrate(command_payload.snapshots) 140 141 142COMMAND_HANDLERS: t.Dict[CommandType, t.Callable[[SnapshotEvaluator, str], None]] = { 143 CommandType.EVALUATE: evaluate, 144 CommandType.PROMOTE: promote, 145 CommandType.DEMOTE: demote, 146 CommandType.CLEANUP: cleanup, 147 CommandType.CREATE_TABLES: create_tables, 148 CommandType.MIGRATE_TABLES: migrate_tables, 149}
class
CommandType(builtins.str, enum.Enum):
18class CommandType(str, Enum): 19 EVALUATE = "evaluate" 20 PROMOTE = "promote" 21 DEMOTE = "demote" 22 CLEANUP = "cleanup" 23 CREATE_TABLES = "create_tables" 24 MIGRATE_TABLES = "migrate_tables" 25 26 # This makes it easy to integrate with argparse 27 def __str__(self) -> str: 28 return self.value
An enumeration.
EVALUATE = <CommandType.EVALUATE: 'evaluate'>
PROMOTE = <CommandType.PROMOTE: 'promote'>
DEMOTE = <CommandType.DEMOTE: 'demote'>
CLEANUP = <CommandType.CLEANUP: 'cleanup'>
CREATE_TABLES = <CommandType.CREATE_TABLES: 'create_tables'>
MIGRATE_TABLES = <CommandType.MIGRATE_TABLES: 'migrate_tables'>
Inherited Members
- enum.Enum
- name
- value
- builtins.str
- encode
- replace
- split
- rsplit
- join
- capitalize
- casefold
- title
- center
- count
- expandtabs
- find
- partition
- index
- ljust
- lower
- lstrip
- rfind
- rindex
- rjust
- rstrip
- rpartition
- splitlines
- strip
- swapcase
- translate
- upper
- startswith
- endswith
- removeprefix
- removesuffix
- isascii
- islower
- isupper
- istitle
- isspace
- isdecimal
- isdigit
- isnumeric
- isalpha
- isalnum
- isidentifier
- isprintable
- zfill
- format
- format_map
- maketrans
31class EvaluateCommandPayload(PydanticModel): 32 snapshot: Snapshot 33 parent_snapshots: t.Dict[str, Snapshot] 34 start: TimeLike 35 end: TimeLike 36 latest: TimeLike 37 is_dev: bool
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
40class PromoteCommandPayload(PydanticModel): 41 snapshots: t.List[SnapshotTableInfo] 42 environment: str 43 is_dev: bool
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
46class DemoteCommandPayload(PydanticModel): 47 snapshots: t.List[SnapshotTableInfo] 48 environment: str
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
51class CleanupCommandPayload(PydanticModel): 52 environments: t.List[Environment] 53 snapshots: t.List[SnapshotTableInfo]
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
56class CreateTablesCommandPayload(PydanticModel): 57 target_snapshot_ids: t.List[SnapshotId] 58 snapshots: t.List[Snapshot]
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
Inherited Members
- pydantic.main.BaseModel
- BaseModel
- parse_obj
- parse_raw
- parse_file
- from_orm
- construct
- copy
- schema
- schema_json
- validate
- update_forward_refs
def
evaluate( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.EvaluateCommandPayload]) -> None:
65def evaluate( 66 evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload] 67) -> None: 68 if isinstance(command_payload, str): 69 command_payload = EvaluateCommandPayload.parse_raw(command_payload) 70 71 parent_snapshots = command_payload.parent_snapshots 72 parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot 73 74 evaluator.evaluate( 75 command_payload.snapshot, 76 command_payload.start, 77 command_payload.end, 78 command_payload.latest, 79 snapshots=parent_snapshots, 80 is_dev=command_payload.is_dev, 81 ) 82 evaluator.audit( 83 snapshot=command_payload.snapshot, 84 start=command_payload.start, 85 end=command_payload.end, 86 latest=command_payload.latest, 87 snapshots=parent_snapshots, 88 is_dev=command_payload.is_dev, 89 )
def
promote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.PromoteCommandPayload]) -> None:
92def promote( 93 evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload] 94) -> None: 95 if isinstance(command_payload, str): 96 command_payload = PromoteCommandPayload.parse_raw(command_payload) 97 evaluator.promote( 98 command_payload.snapshots, 99 command_payload.environment, 100 is_dev=command_payload.is_dev, 101 )
def
demote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.DemoteCommandPayload]) -> None:
104def demote( 105 evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload] 106) -> None: 107 if isinstance(command_payload, str): 108 command_payload = DemoteCommandPayload.parse_raw(command_payload) 109 evaluator.demote(command_payload.snapshots, command_payload.environment)
def
cleanup( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CleanupCommandPayload]) -> None:
112def cleanup( 113 evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload] 114) -> None: 115 if isinstance(command_payload, str): 116 command_payload = CleanupCommandPayload.parse_raw(command_payload) 117 for environment in command_payload.environments: 118 evaluator.demote(environment.snapshots, environment.name) 119 evaluator.cleanup(command_payload.snapshots)
def
create_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CreateTablesCommandPayload]) -> None:
122def create_tables( 123 evaluator: SnapshotEvaluator, 124 command_payload: t.Union[str, CreateTablesCommandPayload], 125) -> None: 126 if isinstance(command_payload, str): 127 command_payload = CreateTablesCommandPayload.parse_raw(command_payload) 128 129 snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots} 130 target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids] 131 evaluator.create(target_snapshots, snapshots_by_id)
def
migrate_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.MigrateTablesCommandPayload]) -> None:
134def migrate_tables( 135 evaluator: SnapshotEvaluator, 136 command_payload: t.Union[str, MigrateTablesCommandPayload], 137) -> None: 138 if isinstance(command_payload, str): 139 command_payload = MigrateTablesCommandPayload.parse_raw(command_payload) 140 evaluator.migrate(command_payload.snapshots)