sqlmesh.core.state_sync.base
1from __future__ import annotations 2 3import abc 4import typing as t 5 6from sqlmesh.core import scheduler 7from sqlmesh.core.environment import Environment 8from sqlmesh.core.snapshot import ( 9 Intervals, 10 Snapshot, 11 SnapshotId, 12 SnapshotIdLike, 13 SnapshotInfoLike, 14 SnapshotNameVersionLike, 15 SnapshotTableInfo, 16) 17from sqlmesh.utils.date import TimeLike, now, to_datetime 18from sqlmesh.utils.errors import SQLMeshError 19 20 21class StateReader(abc.ABC): 22 """Abstract base class for read-only operations on snapshot and environment state.""" 23 24 @abc.abstractmethod 25 def get_snapshots( 26 self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]] 27 ) -> t.Dict[SnapshotId, Snapshot]: 28 """Bulk fetch snapshots given the corresponding snapshot ids. 29 30 Args: 31 snapshot_ids: Iterable of snapshot ids to get. If not provided all 32 available snapshots will be returned. 33 34 Returns: 35 A dictionary of snapshot ids to snapshots for ones that could be found. 36 """ 37 38 @abc.abstractmethod 39 def get_snapshots_with_same_version( 40 self, snapshots: t.Iterable[SnapshotNameVersionLike] 41 ) -> t.List[Snapshot]: 42 """Fetches all snapshots that share the same version as the snapshots. 43 44 The output includes the snapshots with the specified version. 45 46 Args: 47 snapshots: The collection of target name / version pairs. 48 49 Returns: 50 The list of Snapshot objects. 51 """ 52 53 @abc.abstractmethod 54 def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]: 55 """Checks if multiple snapshots exist in the state sync. 56 57 Args: 58 snapshot_ids: Iterable of snapshot ids to bulk check. 59 60 Returns: 61 A set of all the existing snapshot ids. 62 """ 63 64 @abc.abstractmethod 65 def get_environment(self, environment: str) -> t.Optional[Environment]: 66 """Fetches the environment if it exists. 67 68 Args: 69 environment: The environment 70 71 Returns: 72 The environment object. 73 """ 74 75 @abc.abstractmethod 76 def get_environments(self) -> t.List[Environment]: 77 """Fetches all environments. 78 79 Returns: 80 A list of all environments. 81 """ 82 83 @abc.abstractmethod 84 def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]: 85 """Get all snapshots by model name. 86 87 Returns: 88 The list of snapshots. 89 """ 90 91 def missing_intervals( 92 self, 93 env_or_snapshots: str | Environment | t.Iterable[Snapshot], 94 start: t.Optional[TimeLike] = None, 95 end: t.Optional[TimeLike] = None, 96 latest: t.Optional[TimeLike] = None, 97 restatements: t.Optional[t.Iterable[str]] = None, 98 ) -> t.Dict[Snapshot, Intervals]: 99 """Find missing intervals for an environment or a list of snapshots. 100 101 Args: 102 env_or_snapshots: The environment or snapshots to find missing intervals for. 103 start: The start of the time range to look for. 104 end: The end of the time range to look for. 105 latest: The latest datetime to use for non-incremental queries. 106 107 Returns: 108 A dictionary of SnapshotId to Intervals. 109 """ 110 111 if isinstance(env_or_snapshots, str): 112 env = self.get_environment(env_or_snapshots) 113 elif isinstance(env_or_snapshots, Environment): 114 env = env_or_snapshots 115 else: 116 env = None 117 118 if env: 119 snapshots_by_id = self.get_snapshots(env.snapshots) 120 start = start or env.start_at 121 end = end or env.end_at 122 elif isinstance(env_or_snapshots, str): 123 snapshots_by_id = {} 124 elif not isinstance(env_or_snapshots, Environment): 125 snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots} 126 else: 127 raise SQLMeshError("This shouldn't be possible.") 128 129 if not snapshots_by_id: 130 return {} 131 132 unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version] 133 134 snapshots_by_id = { 135 **snapshots_by_id, 136 **(self.get_snapshots(unversioned) if unversioned else {}), 137 } 138 139 all_snapshots = { 140 **snapshots_by_id, 141 **{ 142 snapshot.snapshot_id: snapshot 143 for snapshot in self.get_snapshots_with_same_version( 144 snapshot for snapshot in snapshots_by_id.values() if snapshot.version 145 ) 146 }, 147 } 148 149 missing = {} 150 start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values())) 151 end_date = end or now() 152 restatements = set(restatements or []) 153 154 for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots): 155 if snapshot.name in restatements: 156 snapshot.remove_interval(start_date, end_date) 157 intervals = snapshot.missing_intervals( 158 max( 159 start_date, 160 to_datetime( 161 scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date 162 ), 163 ), 164 end_date, 165 latest=latest, 166 ) 167 if intervals: 168 missing[snapshot] = intervals 169 return missing 170 171 172class StateSync(StateReader, abc.ABC): 173 """Abstract base class for snapshot and environment state management.""" 174 175 @abc.abstractmethod 176 def init_schema(self) -> None: 177 """Optional initialization of the sync.""" 178 179 @abc.abstractmethod 180 def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None: 181 """Push snapshots into the state sync. 182 183 This method only allows for pushing new snapshots. If existing snapshots are found, 184 this method should raise an error. 185 186 Raises: 187 SQLMeshError when existing snapshots are pushed. 188 189 Args: 190 snapshots: A list of snapshots to save in the state sync. 191 """ 192 193 @abc.abstractmethod 194 def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None: 195 """Delete snapshots from the state sync. 196 197 Args: 198 snapshot_ids: A list of snapshot like objects to delete. 199 """ 200 201 @abc.abstractmethod 202 def delete_expired_snapshots(self) -> t.List[Snapshot]: 203 """Removes expired snapshots. 204 205 Expired snapshots are snapshots that have exceeded their time-to-live 206 and are no longer in use within an environment. 207 208 Returns: 209 The list of removed snapshots. 210 """ 211 212 @abc.abstractmethod 213 def add_interval( 214 self, 215 snapshot_id: SnapshotIdLike, 216 start: TimeLike, 217 end: TimeLike, 218 is_dev: bool = False, 219 ) -> None: 220 """Add an interval to a snapshot and sync it to the store. 221 222 Snapshots must be pushed before adding intervals to them. 223 224 Args: 225 snapshot_id: The snapshot like object to add an interval to. 226 start: The start of the interval to add. 227 end: The end of the interval to add. 228 is_dev: Indicates whether the given interval is being added while in 229 development mode. 230 """ 231 232 @abc.abstractmethod 233 def remove_interval( 234 self, 235 snapshots: t.Iterable[SnapshotInfoLike], 236 start: TimeLike, 237 end: TimeLike, 238 all_snapshots: t.Optional[t.Iterable[Snapshot]] = None, 239 ) -> None: 240 """Remove an interval from a list of snapshots and sync it to the store. 241 242 Because multiple snapshots can be pointing to the same version or physical table, this method 243 can also grab all snapshots tied to the passed in version. 244 245 Args: 246 snapshots: The snapshot info like object to remove intervals from. 247 start: The start of the interval to add. 248 end: The end of the interval to add. 249 all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions. 250 """ 251 252 @abc.abstractmethod 253 def promote( 254 self, environment: Environment, no_gaps: bool = False 255 ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]: 256 """Update the environment to reflect the current state. 257 258 This method verifies that snapshots have been pushed. 259 260 Args: 261 environment: The environment to promote. 262 no_gaps: Whether to ensure that new snapshots for models that are already a 263 part of the target environment have no data gaps when compared against previous 264 snapshots for same models. 265 266 Returns: 267 A tuple of (added snapshot table infos, removed snapshot table infos) 268 """ 269 270 @abc.abstractmethod 271 def delete_expired_environments(self) -> t.List[Environment]: 272 """Removes expired environments. 273 274 Expired environments are environments that have exceeded their time-to-live value. 275 276 Returns: 277 The list of removed environments. 278 """ 279 280 @abc.abstractmethod 281 def unpause_snapshots( 282 self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike 283 ) -> None: 284 """Unpauses target snapshots. 285 286 Unpaused snapshots are scheduled for evaluation on a recurring basis. 287 Once unpaused a snapshot can't be paused again. 288 289 Args: 290 snapshots: Target snapshots. 291 unpaused_dt: The datetime object which indicates when target snapshots 292 were unpaused. 293 """
22class StateReader(abc.ABC): 23 """Abstract base class for read-only operations on snapshot and environment state.""" 24 25 @abc.abstractmethod 26 def get_snapshots( 27 self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]] 28 ) -> t.Dict[SnapshotId, Snapshot]: 29 """Bulk fetch snapshots given the corresponding snapshot ids. 30 31 Args: 32 snapshot_ids: Iterable of snapshot ids to get. If not provided all 33 available snapshots will be returned. 34 35 Returns: 36 A dictionary of snapshot ids to snapshots for ones that could be found. 37 """ 38 39 @abc.abstractmethod 40 def get_snapshots_with_same_version( 41 self, snapshots: t.Iterable[SnapshotNameVersionLike] 42 ) -> t.List[Snapshot]: 43 """Fetches all snapshots that share the same version as the snapshots. 44 45 The output includes the snapshots with the specified version. 46 47 Args: 48 snapshots: The collection of target name / version pairs. 49 50 Returns: 51 The list of Snapshot objects. 52 """ 53 54 @abc.abstractmethod 55 def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]: 56 """Checks if multiple snapshots exist in the state sync. 57 58 Args: 59 snapshot_ids: Iterable of snapshot ids to bulk check. 60 61 Returns: 62 A set of all the existing snapshot ids. 63 """ 64 65 @abc.abstractmethod 66 def get_environment(self, environment: str) -> t.Optional[Environment]: 67 """Fetches the environment if it exists. 68 69 Args: 70 environment: The environment 71 72 Returns: 73 The environment object. 74 """ 75 76 @abc.abstractmethod 77 def get_environments(self) -> t.List[Environment]: 78 """Fetches all environments. 79 80 Returns: 81 A list of all environments. 82 """ 83 84 @abc.abstractmethod 85 def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]: 86 """Get all snapshots by model name. 87 88 Returns: 89 The list of snapshots. 90 """ 91 92 def missing_intervals( 93 self, 94 env_or_snapshots: str | Environment | t.Iterable[Snapshot], 95 start: t.Optional[TimeLike] = None, 96 end: t.Optional[TimeLike] = None, 97 latest: t.Optional[TimeLike] = None, 98 restatements: t.Optional[t.Iterable[str]] = None, 99 ) -> t.Dict[Snapshot, Intervals]: 100 """Find missing intervals for an environment or a list of snapshots. 101 102 Args: 103 env_or_snapshots: The environment or snapshots to find missing intervals for. 104 start: The start of the time range to look for. 105 end: The end of the time range to look for. 106 latest: The latest datetime to use for non-incremental queries. 107 108 Returns: 109 A dictionary of SnapshotId to Intervals. 110 """ 111 112 if isinstance(env_or_snapshots, str): 113 env = self.get_environment(env_or_snapshots) 114 elif isinstance(env_or_snapshots, Environment): 115 env = env_or_snapshots 116 else: 117 env = None 118 119 if env: 120 snapshots_by_id = self.get_snapshots(env.snapshots) 121 start = start or env.start_at 122 end = end or env.end_at 123 elif isinstance(env_or_snapshots, str): 124 snapshots_by_id = {} 125 elif not isinstance(env_or_snapshots, Environment): 126 snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots} 127 else: 128 raise SQLMeshError("This shouldn't be possible.") 129 130 if not snapshots_by_id: 131 return {} 132 133 unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version] 134 135 snapshots_by_id = { 136 **snapshots_by_id, 137 **(self.get_snapshots(unversioned) if unversioned else {}), 138 } 139 140 all_snapshots = { 141 **snapshots_by_id, 142 **{ 143 snapshot.snapshot_id: snapshot 144 for snapshot in self.get_snapshots_with_same_version( 145 snapshot for snapshot in snapshots_by_id.values() if snapshot.version 146 ) 147 }, 148 } 149 150 missing = {} 151 start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values())) 152 end_date = end or now() 153 restatements = set(restatements or []) 154 155 for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots): 156 if snapshot.name in restatements: 157 snapshot.remove_interval(start_date, end_date) 158 intervals = snapshot.missing_intervals( 159 max( 160 start_date, 161 to_datetime( 162 scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date 163 ), 164 ), 165 end_date, 166 latest=latest, 167 ) 168 if intervals: 169 missing[snapshot] = intervals 170 return missing
Abstract base class for read-only operations on snapshot and environment state.
25 @abc.abstractmethod 26 def get_snapshots( 27 self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]] 28 ) -> t.Dict[SnapshotId, Snapshot]: 29 """Bulk fetch snapshots given the corresponding snapshot ids. 30 31 Args: 32 snapshot_ids: Iterable of snapshot ids to get. If not provided all 33 available snapshots will be returned. 34 35 Returns: 36 A dictionary of snapshot ids to snapshots for ones that could be found. 37 """
Bulk fetch snapshots given the corresponding snapshot ids.
Arguments:
- snapshot_ids: Iterable of snapshot ids to get. If not provided all available snapshots will be returned.
Returns:
A dictionary of snapshot ids to snapshots for ones that could be found.
39 @abc.abstractmethod 40 def get_snapshots_with_same_version( 41 self, snapshots: t.Iterable[SnapshotNameVersionLike] 42 ) -> t.List[Snapshot]: 43 """Fetches all snapshots that share the same version as the snapshots. 44 45 The output includes the snapshots with the specified version. 46 47 Args: 48 snapshots: The collection of target name / version pairs. 49 50 Returns: 51 The list of Snapshot objects. 52 """
Fetches all snapshots that share the same version as the snapshots.
The output includes the snapshots with the specified version.
Arguments:
- snapshots: The collection of target name / version pairs.
Returns:
The list of Snapshot objects.
54 @abc.abstractmethod 55 def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]: 56 """Checks if multiple snapshots exist in the state sync. 57 58 Args: 59 snapshot_ids: Iterable of snapshot ids to bulk check. 60 61 Returns: 62 A set of all the existing snapshot ids. 63 """
Checks if multiple snapshots exist in the state sync.
Arguments:
- snapshot_ids: Iterable of snapshot ids to bulk check.
Returns:
A set of all the existing snapshot ids.
65 @abc.abstractmethod 66 def get_environment(self, environment: str) -> t.Optional[Environment]: 67 """Fetches the environment if it exists. 68 69 Args: 70 environment: The environment 71 72 Returns: 73 The environment object. 74 """
Fetches the environment if it exists.
Arguments:
- environment: The environment
Returns:
The environment object.
76 @abc.abstractmethod 77 def get_environments(self) -> t.List[Environment]: 78 """Fetches all environments. 79 80 Returns: 81 A list of all environments. 82 """
Fetches all environments.
Returns:
A list of all environments.
84 @abc.abstractmethod 85 def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]: 86 """Get all snapshots by model name. 87 88 Returns: 89 The list of snapshots. 90 """
Get all snapshots by model name.
Returns:
The list of snapshots.
92 def missing_intervals( 93 self, 94 env_or_snapshots: str | Environment | t.Iterable[Snapshot], 95 start: t.Optional[TimeLike] = None, 96 end: t.Optional[TimeLike] = None, 97 latest: t.Optional[TimeLike] = None, 98 restatements: t.Optional[t.Iterable[str]] = None, 99 ) -> t.Dict[Snapshot, Intervals]: 100 """Find missing intervals for an environment or a list of snapshots. 101 102 Args: 103 env_or_snapshots: The environment or snapshots to find missing intervals for. 104 start: The start of the time range to look for. 105 end: The end of the time range to look for. 106 latest: The latest datetime to use for non-incremental queries. 107 108 Returns: 109 A dictionary of SnapshotId to Intervals. 110 """ 111 112 if isinstance(env_or_snapshots, str): 113 env = self.get_environment(env_or_snapshots) 114 elif isinstance(env_or_snapshots, Environment): 115 env = env_or_snapshots 116 else: 117 env = None 118 119 if env: 120 snapshots_by_id = self.get_snapshots(env.snapshots) 121 start = start or env.start_at 122 end = end or env.end_at 123 elif isinstance(env_or_snapshots, str): 124 snapshots_by_id = {} 125 elif not isinstance(env_or_snapshots, Environment): 126 snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots} 127 else: 128 raise SQLMeshError("This shouldn't be possible.") 129 130 if not snapshots_by_id: 131 return {} 132 133 unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version] 134 135 snapshots_by_id = { 136 **snapshots_by_id, 137 **(self.get_snapshots(unversioned) if unversioned else {}), 138 } 139 140 all_snapshots = { 141 **snapshots_by_id, 142 **{ 143 snapshot.snapshot_id: snapshot 144 for snapshot in self.get_snapshots_with_same_version( 145 snapshot for snapshot in snapshots_by_id.values() if snapshot.version 146 ) 147 }, 148 } 149 150 missing = {} 151 start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values())) 152 end_date = end or now() 153 restatements = set(restatements or []) 154 155 for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots): 156 if snapshot.name in restatements: 157 snapshot.remove_interval(start_date, end_date) 158 intervals = snapshot.missing_intervals( 159 max( 160 start_date, 161 to_datetime( 162 scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date 163 ), 164 ), 165 end_date, 166 latest=latest, 167 ) 168 if intervals: 169 missing[snapshot] = intervals 170 return missing
Find missing intervals for an environment or a list of snapshots.
Arguments:
- env_or_snapshots: The environment or snapshots to find missing intervals for.
- start: The start of the time range to look for.
- end: The end of the time range to look for.
- latest: The latest datetime to use for non-incremental queries.
Returns:
A dictionary of SnapshotId to Intervals.
173class StateSync(StateReader, abc.ABC): 174 """Abstract base class for snapshot and environment state management.""" 175 176 @abc.abstractmethod 177 def init_schema(self) -> None: 178 """Optional initialization of the sync.""" 179 180 @abc.abstractmethod 181 def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None: 182 """Push snapshots into the state sync. 183 184 This method only allows for pushing new snapshots. If existing snapshots are found, 185 this method should raise an error. 186 187 Raises: 188 SQLMeshError when existing snapshots are pushed. 189 190 Args: 191 snapshots: A list of snapshots to save in the state sync. 192 """ 193 194 @abc.abstractmethod 195 def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None: 196 """Delete snapshots from the state sync. 197 198 Args: 199 snapshot_ids: A list of snapshot like objects to delete. 200 """ 201 202 @abc.abstractmethod 203 def delete_expired_snapshots(self) -> t.List[Snapshot]: 204 """Removes expired snapshots. 205 206 Expired snapshots are snapshots that have exceeded their time-to-live 207 and are no longer in use within an environment. 208 209 Returns: 210 The list of removed snapshots. 211 """ 212 213 @abc.abstractmethod 214 def add_interval( 215 self, 216 snapshot_id: SnapshotIdLike, 217 start: TimeLike, 218 end: TimeLike, 219 is_dev: bool = False, 220 ) -> None: 221 """Add an interval to a snapshot and sync it to the store. 222 223 Snapshots must be pushed before adding intervals to them. 224 225 Args: 226 snapshot_id: The snapshot like object to add an interval to. 227 start: The start of the interval to add. 228 end: The end of the interval to add. 229 is_dev: Indicates whether the given interval is being added while in 230 development mode. 231 """ 232 233 @abc.abstractmethod 234 def remove_interval( 235 self, 236 snapshots: t.Iterable[SnapshotInfoLike], 237 start: TimeLike, 238 end: TimeLike, 239 all_snapshots: t.Optional[t.Iterable[Snapshot]] = None, 240 ) -> None: 241 """Remove an interval from a list of snapshots and sync it to the store. 242 243 Because multiple snapshots can be pointing to the same version or physical table, this method 244 can also grab all snapshots tied to the passed in version. 245 246 Args: 247 snapshots: The snapshot info like object to remove intervals from. 248 start: The start of the interval to add. 249 end: The end of the interval to add. 250 all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions. 251 """ 252 253 @abc.abstractmethod 254 def promote( 255 self, environment: Environment, no_gaps: bool = False 256 ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]: 257 """Update the environment to reflect the current state. 258 259 This method verifies that snapshots have been pushed. 260 261 Args: 262 environment: The environment to promote. 263 no_gaps: Whether to ensure that new snapshots for models that are already a 264 part of the target environment have no data gaps when compared against previous 265 snapshots for same models. 266 267 Returns: 268 A tuple of (added snapshot table infos, removed snapshot table infos) 269 """ 270 271 @abc.abstractmethod 272 def delete_expired_environments(self) -> t.List[Environment]: 273 """Removes expired environments. 274 275 Expired environments are environments that have exceeded their time-to-live value. 276 277 Returns: 278 The list of removed environments. 279 """ 280 281 @abc.abstractmethod 282 def unpause_snapshots( 283 self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike 284 ) -> None: 285 """Unpauses target snapshots. 286 287 Unpaused snapshots are scheduled for evaluation on a recurring basis. 288 Once unpaused a snapshot can't be paused again. 289 290 Args: 291 snapshots: Target snapshots. 292 unpaused_dt: The datetime object which indicates when target snapshots 293 were unpaused. 294 """
Abstract base class for snapshot and environment state management.
176 @abc.abstractmethod 177 def init_schema(self) -> None: 178 """Optional initialization of the sync."""
Optional initialization of the sync.
180 @abc.abstractmethod 181 def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None: 182 """Push snapshots into the state sync. 183 184 This method only allows for pushing new snapshots. If existing snapshots are found, 185 this method should raise an error. 186 187 Raises: 188 SQLMeshError when existing snapshots are pushed. 189 190 Args: 191 snapshots: A list of snapshots to save in the state sync. 192 """
Push snapshots into the state sync.
This method only allows for pushing new snapshots. If existing snapshots are found, this method should raise an error.
Raises:
- SQLMeshError when existing snapshots are pushed.
Arguments:
- snapshots: A list of snapshots to save in the state sync.
194 @abc.abstractmethod 195 def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None: 196 """Delete snapshots from the state sync. 197 198 Args: 199 snapshot_ids: A list of snapshot like objects to delete. 200 """
Delete snapshots from the state sync.
Arguments:
- snapshot_ids: A list of snapshot like objects to delete.
202 @abc.abstractmethod 203 def delete_expired_snapshots(self) -> t.List[Snapshot]: 204 """Removes expired snapshots. 205 206 Expired snapshots are snapshots that have exceeded their time-to-live 207 and are no longer in use within an environment. 208 209 Returns: 210 The list of removed snapshots. 211 """
Removes expired snapshots.
Expired snapshots are snapshots that have exceeded their time-to-live and are no longer in use within an environment.
Returns:
The list of removed snapshots.
213 @abc.abstractmethod 214 def add_interval( 215 self, 216 snapshot_id: SnapshotIdLike, 217 start: TimeLike, 218 end: TimeLike, 219 is_dev: bool = False, 220 ) -> None: 221 """Add an interval to a snapshot and sync it to the store. 222 223 Snapshots must be pushed before adding intervals to them. 224 225 Args: 226 snapshot_id: The snapshot like object to add an interval to. 227 start: The start of the interval to add. 228 end: The end of the interval to add. 229 is_dev: Indicates whether the given interval is being added while in 230 development mode. 231 """
Add an interval to a snapshot and sync it to the store.
Snapshots must be pushed before adding intervals to them.
Arguments:
- snapshot_id: The snapshot like object to add an interval to.
- start: The start of the interval to add.
- end: The end of the interval to add.
- is_dev: Indicates whether the given interval is being added while in development mode.
233 @abc.abstractmethod 234 def remove_interval( 235 self, 236 snapshots: t.Iterable[SnapshotInfoLike], 237 start: TimeLike, 238 end: TimeLike, 239 all_snapshots: t.Optional[t.Iterable[Snapshot]] = None, 240 ) -> None: 241 """Remove an interval from a list of snapshots and sync it to the store. 242 243 Because multiple snapshots can be pointing to the same version or physical table, this method 244 can also grab all snapshots tied to the passed in version. 245 246 Args: 247 snapshots: The snapshot info like object to remove intervals from. 248 start: The start of the interval to add. 249 end: The end of the interval to add. 250 all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions. 251 """
Remove an interval from a list of snapshots and sync it to the store.
Because multiple snapshots can be pointing to the same version or physical table, this method can also grab all snapshots tied to the passed in version.
Arguments:
- snapshots: The snapshot info like object to remove intervals from.
- start: The start of the interval to add.
- end: The end of the interval to add.
- all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
253 @abc.abstractmethod 254 def promote( 255 self, environment: Environment, no_gaps: bool = False 256 ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]: 257 """Update the environment to reflect the current state. 258 259 This method verifies that snapshots have been pushed. 260 261 Args: 262 environment: The environment to promote. 263 no_gaps: Whether to ensure that new snapshots for models that are already a 264 part of the target environment have no data gaps when compared against previous 265 snapshots for same models. 266 267 Returns: 268 A tuple of (added snapshot table infos, removed snapshot table infos) 269 """
Update the environment to reflect the current state.
This method verifies that snapshots have been pushed.
Arguments:
- environment: The environment to promote.
- no_gaps: Whether to ensure that new snapshots for models that are already a part of the target environment have no data gaps when compared against previous snapshots for same models.
Returns:
A tuple of (added snapshot table infos, removed snapshot table infos)
271 @abc.abstractmethod 272 def delete_expired_environments(self) -> t.List[Environment]: 273 """Removes expired environments. 274 275 Expired environments are environments that have exceeded their time-to-live value. 276 277 Returns: 278 The list of removed environments. 279 """
Removes expired environments.
Expired environments are environments that have exceeded their time-to-live value.
Returns:
The list of removed environments.
281 @abc.abstractmethod 282 def unpause_snapshots( 283 self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike 284 ) -> None: 285 """Unpauses target snapshots. 286 287 Unpaused snapshots are scheduled for evaluation on a recurring basis. 288 Once unpaused a snapshot can't be paused again. 289 290 Args: 291 snapshots: Target snapshots. 292 unpaused_dt: The datetime object which indicates when target snapshots 293 were unpaused. 294 """
Unpauses target snapshots.
Unpaused snapshots are scheduled for evaluation on a recurring basis. Once unpaused a snapshot can't be paused again.
Arguments:
- snapshots: Target snapshots.
- unpaused_dt: The datetime object which indicates when target snapshots were unpaused.