Edit on GitHub

sqlmesh.core.state_sync.base

  1from __future__ import annotations
  2
  3import abc
  4import typing as t
  5
  6from sqlmesh.core import scheduler
  7from sqlmesh.core.environment import Environment
  8from sqlmesh.core.snapshot import (
  9    Intervals,
 10    Snapshot,
 11    SnapshotId,
 12    SnapshotIdLike,
 13    SnapshotInfoLike,
 14    SnapshotNameVersionLike,
 15    SnapshotTableInfo,
 16)
 17from sqlmesh.utils.date import TimeLike, now, to_datetime
 18from sqlmesh.utils.errors import SQLMeshError
 19
 20
 21class StateReader(abc.ABC):
 22    """Abstract base class for read-only operations on snapshot and environment state."""
 23
 24    @abc.abstractmethod
 25    def get_snapshots(
 26        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
 27    ) -> t.Dict[SnapshotId, Snapshot]:
 28        """Bulk fetch snapshots given the corresponding snapshot ids.
 29
 30        Args:
 31            snapshot_ids: Iterable of snapshot ids to get. If not provided all
 32                available snapshots will be returned.
 33
 34        Returns:
 35            A dictionary of snapshot ids to snapshots for ones that could be found.
 36        """
 37
 38    @abc.abstractmethod
 39    def get_snapshots_with_same_version(
 40        self, snapshots: t.Iterable[SnapshotNameVersionLike]
 41    ) -> t.List[Snapshot]:
 42        """Fetches all snapshots that share the same version as the snapshots.
 43
 44        The output includes the snapshots with the specified version.
 45
 46        Args:
 47            snapshots: The collection of target name / version pairs.
 48
 49        Returns:
 50            The list of Snapshot objects.
 51        """
 52
 53    @abc.abstractmethod
 54    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
 55        """Checks if multiple snapshots exist in the state sync.
 56
 57        Args:
 58            snapshot_ids: Iterable of snapshot ids to bulk check.
 59
 60        Returns:
 61            A set of all the existing snapshot ids.
 62        """
 63
 64    @abc.abstractmethod
 65    def get_environment(self, environment: str) -> t.Optional[Environment]:
 66        """Fetches the environment if it exists.
 67
 68        Args:
 69            environment: The environment
 70
 71        Returns:
 72            The environment object.
 73        """
 74
 75    @abc.abstractmethod
 76    def get_environments(self) -> t.List[Environment]:
 77        """Fetches all environments.
 78
 79        Returns:
 80            A list of all environments.
 81        """
 82
 83    @abc.abstractmethod
 84    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
 85        """Get all snapshots by model name.
 86
 87        Returns:
 88            The list of snapshots.
 89        """
 90
 91    def missing_intervals(
 92        self,
 93        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
 94        start: t.Optional[TimeLike] = None,
 95        end: t.Optional[TimeLike] = None,
 96        latest: t.Optional[TimeLike] = None,
 97        restatements: t.Optional[t.Iterable[str]] = None,
 98    ) -> t.Dict[Snapshot, Intervals]:
 99        """Find missing intervals for an environment or a list of snapshots.
100
101        Args:
102            env_or_snapshots: The environment or snapshots to find missing intervals for.
103            start: The start of the time range to look for.
104            end: The end of the time range to look for.
105            latest: The latest datetime to use for non-incremental queries.
106
107        Returns:
108            A dictionary of SnapshotId to Intervals.
109        """
110
111        if isinstance(env_or_snapshots, str):
112            env = self.get_environment(env_or_snapshots)
113        elif isinstance(env_or_snapshots, Environment):
114            env = env_or_snapshots
115        else:
116            env = None
117
118        if env:
119            snapshots_by_id = self.get_snapshots(env.snapshots)
120            start = start or env.start_at
121            end = end or env.end_at
122        elif isinstance(env_or_snapshots, str):
123            snapshots_by_id = {}
124        elif not isinstance(env_or_snapshots, Environment):
125            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
126        else:
127            raise SQLMeshError("This shouldn't be possible.")
128
129        if not snapshots_by_id:
130            return {}
131
132        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
133
134        snapshots_by_id = {
135            **snapshots_by_id,
136            **(self.get_snapshots(unversioned) if unversioned else {}),
137        }
138
139        all_snapshots = {
140            **snapshots_by_id,
141            **{
142                snapshot.snapshot_id: snapshot
143                for snapshot in self.get_snapshots_with_same_version(
144                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
145                )
146            },
147        }
148
149        missing = {}
150        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
151        end_date = end or now()
152        restatements = set(restatements or [])
153
154        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
155            if snapshot.name in restatements:
156                snapshot.remove_interval(start_date, end_date)
157            intervals = snapshot.missing_intervals(
158                max(
159                    start_date,
160                    to_datetime(
161                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
162                    ),
163                ),
164                end_date,
165                latest=latest,
166            )
167            if intervals:
168                missing[snapshot] = intervals
169        return missing
170
171
172class StateSync(StateReader, abc.ABC):
173    """Abstract base class for snapshot and environment state management."""
174
175    @abc.abstractmethod
176    def init_schema(self) -> None:
177        """Optional initialization of the sync."""
178
179    @abc.abstractmethod
180    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
181        """Push snapshots into the state sync.
182
183        This method only allows for pushing new snapshots. If existing snapshots are found,
184        this method should raise an error.
185
186        Raises:
187            SQLMeshError when existing snapshots are pushed.
188
189        Args:
190            snapshots: A list of snapshots to save in the state sync.
191        """
192
193    @abc.abstractmethod
194    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
195        """Delete snapshots from the state sync.
196
197        Args:
198            snapshot_ids: A list of snapshot like objects to delete.
199        """
200
201    @abc.abstractmethod
202    def delete_expired_snapshots(self) -> t.List[Snapshot]:
203        """Removes expired snapshots.
204
205        Expired snapshots are snapshots that have exceeded their time-to-live
206        and are no longer in use within an environment.
207
208        Returns:
209            The list of removed snapshots.
210        """
211
212    @abc.abstractmethod
213    def add_interval(
214        self,
215        snapshot_id: SnapshotIdLike,
216        start: TimeLike,
217        end: TimeLike,
218        is_dev: bool = False,
219    ) -> None:
220        """Add an interval to a snapshot and sync it to the store.
221
222        Snapshots must be pushed before adding intervals to them.
223
224        Args:
225            snapshot_id: The snapshot like object to add an interval to.
226            start: The start of the interval to add.
227            end: The end of the interval to add.
228            is_dev: Indicates whether the given interval is being added while in
229                development mode.
230        """
231
232    @abc.abstractmethod
233    def remove_interval(
234        self,
235        snapshots: t.Iterable[SnapshotInfoLike],
236        start: TimeLike,
237        end: TimeLike,
238        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
239    ) -> None:
240        """Remove an interval from a list of snapshots and sync it to the store.
241
242        Because multiple snapshots can be pointing to the same version or physical table, this method
243        can also grab all snapshots tied to the passed in version.
244
245        Args:
246            snapshots: The snapshot info like object to remove intervals from.
247            start: The start of the interval to add.
248            end: The end of the interval to add.
249            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
250        """
251
252    @abc.abstractmethod
253    def promote(
254        self, environment: Environment, no_gaps: bool = False
255    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
256        """Update the environment to reflect the current state.
257
258        This method verifies that snapshots have been pushed.
259
260        Args:
261            environment: The environment to promote.
262            no_gaps:  Whether to ensure that new snapshots for models that are already a
263                part of the target environment have no data gaps when compared against previous
264                snapshots for same models.
265
266        Returns:
267           A tuple of (added snapshot table infos, removed snapshot table infos)
268        """
269
270    @abc.abstractmethod
271    def delete_expired_environments(self) -> t.List[Environment]:
272        """Removes expired environments.
273
274        Expired environments are environments that have exceeded their time-to-live value.
275
276        Returns:
277            The list of removed environments.
278        """
279
280    @abc.abstractmethod
281    def unpause_snapshots(
282        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
283    ) -> None:
284        """Unpauses target snapshots.
285
286        Unpaused snapshots are scheduled for evaluation on a recurring basis.
287        Once unpaused a snapshot can't be paused again.
288
289        Args:
290            snapshots: Target snapshots.
291            unpaused_dt: The datetime object which indicates when target snapshots
292                were unpaused.
293        """
class StateReader(abc.ABC):
 22class StateReader(abc.ABC):
 23    """Abstract base class for read-only operations on snapshot and environment state."""
 24
 25    @abc.abstractmethod
 26    def get_snapshots(
 27        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
 28    ) -> t.Dict[SnapshotId, Snapshot]:
 29        """Bulk fetch snapshots given the corresponding snapshot ids.
 30
 31        Args:
 32            snapshot_ids: Iterable of snapshot ids to get. If not provided all
 33                available snapshots will be returned.
 34
 35        Returns:
 36            A dictionary of snapshot ids to snapshots for ones that could be found.
 37        """
 38
 39    @abc.abstractmethod
 40    def get_snapshots_with_same_version(
 41        self, snapshots: t.Iterable[SnapshotNameVersionLike]
 42    ) -> t.List[Snapshot]:
 43        """Fetches all snapshots that share the same version as the snapshots.
 44
 45        The output includes the snapshots with the specified version.
 46
 47        Args:
 48            snapshots: The collection of target name / version pairs.
 49
 50        Returns:
 51            The list of Snapshot objects.
 52        """
 53
 54    @abc.abstractmethod
 55    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
 56        """Checks if multiple snapshots exist in the state sync.
 57
 58        Args:
 59            snapshot_ids: Iterable of snapshot ids to bulk check.
 60
 61        Returns:
 62            A set of all the existing snapshot ids.
 63        """
 64
 65    @abc.abstractmethod
 66    def get_environment(self, environment: str) -> t.Optional[Environment]:
 67        """Fetches the environment if it exists.
 68
 69        Args:
 70            environment: The environment
 71
 72        Returns:
 73            The environment object.
 74        """
 75
 76    @abc.abstractmethod
 77    def get_environments(self) -> t.List[Environment]:
 78        """Fetches all environments.
 79
 80        Returns:
 81            A list of all environments.
 82        """
 83
 84    @abc.abstractmethod
 85    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
 86        """Get all snapshots by model name.
 87
 88        Returns:
 89            The list of snapshots.
 90        """
 91
 92    def missing_intervals(
 93        self,
 94        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
 95        start: t.Optional[TimeLike] = None,
 96        end: t.Optional[TimeLike] = None,
 97        latest: t.Optional[TimeLike] = None,
 98        restatements: t.Optional[t.Iterable[str]] = None,
 99    ) -> t.Dict[Snapshot, Intervals]:
100        """Find missing intervals for an environment or a list of snapshots.
101
102        Args:
103            env_or_snapshots: The environment or snapshots to find missing intervals for.
104            start: The start of the time range to look for.
105            end: The end of the time range to look for.
106            latest: The latest datetime to use for non-incremental queries.
107
108        Returns:
109            A dictionary of SnapshotId to Intervals.
110        """
111
112        if isinstance(env_or_snapshots, str):
113            env = self.get_environment(env_or_snapshots)
114        elif isinstance(env_or_snapshots, Environment):
115            env = env_or_snapshots
116        else:
117            env = None
118
119        if env:
120            snapshots_by_id = self.get_snapshots(env.snapshots)
121            start = start or env.start_at
122            end = end or env.end_at
123        elif isinstance(env_or_snapshots, str):
124            snapshots_by_id = {}
125        elif not isinstance(env_or_snapshots, Environment):
126            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
127        else:
128            raise SQLMeshError("This shouldn't be possible.")
129
130        if not snapshots_by_id:
131            return {}
132
133        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
134
135        snapshots_by_id = {
136            **snapshots_by_id,
137            **(self.get_snapshots(unversioned) if unversioned else {}),
138        }
139
140        all_snapshots = {
141            **snapshots_by_id,
142            **{
143                snapshot.snapshot_id: snapshot
144                for snapshot in self.get_snapshots_with_same_version(
145                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
146                )
147            },
148        }
149
150        missing = {}
151        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
152        end_date = end or now()
153        restatements = set(restatements or [])
154
155        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
156            if snapshot.name in restatements:
157                snapshot.remove_interval(start_date, end_date)
158            intervals = snapshot.missing_intervals(
159                max(
160                    start_date,
161                    to_datetime(
162                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
163                    ),
164                ),
165                end_date,
166                latest=latest,
167            )
168            if intervals:
169                missing[snapshot] = intervals
170        return missing

Abstract base class for read-only operations on snapshot and environment state.

25    @abc.abstractmethod
26    def get_snapshots(
27        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
28    ) -> t.Dict[SnapshotId, Snapshot]:
29        """Bulk fetch snapshots given the corresponding snapshot ids.
30
31        Args:
32            snapshot_ids: Iterable of snapshot ids to get. If not provided all
33                available snapshots will be returned.
34
35        Returns:
36            A dictionary of snapshot ids to snapshots for ones that could be found.
37        """

Bulk fetch snapshots given the corresponding snapshot ids.

Arguments:
  • snapshot_ids: Iterable of snapshot ids to get. If not provided all available snapshots will be returned.
Returns:

A dictionary of snapshot ids to snapshots for ones that could be found.

39    @abc.abstractmethod
40    def get_snapshots_with_same_version(
41        self, snapshots: t.Iterable[SnapshotNameVersionLike]
42    ) -> t.List[Snapshot]:
43        """Fetches all snapshots that share the same version as the snapshots.
44
45        The output includes the snapshots with the specified version.
46
47        Args:
48            snapshots: The collection of target name / version pairs.
49
50        Returns:
51            The list of Snapshot objects.
52        """

Fetches all snapshots that share the same version as the snapshots.

The output includes the snapshots with the specified version.

Arguments:
  • snapshots: The collection of target name / version pairs.
Returns:

The list of Snapshot objects.

54    @abc.abstractmethod
55    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
56        """Checks if multiple snapshots exist in the state sync.
57
58        Args:
59            snapshot_ids: Iterable of snapshot ids to bulk check.
60
61        Returns:
62            A set of all the existing snapshot ids.
63        """

Checks if multiple snapshots exist in the state sync.

Arguments:
  • snapshot_ids: Iterable of snapshot ids to bulk check.
Returns:

A set of all the existing snapshot ids.

@abc.abstractmethod
def get_environment(self, environment: str) -> Optional[sqlmesh.core.environment.Environment]:
65    @abc.abstractmethod
66    def get_environment(self, environment: str) -> t.Optional[Environment]:
67        """Fetches the environment if it exists.
68
69        Args:
70            environment: The environment
71
72        Returns:
73            The environment object.
74        """

Fetches the environment if it exists.

Arguments:
  • environment: The environment
Returns:

The environment object.

@abc.abstractmethod
def get_environments(self) -> List[sqlmesh.core.environment.Environment]:
76    @abc.abstractmethod
77    def get_environments(self) -> t.List[Environment]:
78        """Fetches all environments.
79
80        Returns:
81            A list of all environments.
82        """

Fetches all environments.

Returns:

A list of all environments.

@abc.abstractmethod
def get_snapshots_by_models(self, *names: str) -> List[sqlmesh.core.snapshot.definition.Snapshot]:
84    @abc.abstractmethod
85    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
86        """Get all snapshots by model name.
87
88        Returns:
89            The list of snapshots.
90        """

Get all snapshots by model name.

Returns:

The list of snapshots.

def missing_intervals( self, env_or_snapshots: Union[str, sqlmesh.core.environment.Environment, Iterable[sqlmesh.core.snapshot.definition.Snapshot]], start: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, end: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, latest: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, restatements: Optional[Iterable[str]] = None) -> Dict[sqlmesh.core.snapshot.definition.Snapshot, List[Tuple[int, int]]]:
 92    def missing_intervals(
 93        self,
 94        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
 95        start: t.Optional[TimeLike] = None,
 96        end: t.Optional[TimeLike] = None,
 97        latest: t.Optional[TimeLike] = None,
 98        restatements: t.Optional[t.Iterable[str]] = None,
 99    ) -> t.Dict[Snapshot, Intervals]:
100        """Find missing intervals for an environment or a list of snapshots.
101
102        Args:
103            env_or_snapshots: The environment or snapshots to find missing intervals for.
104            start: The start of the time range to look for.
105            end: The end of the time range to look for.
106            latest: The latest datetime to use for non-incremental queries.
107
108        Returns:
109            A dictionary of SnapshotId to Intervals.
110        """
111
112        if isinstance(env_or_snapshots, str):
113            env = self.get_environment(env_or_snapshots)
114        elif isinstance(env_or_snapshots, Environment):
115            env = env_or_snapshots
116        else:
117            env = None
118
119        if env:
120            snapshots_by_id = self.get_snapshots(env.snapshots)
121            start = start or env.start_at
122            end = end or env.end_at
123        elif isinstance(env_or_snapshots, str):
124            snapshots_by_id = {}
125        elif not isinstance(env_or_snapshots, Environment):
126            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
127        else:
128            raise SQLMeshError("This shouldn't be possible.")
129
130        if not snapshots_by_id:
131            return {}
132
133        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
134
135        snapshots_by_id = {
136            **snapshots_by_id,
137            **(self.get_snapshots(unversioned) if unversioned else {}),
138        }
139
140        all_snapshots = {
141            **snapshots_by_id,
142            **{
143                snapshot.snapshot_id: snapshot
144                for snapshot in self.get_snapshots_with_same_version(
145                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
146                )
147            },
148        }
149
150        missing = {}
151        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
152        end_date = end or now()
153        restatements = set(restatements or [])
154
155        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
156            if snapshot.name in restatements:
157                snapshot.remove_interval(start_date, end_date)
158            intervals = snapshot.missing_intervals(
159                max(
160                    start_date,
161                    to_datetime(
162                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
163                    ),
164                ),
165                end_date,
166                latest=latest,
167            )
168            if intervals:
169                missing[snapshot] = intervals
170        return missing

Find missing intervals for an environment or a list of snapshots.

Arguments:
  • env_or_snapshots: The environment or snapshots to find missing intervals for.
  • start: The start of the time range to look for.
  • end: The end of the time range to look for.
  • latest: The latest datetime to use for non-incremental queries.
Returns:

A dictionary of SnapshotId to Intervals.

class StateSync(StateReader, abc.ABC):
173class StateSync(StateReader, abc.ABC):
174    """Abstract base class for snapshot and environment state management."""
175
176    @abc.abstractmethod
177    def init_schema(self) -> None:
178        """Optional initialization of the sync."""
179
180    @abc.abstractmethod
181    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
182        """Push snapshots into the state sync.
183
184        This method only allows for pushing new snapshots. If existing snapshots are found,
185        this method should raise an error.
186
187        Raises:
188            SQLMeshError when existing snapshots are pushed.
189
190        Args:
191            snapshots: A list of snapshots to save in the state sync.
192        """
193
194    @abc.abstractmethod
195    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
196        """Delete snapshots from the state sync.
197
198        Args:
199            snapshot_ids: A list of snapshot like objects to delete.
200        """
201
202    @abc.abstractmethod
203    def delete_expired_snapshots(self) -> t.List[Snapshot]:
204        """Removes expired snapshots.
205
206        Expired snapshots are snapshots that have exceeded their time-to-live
207        and are no longer in use within an environment.
208
209        Returns:
210            The list of removed snapshots.
211        """
212
213    @abc.abstractmethod
214    def add_interval(
215        self,
216        snapshot_id: SnapshotIdLike,
217        start: TimeLike,
218        end: TimeLike,
219        is_dev: bool = False,
220    ) -> None:
221        """Add an interval to a snapshot and sync it to the store.
222
223        Snapshots must be pushed before adding intervals to them.
224
225        Args:
226            snapshot_id: The snapshot like object to add an interval to.
227            start: The start of the interval to add.
228            end: The end of the interval to add.
229            is_dev: Indicates whether the given interval is being added while in
230                development mode.
231        """
232
233    @abc.abstractmethod
234    def remove_interval(
235        self,
236        snapshots: t.Iterable[SnapshotInfoLike],
237        start: TimeLike,
238        end: TimeLike,
239        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
240    ) -> None:
241        """Remove an interval from a list of snapshots and sync it to the store.
242
243        Because multiple snapshots can be pointing to the same version or physical table, this method
244        can also grab all snapshots tied to the passed in version.
245
246        Args:
247            snapshots: The snapshot info like object to remove intervals from.
248            start: The start of the interval to add.
249            end: The end of the interval to add.
250            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
251        """
252
253    @abc.abstractmethod
254    def promote(
255        self, environment: Environment, no_gaps: bool = False
256    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
257        """Update the environment to reflect the current state.
258
259        This method verifies that snapshots have been pushed.
260
261        Args:
262            environment: The environment to promote.
263            no_gaps:  Whether to ensure that new snapshots for models that are already a
264                part of the target environment have no data gaps when compared against previous
265                snapshots for same models.
266
267        Returns:
268           A tuple of (added snapshot table infos, removed snapshot table infos)
269        """
270
271    @abc.abstractmethod
272    def delete_expired_environments(self) -> t.List[Environment]:
273        """Removes expired environments.
274
275        Expired environments are environments that have exceeded their time-to-live value.
276
277        Returns:
278            The list of removed environments.
279        """
280
281    @abc.abstractmethod
282    def unpause_snapshots(
283        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
284    ) -> None:
285        """Unpauses target snapshots.
286
287        Unpaused snapshots are scheduled for evaluation on a recurring basis.
288        Once unpaused a snapshot can't be paused again.
289
290        Args:
291            snapshots: Target snapshots.
292            unpaused_dt: The datetime object which indicates when target snapshots
293                were unpaused.
294        """

Abstract base class for snapshot and environment state management.

@abc.abstractmethod
def init_schema(self) -> None:
176    @abc.abstractmethod
177    def init_schema(self) -> None:
178        """Optional initialization of the sync."""

Optional initialization of the sync.

@abc.abstractmethod
def push_snapshots( self, snapshots: Iterable[sqlmesh.core.snapshot.definition.Snapshot]) -> None:
180    @abc.abstractmethod
181    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
182        """Push snapshots into the state sync.
183
184        This method only allows for pushing new snapshots. If existing snapshots are found,
185        this method should raise an error.
186
187        Raises:
188            SQLMeshError when existing snapshots are pushed.
189
190        Args:
191            snapshots: A list of snapshots to save in the state sync.
192        """

Push snapshots into the state sync.

This method only allows for pushing new snapshots. If existing snapshots are found, this method should raise an error.

Raises:
  • SQLMeshError when existing snapshots are pushed.
Arguments:
  • snapshots: A list of snapshots to save in the state sync.
@abc.abstractmethod
def delete_snapshots( self, snapshot_ids: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]]) -> None:
194    @abc.abstractmethod
195    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
196        """Delete snapshots from the state sync.
197
198        Args:
199            snapshot_ids: A list of snapshot like objects to delete.
200        """

Delete snapshots from the state sync.

Arguments:
  • snapshot_ids: A list of snapshot like objects to delete.
@abc.abstractmethod
def delete_expired_snapshots(self) -> List[sqlmesh.core.snapshot.definition.Snapshot]:
202    @abc.abstractmethod
203    def delete_expired_snapshots(self) -> t.List[Snapshot]:
204        """Removes expired snapshots.
205
206        Expired snapshots are snapshots that have exceeded their time-to-live
207        and are no longer in use within an environment.
208
209        Returns:
210            The list of removed snapshots.
211        """

Removes expired snapshots.

Expired snapshots are snapshots that have exceeded their time-to-live and are no longer in use within an environment.

Returns:

The list of removed snapshots.

@abc.abstractmethod
def add_interval( self, snapshot_id: Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot], start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], is_dev: bool = False) -> None:
213    @abc.abstractmethod
214    def add_interval(
215        self,
216        snapshot_id: SnapshotIdLike,
217        start: TimeLike,
218        end: TimeLike,
219        is_dev: bool = False,
220    ) -> None:
221        """Add an interval to a snapshot and sync it to the store.
222
223        Snapshots must be pushed before adding intervals to them.
224
225        Args:
226            snapshot_id: The snapshot like object to add an interval to.
227            start: The start of the interval to add.
228            end: The end of the interval to add.
229            is_dev: Indicates whether the given interval is being added while in
230                development mode.
231        """

Add an interval to a snapshot and sync it to the store.

Snapshots must be pushed before adding intervals to them.

Arguments:
  • snapshot_id: The snapshot like object to add an interval to.
  • start: The start of the interval to add.
  • end: The end of the interval to add.
  • is_dev: Indicates whether the given interval is being added while in development mode.
@abc.abstractmethod
def remove_interval( self, snapshots: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]], start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], all_snapshots: Optional[Iterable[sqlmesh.core.snapshot.definition.Snapshot]] = None) -> None:
233    @abc.abstractmethod
234    def remove_interval(
235        self,
236        snapshots: t.Iterable[SnapshotInfoLike],
237        start: TimeLike,
238        end: TimeLike,
239        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
240    ) -> None:
241        """Remove an interval from a list of snapshots and sync it to the store.
242
243        Because multiple snapshots can be pointing to the same version or physical table, this method
244        can also grab all snapshots tied to the passed in version.
245
246        Args:
247            snapshots: The snapshot info like object to remove intervals from.
248            start: The start of the interval to add.
249            end: The end of the interval to add.
250            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
251        """

Remove an interval from a list of snapshots and sync it to the store.

Because multiple snapshots can be pointing to the same version or physical table, this method can also grab all snapshots tied to the passed in version.

Arguments:
  • snapshots: The snapshot info like object to remove intervals from.
  • start: The start of the interval to add.
  • end: The end of the interval to add.
  • all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
@abc.abstractmethod
def promote( self, environment: sqlmesh.core.environment.Environment, no_gaps: bool = False) -> Tuple[List[sqlmesh.core.snapshot.definition.SnapshotTableInfo], List[sqlmesh.core.snapshot.definition.SnapshotTableInfo]]:
253    @abc.abstractmethod
254    def promote(
255        self, environment: Environment, no_gaps: bool = False
256    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
257        """Update the environment to reflect the current state.
258
259        This method verifies that snapshots have been pushed.
260
261        Args:
262            environment: The environment to promote.
263            no_gaps:  Whether to ensure that new snapshots for models that are already a
264                part of the target environment have no data gaps when compared against previous
265                snapshots for same models.
266
267        Returns:
268           A tuple of (added snapshot table infos, removed snapshot table infos)
269        """

Update the environment to reflect the current state.

This method verifies that snapshots have been pushed.

Arguments:
  • environment: The environment to promote.
  • no_gaps: Whether to ensure that new snapshots for models that are already a part of the target environment have no data gaps when compared against previous snapshots for same models.
Returns:

A tuple of (added snapshot table infos, removed snapshot table infos)

@abc.abstractmethod
def delete_expired_environments(self) -> List[sqlmesh.core.environment.Environment]:
271    @abc.abstractmethod
272    def delete_expired_environments(self) -> t.List[Environment]:
273        """Removes expired environments.
274
275        Expired environments are environments that have exceeded their time-to-live value.
276
277        Returns:
278            The list of removed environments.
279        """

Removes expired environments.

Expired environments are environments that have exceeded their time-to-live value.

Returns:

The list of removed environments.

@abc.abstractmethod
def unpause_snapshots( self, snapshots: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]], unpaused_dt: Union[datetime.date, datetime.datetime, str, int, float]) -> None:
281    @abc.abstractmethod
282    def unpause_snapshots(
283        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
284    ) -> None:
285        """Unpauses target snapshots.
286
287        Unpaused snapshots are scheduled for evaluation on a recurring basis.
288        Once unpaused a snapshot can't be paused again.
289
290        Args:
291            snapshots: Target snapshots.
292            unpaused_dt: The datetime object which indicates when target snapshots
293                were unpaused.
294        """

Unpauses target snapshots.

Unpaused snapshots are scheduled for evaluation on a recurring basis. Once unpaused a snapshot can't be paused again.

Arguments:
  • snapshots: Target snapshots.
  • unpaused_dt: The datetime object which indicates when target snapshots were unpaused.