Edit on GitHub

sqlmesh.utils.date

  1from __future__ import annotations
  2
  3import time
  4import typing as t
  5import warnings
  6
  7warnings.filterwarnings(
  8    "ignore",
  9    message="The localize method is no longer necessary, as this time zone supports the fold attribute",
 10)
 11from datetime import date, datetime, timedelta, timezone
 12
 13import dateparser
 14from sqlglot import exp
 15
 16UTC = timezone.utc
 17TimeLike = t.Union[date, datetime, str, int, float]
 18MILLIS_THRESHOLD = time.time() + 100 * 365 * 24 * 3600
 19DATE_INT_FMT = "%Y%m%d"
 20
 21if t.TYPE_CHECKING:
 22    from sqlmesh.core.scheduler import Interval
 23
 24
 25def now() -> datetime:
 26    """
 27    Current utc datetime.
 28
 29    Returns:
 30        A datetime object with tz utc.
 31    """
 32    return datetime.utcnow().replace(tzinfo=UTC)
 33
 34
 35def now_timestamp() -> int:
 36    """
 37    Current utc timestamp.
 38
 39    Returns:
 40        UTC epoch millis timestamp
 41    """
 42    return to_timestamp(now())
 43
 44
 45def now_ds() -> str:
 46    """
 47    Current utc ds.
 48
 49    Returns:
 50        Today's ds string.
 51    """
 52    return to_ds(now())
 53
 54
 55def yesterday() -> datetime:
 56    """
 57    Yesterday utc datetime.
 58
 59    Returns:
 60        A datetime object with tz utc representing yesterday's date
 61    """
 62    return to_datetime("yesterday")
 63
 64
 65def yesterday_ds() -> str:
 66    """
 67    Yesterday utc ds.
 68
 69    Returns:
 70        Yesterday's ds string.
 71    """
 72    return to_ds("yesterday")
 73
 74
 75def yesterday_timestamp() -> int:
 76    """
 77    Yesterday utc timestamp.
 78
 79    Returns:
 80        UTC epoch millis timestamp of yesterday
 81    """
 82    return to_timestamp(yesterday())
 83
 84
 85def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int:
 86    """
 87    Converts a value into an epoch millis timestamp.
 88
 89    Args:
 90        value: A variety of date formats. If value is a string, it must be in iso format.
 91        relative_base: The datetime to reference for time expressions that are using relative terms
 92
 93    Returns:
 94        Epoch millis timestamp.
 95    """
 96    return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000)
 97
 98
 99def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime:
100    """Converts a value into a UTC datetime object.
101
102    Args:
103        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
104        if it is larger than MILLIS_THRESHOLD.
105        relative_base: The datetime to reference for time expressions that are using relative terms
106
107    Raises:
108        ValueError if value cannot be converted to a datetime.
109
110    Returns:
111        A datetime object with tz utc.
112    """
113    if isinstance(value, datetime):
114        dt: t.Optional[datetime] = value
115    elif isinstance(value, date):
116        dt = datetime(value.year, value.month, value.day)
117    elif isinstance(value, exp.Expression):
118        return to_datetime(value.name)
119    else:
120        try:
121            epoch = float(value)
122        except ValueError:
123            epoch = None
124
125        if epoch is None:
126            dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()})
127        else:
128            try:
129                dt = datetime.strptime(str(value), DATE_INT_FMT)
130            except ValueError:
131                dt = datetime.fromtimestamp(
132                    epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC
133                )
134
135    if dt is None:
136        raise ValueError(f"Could not convert `{value}` to datetime.")
137
138    if dt.tzinfo:
139        return dt.astimezone(UTC)
140    return dt.replace(tzinfo=UTC)
141
142
143def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date:
144    """Converts a value into a UTC date object
145    Args:
146        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
147        if it is larger than MILLIS_THRESHOLD.
148        relative_base: The datetime to reference for time expressions that are using relative terms
149
150    Raises:
151        ValueError if value cannot be converted to a date.
152
153    Returns:
154        A date object with tz utc.
155    """
156    return to_datetime(value, relative_base).date()
157
158
159def date_dict(
160    start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False
161) -> t.Dict[str, t.Union[str, datetime, float, int]]:
162    """Creates a kwarg dictionary of datetime variables for use in SQL Contexts.
163
164    Keys are like start_date, start_ds, end_date, end_ds...
165
166    Args:
167        start: Start time.
168        end: End time.
169        latest: Latest time.
170        only_latest: Only the latest timestamps will be returned.
171
172    Returns:
173        A dictionary with various keys pointing to datetime formats.
174    """
175    kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {}
176
177    prefixes = [("latest", to_datetime(latest))]
178
179    if not only_latest:
180        prefixes.append(("start", to_datetime(start)))
181        prefixes.append(("end", to_datetime(end)))
182
183    for prefix, time_like in prefixes:
184        dt = to_datetime(time_like)
185        millis = to_timestamp(time_like)
186        kwargs[f"{prefix}_date"] = dt
187        kwargs[f"{prefix}_ds"] = to_ds(time_like)
188        kwargs[f"{prefix}_ts"] = dt.isoformat()
189        kwargs[f"{prefix}_epoch"] = millis / 1000
190        kwargs[f"{prefix}_millis"] = millis
191    return kwargs
192
193
194def to_ds(obj: TimeLike) -> str:
195    """Converts a TimeLike object into YYYY-MM-DD formatted string."""
196    return to_datetime(obj).isoformat()[0:10]
197
198
199def is_date(obj: TimeLike) -> bool:
200    """Checks if a TimeLike object should be treated like a date."""
201    if isinstance(obj, date) and not isinstance(obj, datetime):
202        return True
203
204    try:
205        time.strptime(str(obj).replace("-", ""), DATE_INT_FMT)
206        return True
207    except ValueError:
208        return False
209
210
211def make_inclusive(start: TimeLike, end: TimeLike) -> Interval:
212    """Adjust start and end times to to become inclusive datetimes.
213
214    SQLMesh treats start and end times as inclusive so that filters can be written as
215
216    SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds.
217    SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.
218
219    In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if
220    start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps
221    are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is
222    treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
223
224    Args:
225        start: Start timelike object.
226        end: End timelike object.
227
228    Example:
229        >>> make_inclusive("2020-01-01", "2020-01-01")
230        (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
231
232    Returns:
233        A tuple of inclusive datetime objects.
234    """
235    start_dt = to_datetime(start)
236    end_dt = to_datetime(end)
237    if is_date(end):
238        end_dt = end_dt + timedelta(days=1)
239    return (start_dt, end_dt - timedelta(milliseconds=1))
240
241
242def preserve_time_like_kind(input_value: TimeLike, output_value: TimeLike) -> TimeLike:
243    if is_date(input_value):
244        return to_date(output_value)
245    return output_value
246
247
248def validate_date_range(
249    start: t.Optional[TimeLike],
250    end: t.Optional[TimeLike],
251) -> None:
252    if start and end and to_datetime(start) > to_datetime(end):
253        raise ValueError(
254            f"Start date / time ({start}) can't be greater than end date / time ({end})"
255        )
def now() -> datetime.datetime:
26def now() -> datetime:
27    """
28    Current utc datetime.
29
30    Returns:
31        A datetime object with tz utc.
32    """
33    return datetime.utcnow().replace(tzinfo=UTC)

Current utc datetime.

Returns:

A datetime object with tz utc.

def now_timestamp() -> int:
36def now_timestamp() -> int:
37    """
38    Current utc timestamp.
39
40    Returns:
41        UTC epoch millis timestamp
42    """
43    return to_timestamp(now())

Current utc timestamp.

Returns:

UTC epoch millis timestamp

def now_ds() -> str:
46def now_ds() -> str:
47    """
48    Current utc ds.
49
50    Returns:
51        Today's ds string.
52    """
53    return to_ds(now())

Current utc ds.

Returns:

Today's ds string.

def yesterday() -> datetime.datetime:
56def yesterday() -> datetime:
57    """
58    Yesterday utc datetime.
59
60    Returns:
61        A datetime object with tz utc representing yesterday's date
62    """
63    return to_datetime("yesterday")

Yesterday utc datetime.

Returns:

A datetime object with tz utc representing yesterday's date

def yesterday_ds() -> str:
66def yesterday_ds() -> str:
67    """
68    Yesterday utc ds.
69
70    Returns:
71        Yesterday's ds string.
72    """
73    return to_ds("yesterday")

Yesterday utc ds.

Returns:

Yesterday's ds string.

def yesterday_timestamp() -> int:
76def yesterday_timestamp() -> int:
77    """
78    Yesterday utc timestamp.
79
80    Returns:
81        UTC epoch millis timestamp of yesterday
82    """
83    return to_timestamp(yesterday())

Yesterday utc timestamp.

Returns:

UTC epoch millis timestamp of yesterday

def to_timestamp( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> int:
86def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int:
87    """
88    Converts a value into an epoch millis timestamp.
89
90    Args:
91        value: A variety of date formats. If value is a string, it must be in iso format.
92        relative_base: The datetime to reference for time expressions that are using relative terms
93
94    Returns:
95        Epoch millis timestamp.
96    """
97    return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000)

Converts a value into an epoch millis timestamp.

Arguments:
  • value: A variety of date formats. If value is a string, it must be in iso format.
  • relative_base: The datetime to reference for time expressions that are using relative terms
Returns:

Epoch millis timestamp.

def to_datetime( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> datetime.datetime:
100def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime:
101    """Converts a value into a UTC datetime object.
102
103    Args:
104        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
105        if it is larger than MILLIS_THRESHOLD.
106        relative_base: The datetime to reference for time expressions that are using relative terms
107
108    Raises:
109        ValueError if value cannot be converted to a datetime.
110
111    Returns:
112        A datetime object with tz utc.
113    """
114    if isinstance(value, datetime):
115        dt: t.Optional[datetime] = value
116    elif isinstance(value, date):
117        dt = datetime(value.year, value.month, value.day)
118    elif isinstance(value, exp.Expression):
119        return to_datetime(value.name)
120    else:
121        try:
122            epoch = float(value)
123        except ValueError:
124            epoch = None
125
126        if epoch is None:
127            dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()})
128        else:
129            try:
130                dt = datetime.strptime(str(value), DATE_INT_FMT)
131            except ValueError:
132                dt = datetime.fromtimestamp(
133                    epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC
134                )
135
136    if dt is None:
137        raise ValueError(f"Could not convert `{value}` to datetime.")
138
139    if dt.tzinfo:
140        return dt.astimezone(UTC)
141    return dt.replace(tzinfo=UTC)

Converts a value into a UTC datetime object.

Arguments:
  • value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
  • if it is larger than MILLIS_THRESHOLD.
  • relative_base: The datetime to reference for time expressions that are using relative terms
Raises:
  • ValueError if value cannot be converted to a datetime.
Returns:

A datetime object with tz utc.

def to_date( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> datetime.date:
144def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date:
145    """Converts a value into a UTC date object
146    Args:
147        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
148        if it is larger than MILLIS_THRESHOLD.
149        relative_base: The datetime to reference for time expressions that are using relative terms
150
151    Raises:
152        ValueError if value cannot be converted to a date.
153
154    Returns:
155        A date object with tz utc.
156    """
157    return to_datetime(value, relative_base).date()

Converts a value into a UTC date object

Arguments:
  • value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
  • if it is larger than MILLIS_THRESHOLD.
  • relative_base: The datetime to reference for time expressions that are using relative terms
Raises:
  • ValueError if value cannot be converted to a date.
Returns:

A date object with tz utc.

def date_dict( start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], latest: Union[datetime.date, datetime.datetime, str, int, float], only_latest: bool = False) -> Dict[str, Union[str, datetime.datetime, float, int]]:
160def date_dict(
161    start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False
162) -> t.Dict[str, t.Union[str, datetime, float, int]]:
163    """Creates a kwarg dictionary of datetime variables for use in SQL Contexts.
164
165    Keys are like start_date, start_ds, end_date, end_ds...
166
167    Args:
168        start: Start time.
169        end: End time.
170        latest: Latest time.
171        only_latest: Only the latest timestamps will be returned.
172
173    Returns:
174        A dictionary with various keys pointing to datetime formats.
175    """
176    kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {}
177
178    prefixes = [("latest", to_datetime(latest))]
179
180    if not only_latest:
181        prefixes.append(("start", to_datetime(start)))
182        prefixes.append(("end", to_datetime(end)))
183
184    for prefix, time_like in prefixes:
185        dt = to_datetime(time_like)
186        millis = to_timestamp(time_like)
187        kwargs[f"{prefix}_date"] = dt
188        kwargs[f"{prefix}_ds"] = to_ds(time_like)
189        kwargs[f"{prefix}_ts"] = dt.isoformat()
190        kwargs[f"{prefix}_epoch"] = millis / 1000
191        kwargs[f"{prefix}_millis"] = millis
192    return kwargs

Creates a kwarg dictionary of datetime variables for use in SQL Contexts.

Keys are like start_date, start_ds, end_date, end_ds...

Arguments:
  • start: Start time.
  • end: End time.
  • latest: Latest time.
  • only_latest: Only the latest timestamps will be returned.
Returns:

A dictionary with various keys pointing to datetime formats.

def to_ds(obj: Union[datetime.date, datetime.datetime, str, int, float]) -> str:
195def to_ds(obj: TimeLike) -> str:
196    """Converts a TimeLike object into YYYY-MM-DD formatted string."""
197    return to_datetime(obj).isoformat()[0:10]

Converts a TimeLike object into YYYY-MM-DD formatted string.

def is_date(obj: Union[datetime.date, datetime.datetime, str, int, float]) -> bool:
200def is_date(obj: TimeLike) -> bool:
201    """Checks if a TimeLike object should be treated like a date."""
202    if isinstance(obj, date) and not isinstance(obj, datetime):
203        return True
204
205    try:
206        time.strptime(str(obj).replace("-", ""), DATE_INT_FMT)
207        return True
208    except ValueError:
209        return False

Checks if a TimeLike object should be treated like a date.

def make_inclusive( start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float]) -> Tuple[datetime.datetime, datetime.datetime]:
212def make_inclusive(start: TimeLike, end: TimeLike) -> Interval:
213    """Adjust start and end times to to become inclusive datetimes.
214
215    SQLMesh treats start and end times as inclusive so that filters can be written as
216
217    SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds.
218    SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.
219
220    In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if
221    start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps
222    are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is
223    treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
224
225    Args:
226        start: Start timelike object.
227        end: End timelike object.
228
229    Example:
230        >>> make_inclusive("2020-01-01", "2020-01-01")
231        (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
232
233    Returns:
234        A tuple of inclusive datetime objects.
235    """
236    start_dt = to_datetime(start)
237    end_dt = to_datetime(end)
238    if is_date(end):
239        end_dt = end_dt + timedelta(days=1)
240    return (start_dt, end_dt - timedelta(milliseconds=1))

Adjust start and end times to to become inclusive datetimes.

SQLMesh treats start and end times as inclusive so that filters can be written as

SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds. SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.

In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.

Arguments:
  • start: Start timelike object.
  • end: End timelike object.
Example:
>>> make_inclusive("2020-01-01", "2020-01-01")
(datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
Returns:

A tuple of inclusive datetime objects.

def preserve_time_like_kind( input_value: Union[datetime.date, datetime.datetime, str, int, float], output_value: Union[datetime.date, datetime.datetime, str, int, float]) -> Union[datetime.date, datetime.datetime, str, int, float]:
243def preserve_time_like_kind(input_value: TimeLike, output_value: TimeLike) -> TimeLike:
244    if is_date(input_value):
245        return to_date(output_value)
246    return output_value
def validate_date_range( start: Union[datetime.date, datetime.datetime, str, int, float, NoneType], end: Union[datetime.date, datetime.datetime, str, int, float, NoneType]) -> None:
249def validate_date_range(
250    start: t.Optional[TimeLike],
251    end: t.Optional[TimeLike],
252) -> None:
253    if start and end and to_datetime(start) > to_datetime(end):
254        raise ValueError(
255            f"Start date / time ({start}) can't be greater than end date / time ({end})"
256        )