sqlmesh.utils.date
1from __future__ import annotations 2 3import time 4import typing as t 5import warnings 6 7warnings.filterwarnings( 8 "ignore", 9 message="The localize method is no longer necessary, as this time zone supports the fold attribute", 10) 11from datetime import date, datetime, timedelta, timezone 12 13import dateparser 14from sqlglot import exp 15 16UTC = timezone.utc 17TimeLike = t.Union[date, datetime, str, int, float] 18MILLIS_THRESHOLD = time.time() + 100 * 365 * 24 * 3600 19DATE_INT_FMT = "%Y%m%d" 20 21if t.TYPE_CHECKING: 22 from sqlmesh.core.scheduler import Interval 23 24 25def now() -> datetime: 26 """ 27 Current utc datetime. 28 29 Returns: 30 A datetime object with tz utc. 31 """ 32 return datetime.utcnow().replace(tzinfo=UTC) 33 34 35def now_timestamp() -> int: 36 """ 37 Current utc timestamp. 38 39 Returns: 40 UTC epoch millis timestamp 41 """ 42 return to_timestamp(now()) 43 44 45def now_ds() -> str: 46 """ 47 Current utc ds. 48 49 Returns: 50 Today's ds string. 51 """ 52 return to_ds(now()) 53 54 55def yesterday() -> datetime: 56 """ 57 Yesterday utc datetime. 58 59 Returns: 60 A datetime object with tz utc representing yesterday's date 61 """ 62 return to_datetime("yesterday") 63 64 65def yesterday_ds() -> str: 66 """ 67 Yesterday utc ds. 68 69 Returns: 70 Yesterday's ds string. 71 """ 72 return to_ds("yesterday") 73 74 75def yesterday_timestamp() -> int: 76 """ 77 Yesterday utc timestamp. 78 79 Returns: 80 UTC epoch millis timestamp of yesterday 81 """ 82 return to_timestamp(yesterday()) 83 84 85def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int: 86 """ 87 Converts a value into an epoch millis timestamp. 88 89 Args: 90 value: A variety of date formats. If value is a string, it must be in iso format. 91 relative_base: The datetime to reference for time expressions that are using relative terms 92 93 Returns: 94 Epoch millis timestamp. 95 """ 96 return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000) 97 98 99def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime: 100 """Converts a value into a UTC datetime object. 101 102 Args: 103 value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs 104 if it is larger than MILLIS_THRESHOLD. 105 relative_base: The datetime to reference for time expressions that are using relative terms 106 107 Raises: 108 ValueError if value cannot be converted to a datetime. 109 110 Returns: 111 A datetime object with tz utc. 112 """ 113 if isinstance(value, datetime): 114 dt: t.Optional[datetime] = value 115 elif isinstance(value, date): 116 dt = datetime(value.year, value.month, value.day) 117 elif isinstance(value, exp.Expression): 118 return to_datetime(value.name) 119 else: 120 try: 121 epoch = float(value) 122 except ValueError: 123 epoch = None 124 125 if epoch is None: 126 dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()}) 127 else: 128 try: 129 dt = datetime.strptime(str(value), DATE_INT_FMT) 130 except ValueError: 131 dt = datetime.fromtimestamp( 132 epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC 133 ) 134 135 if dt is None: 136 raise ValueError(f"Could not convert `{value}` to datetime.") 137 138 if dt.tzinfo: 139 return dt.astimezone(UTC) 140 return dt.replace(tzinfo=UTC) 141 142 143def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date: 144 """Converts a value into a UTC date object 145 Args: 146 value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs 147 if it is larger than MILLIS_THRESHOLD. 148 relative_base: The datetime to reference for time expressions that are using relative terms 149 150 Raises: 151 ValueError if value cannot be converted to a date. 152 153 Returns: 154 A date object with tz utc. 155 """ 156 return to_datetime(value, relative_base).date() 157 158 159def date_dict( 160 start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False 161) -> t.Dict[str, t.Union[str, datetime, float, int]]: 162 """Creates a kwarg dictionary of datetime variables for use in SQL Contexts. 163 164 Keys are like start_date, start_ds, end_date, end_ds... 165 166 Args: 167 start: Start time. 168 end: End time. 169 latest: Latest time. 170 only_latest: Only the latest timestamps will be returned. 171 172 Returns: 173 A dictionary with various keys pointing to datetime formats. 174 """ 175 kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {} 176 177 prefixes = [("latest", to_datetime(latest))] 178 179 if not only_latest: 180 prefixes.append(("start", to_datetime(start))) 181 prefixes.append(("end", to_datetime(end))) 182 183 for prefix, time_like in prefixes: 184 dt = to_datetime(time_like) 185 millis = to_timestamp(time_like) 186 kwargs[f"{prefix}_date"] = dt 187 kwargs[f"{prefix}_ds"] = to_ds(time_like) 188 kwargs[f"{prefix}_ts"] = dt.isoformat() 189 kwargs[f"{prefix}_epoch"] = millis / 1000 190 kwargs[f"{prefix}_millis"] = millis 191 return kwargs 192 193 194def to_ds(obj: TimeLike) -> str: 195 """Converts a TimeLike object into YYYY-MM-DD formatted string.""" 196 return to_datetime(obj).isoformat()[0:10] 197 198 199def is_date(obj: TimeLike) -> bool: 200 """Checks if a TimeLike object should be treated like a date.""" 201 if isinstance(obj, date) and not isinstance(obj, datetime): 202 return True 203 204 try: 205 time.strptime(str(obj).replace("-", ""), DATE_INT_FMT) 206 return True 207 except ValueError: 208 return False 209 210 211def make_inclusive(start: TimeLike, end: TimeLike) -> Interval: 212 """Adjust start and end times to to become inclusive datetimes. 213 214 SQLMesh treats start and end times as inclusive so that filters can be written as 215 216 SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds. 217 SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts. 218 219 In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if 220 start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps 221 are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is 222 treated as an exclusive range and converted to inclusive by subtracting 1 millisecond. 223 224 Args: 225 start: Start timelike object. 226 end: End timelike object. 227 228 Example: 229 >>> make_inclusive("2020-01-01", "2020-01-01") 230 (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc)) 231 232 Returns: 233 A tuple of inclusive datetime objects. 234 """ 235 start_dt = to_datetime(start) 236 end_dt = to_datetime(end) 237 if is_date(end): 238 end_dt = end_dt + timedelta(days=1) 239 return (start_dt, end_dt - timedelta(milliseconds=1)) 240 241 242def preserve_time_like_kind(input_value: TimeLike, output_value: TimeLike) -> TimeLike: 243 if is_date(input_value): 244 return to_date(output_value) 245 return output_value 246 247 248def validate_date_range( 249 start: t.Optional[TimeLike], 250 end: t.Optional[TimeLike], 251) -> None: 252 if start and end and to_datetime(start) > to_datetime(end): 253 raise ValueError( 254 f"Start date / time ({start}) can't be greater than end date / time ({end})" 255 )
26def now() -> datetime: 27 """ 28 Current utc datetime. 29 30 Returns: 31 A datetime object with tz utc. 32 """ 33 return datetime.utcnow().replace(tzinfo=UTC)
Current utc datetime.
Returns:
A datetime object with tz utc.
36def now_timestamp() -> int: 37 """ 38 Current utc timestamp. 39 40 Returns: 41 UTC epoch millis timestamp 42 """ 43 return to_timestamp(now())
Current utc timestamp.
Returns:
UTC epoch millis timestamp
46def now_ds() -> str: 47 """ 48 Current utc ds. 49 50 Returns: 51 Today's ds string. 52 """ 53 return to_ds(now())
Current utc ds.
Returns:
Today's ds string.
56def yesterday() -> datetime: 57 """ 58 Yesterday utc datetime. 59 60 Returns: 61 A datetime object with tz utc representing yesterday's date 62 """ 63 return to_datetime("yesterday")
Yesterday utc datetime.
Returns:
A datetime object with tz utc representing yesterday's date
66def yesterday_ds() -> str: 67 """ 68 Yesterday utc ds. 69 70 Returns: 71 Yesterday's ds string. 72 """ 73 return to_ds("yesterday")
Yesterday utc ds.
Returns:
Yesterday's ds string.
76def yesterday_timestamp() -> int: 77 """ 78 Yesterday utc timestamp. 79 80 Returns: 81 UTC epoch millis timestamp of yesterday 82 """ 83 return to_timestamp(yesterday())
Yesterday utc timestamp.
Returns:
UTC epoch millis timestamp of yesterday
86def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int: 87 """ 88 Converts a value into an epoch millis timestamp. 89 90 Args: 91 value: A variety of date formats. If value is a string, it must be in iso format. 92 relative_base: The datetime to reference for time expressions that are using relative terms 93 94 Returns: 95 Epoch millis timestamp. 96 """ 97 return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000)
Converts a value into an epoch millis timestamp.
Arguments:
- value: A variety of date formats. If value is a string, it must be in iso format.
- relative_base: The datetime to reference for time expressions that are using relative terms
Returns:
Epoch millis timestamp.
100def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime: 101 """Converts a value into a UTC datetime object. 102 103 Args: 104 value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs 105 if it is larger than MILLIS_THRESHOLD. 106 relative_base: The datetime to reference for time expressions that are using relative terms 107 108 Raises: 109 ValueError if value cannot be converted to a datetime. 110 111 Returns: 112 A datetime object with tz utc. 113 """ 114 if isinstance(value, datetime): 115 dt: t.Optional[datetime] = value 116 elif isinstance(value, date): 117 dt = datetime(value.year, value.month, value.day) 118 elif isinstance(value, exp.Expression): 119 return to_datetime(value.name) 120 else: 121 try: 122 epoch = float(value) 123 except ValueError: 124 epoch = None 125 126 if epoch is None: 127 dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()}) 128 else: 129 try: 130 dt = datetime.strptime(str(value), DATE_INT_FMT) 131 except ValueError: 132 dt = datetime.fromtimestamp( 133 epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC 134 ) 135 136 if dt is None: 137 raise ValueError(f"Could not convert `{value}` to datetime.") 138 139 if dt.tzinfo: 140 return dt.astimezone(UTC) 141 return dt.replace(tzinfo=UTC)
Converts a value into a UTC datetime object.
Arguments:
- value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
- if it is larger than MILLIS_THRESHOLD.
- relative_base: The datetime to reference for time expressions that are using relative terms
Raises:
- ValueError if value cannot be converted to a datetime.
Returns:
A datetime object with tz utc.
144def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date: 145 """Converts a value into a UTC date object 146 Args: 147 value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs 148 if it is larger than MILLIS_THRESHOLD. 149 relative_base: The datetime to reference for time expressions that are using relative terms 150 151 Raises: 152 ValueError if value cannot be converted to a date. 153 154 Returns: 155 A date object with tz utc. 156 """ 157 return to_datetime(value, relative_base).date()
Converts a value into a UTC date object
Arguments:
- value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
- if it is larger than MILLIS_THRESHOLD.
- relative_base: The datetime to reference for time expressions that are using relative terms
Raises:
- ValueError if value cannot be converted to a date.
Returns:
A date object with tz utc.
160def date_dict( 161 start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False 162) -> t.Dict[str, t.Union[str, datetime, float, int]]: 163 """Creates a kwarg dictionary of datetime variables for use in SQL Contexts. 164 165 Keys are like start_date, start_ds, end_date, end_ds... 166 167 Args: 168 start: Start time. 169 end: End time. 170 latest: Latest time. 171 only_latest: Only the latest timestamps will be returned. 172 173 Returns: 174 A dictionary with various keys pointing to datetime formats. 175 """ 176 kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {} 177 178 prefixes = [("latest", to_datetime(latest))] 179 180 if not only_latest: 181 prefixes.append(("start", to_datetime(start))) 182 prefixes.append(("end", to_datetime(end))) 183 184 for prefix, time_like in prefixes: 185 dt = to_datetime(time_like) 186 millis = to_timestamp(time_like) 187 kwargs[f"{prefix}_date"] = dt 188 kwargs[f"{prefix}_ds"] = to_ds(time_like) 189 kwargs[f"{prefix}_ts"] = dt.isoformat() 190 kwargs[f"{prefix}_epoch"] = millis / 1000 191 kwargs[f"{prefix}_millis"] = millis 192 return kwargs
Creates a kwarg dictionary of datetime variables for use in SQL Contexts.
Keys are like start_date, start_ds, end_date, end_ds...
Arguments:
- start: Start time.
- end: End time.
- latest: Latest time.
- only_latest: Only the latest timestamps will be returned.
Returns:
A dictionary with various keys pointing to datetime formats.
195def to_ds(obj: TimeLike) -> str: 196 """Converts a TimeLike object into YYYY-MM-DD formatted string.""" 197 return to_datetime(obj).isoformat()[0:10]
Converts a TimeLike object into YYYY-MM-DD formatted string.
200def is_date(obj: TimeLike) -> bool: 201 """Checks if a TimeLike object should be treated like a date.""" 202 if isinstance(obj, date) and not isinstance(obj, datetime): 203 return True 204 205 try: 206 time.strptime(str(obj).replace("-", ""), DATE_INT_FMT) 207 return True 208 except ValueError: 209 return False
Checks if a TimeLike object should be treated like a date.
212def make_inclusive(start: TimeLike, end: TimeLike) -> Interval: 213 """Adjust start and end times to to become inclusive datetimes. 214 215 SQLMesh treats start and end times as inclusive so that filters can be written as 216 217 SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds. 218 SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts. 219 220 In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if 221 start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps 222 are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is 223 treated as an exclusive range and converted to inclusive by subtracting 1 millisecond. 224 225 Args: 226 start: Start timelike object. 227 end: End timelike object. 228 229 Example: 230 >>> make_inclusive("2020-01-01", "2020-01-01") 231 (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc)) 232 233 Returns: 234 A tuple of inclusive datetime objects. 235 """ 236 start_dt = to_datetime(start) 237 end_dt = to_datetime(end) 238 if is_date(end): 239 end_dt = end_dt + timedelta(days=1) 240 return (start_dt, end_dt - timedelta(milliseconds=1))
Adjust start and end times to to become inclusive datetimes.
SQLMesh treats start and end times as inclusive so that filters can be written as
SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds. SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.
In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
Arguments:
- start: Start timelike object.
- end: End timelike object.
Example:
>>> make_inclusive("2020-01-01", "2020-01-01") (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
Returns:
A tuple of inclusive datetime objects.