Fix!: to_datetime now only accepts millis and not epoch seconds (#1540)

this solves an issue where dates between 1970 and 1972 are ambiguous.

Author: tobymao

sqlmesh/utils/cron.py

@@ -27,16 +27,16 @@ def interval_seconds(self) -> int:
         to substitute for the cron call.
         """
         if self._interval_seconds is None:
-            seconds = set()
+            deltas = set()
             curr = self.curr
 
             for _ in range(self.ESTIMATE_SAMPLES_NUM):
                 prev = curr
-                curr = to_datetime(croniter(self.cron, curr).get_next())
-                seconds.add(curr - prev)
+                curr = to_datetime(croniter(self.cron, curr).get_next() * 1000)
+                deltas.add(curr - prev)
 
-            if len(seconds) == 1:
-                self._interval_seconds = int(first(seconds).total_seconds())
+            if len(deltas) == 1:
+                self._interval_seconds = int(first(deltas).total_seconds())
             else:
                 self._interval_seconds = 0
 
@@ -46,12 +46,12 @@ def get_next(self, estimate: bool = False) -> datetime:
         if estimate and self.interval_seconds:
             self.curr = self.curr + timedelta(seconds=self.interval_seconds)
         else:
-            self.curr = to_datetime(croniter(self.cron, self.curr).get_next())
+            self.curr = to_datetime(croniter(self.cron, self.curr).get_next() * 1000)
         return self.curr
 
     def get_prev(self, estimate: bool = False) -> datetime:
         if estimate and self.interval_seconds:
             self.curr = self.curr - timedelta(seconds=self.interval_seconds)
         else:
-            self.curr = to_datetime(croniter(self.cron, self.curr).get_prev())
+            self.curr = to_datetime(croniter(self.cron, self.curr).get_prev() * 1000)
         return self.curr

sqlmesh/utils/date.py

@@ -18,7 +18,6 @@
 
 UTC = timezone.utc
 TimeLike = t.Union[date, datetime, str, int, float]
-MILLIS_THRESHOLD = time.time() + 100 * 365 * 24 * 3600
 DATE_INT_FMT = "%Y%m%d"
 
 if t.TYPE_CHECKING:
@@ -122,8 +121,7 @@ def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) ->
     """Converts a value into a UTC datetime object.
 
     Args:
-        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
-        if it is larger than MILLIS_THRESHOLD.
+        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs.
         relative_base: The datetime to reference for time expressions that are using relative terms
 
     Raises:
@@ -154,9 +152,7 @@ def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) ->
             try:
                 dt = datetime.strptime(str(value), DATE_INT_FMT)
             except ValueError:
-                dt = datetime.fromtimestamp(
-                    epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC
-                )
+                dt = datetime.fromtimestamp(epoch / 1000.0, tz=UTC)
 
     if dt is None:
         raise ValueError(f"Could not convert `{value}` to datetime.")
@@ -169,8 +165,7 @@ def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) ->
 def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date:
     """Converts a value into a UTC date object
     Args:
-        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
-        if it is larger than MILLIS_THRESHOLD.
+        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs.
         relative_base: The datetime to reference for time expressions that are using relative terms
 
     Raises:

tests/utils/test_date.py

@@ -19,14 +19,15 @@ def test_to_datetime() -> None:
     assert to_datetime("2020-01-01T05:00:00+05:00") == target
     assert to_datetime("2020-01-01") == target
     assert to_datetime(datetime(2020, 1, 1)) == target
-    assert to_datetime("1577836800") == target
-    assert to_datetime(1577836800) == target
-    assert to_datetime(1577836800.0) == target
     assert to_datetime(1577836800000) == target
     assert to_datetime(20200101) == target
     assert to_datetime("20200101") == target
     assert to_datetime("0") == datetime(1970, 1, 1, tzinfo=UTC)
 
+    target = datetime(1971, 1, 1).replace(tzinfo=UTC)
+    assert to_datetime("1971-01-01") == target
+    assert to_datetime("31536000000") == target
+
 
 @pytest.mark.parametrize(
     "expression, result",