fix: convert to datetime64[us] when TIMESTAMP or DATETIME is empty

Author: kitagry

pandas_gbq/gbq.py

@@ -630,6 +630,7 @@ def _finalize_dtypes(
     """
     import db_dtypes
     import pandas.api.types
+    import pandas
 
     # If you update this mapping, also update the table at
     # `docs/reading.rst`.
@@ -638,6 +639,14 @@ def _finalize_dtypes(
         "DATETIME": "datetime64[ns]",
         "TIMESTAMP": "datetime64[ns]",
     }
+    if pandas.__version__ > "2.0.0":
+        # when pandas is 2.0.0 or later, default timestamp dtype is 'datetime64[us]'
+        # and we should use 'datetime64[us]' instead of 'datetime64[ns]'
+        dtype_map = {
+            "DATE": db_dtypes.DateDtype(),
+            "DATETIME": "datetime64[us]",
+            "TIMESTAMP": pandas.DatetimeTZDtype(unit="us", tz="UTC"),
+        }
 
     for field in schema_fields:
         # This method doesn't modify ARRAY/REPEATED columns.

tests/unit/test_gbq.py

@@ -113,6 +113,39 @@ def test__bqschema_to_nullsafe_dtypes(type_, expected):
         assert result == {"x": expected}
 
 
+@pytest.mark.parametrize(
+    ("data", "schema_type", "expected"),
+    [
+        (
+            pandas.to_datetime(["2017-01-01T12:00:00Z"]).astype(pandas.DatetimeTZDtype(unit="us", tz="UTC")),
+            "TIMESTAMP",
+            pandas.DatetimeTZDtype(unit="us", tz="UTC"),
+        ),
+        (
+            pandas.to_datetime([]).astype(object),
+            "TIMESTAMP",
+            pandas.DatetimeTZDtype(unit="us", tz="UTC"),
+        ),
+        (
+            pandas.to_datetime(["2017-01-01T12:00:00"]).astype("datetime64[us]"),
+            "DATETIME",
+            numpy.dtype("datetime64[us]"),
+        ),
+        (
+            pandas.to_datetime([]).astype(object),
+            "DATETIME",
+            numpy.dtype("datetime64[us]"),
+        ),
+    ],
+)
+def test__finalize_dtypes(data, schema_type, expected):
+    result = gbq._finalize_dtypes(
+        pandas.DataFrame(dict(x=data)),
+        [dict(name="x", type=schema_type, mode="NULLABLE")],
+    )
+    assert result["x"].dtype == expected
+
+
 @pytest.mark.parametrize(
     ["query_or_table", "expected"],
     [