DEPS: bump pyarrow minimum version from 10.0 to 12.0 (#61723)

Author: jorisvandenbossche

ci/deps/actions-310-minimum_versions.yaml

@@ -41,7 +41,7 @@ dependencies:
   - qtpy=2.3.0
   - openpyxl=3.1.2
   - psycopg2=2.9.6
-  - pyarrow=10.0.1
+  - pyarrow=12.0.1
   - pyiceberg=0.7.1
   - pymysql=1.1.0
   - pyqt=5.15.9

ci/deps/actions-310.yaml

@@ -39,7 +39,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9

ci/deps/actions-311-downstream_compat.yaml

@@ -40,7 +40,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9

ci/deps/actions-311.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

ci/deps/actions-312.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

ci/deps/actions-313.yaml

@@ -41,7 +41,7 @@ dependencies:
   - pyqt>=5.15.9
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6
   - pytables>=3.8.0

doc/source/getting_started/install.rst

@@ -307,7 +307,7 @@ Dependency                                             Minimum Version    pip ex
 `PyTables <https://github.com/PyTables/PyTables>`__    3.8.0              hdf5             HDF5-based reading / writing
 `zlib <https://github.com/madler/zlib>`__                                 hdf5             Compression for HDF5
 `fastparquet <https://github.com/dask/fastparquet>`__  2024.2.0           -                Parquet reading / writing (pyarrow is default)
-`pyarrow <https://github.com/apache/arrow>`__          10.0.1             parquet, feather Parquet, ORC, and feather reading / writing
+`pyarrow <https://github.com/apache/arrow>`__          12.0.1             parquet, feather Parquet, ORC, and feather reading / writing
 `PyIceberg <https://py.iceberg.apache.org/>`__         0.7.1              iceberg          Apache Iceberg reading / writing
 `pyreadstat <https://github.com/Roche/pyreadstat>`__   1.2.6              spss             SPSS files (.sav) reading
 `odfpy <https://github.com/eea/odfpy>`__               1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing

doc/source/whatsnew/v3.0.0.rst

@@ -321,6 +321,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +------------------------+---------------------+
 | Package                | New Minimum Version |
 +========================+=====================+
+| pyarrow                | 12.0.1              |
++------------------------+---------------------+
 | pytz                   | 2023.4              |
 +------------------------+---------------------+
 | fastparquet            | 2024.2.0            |

environment.yml

@@ -43,7 +43,7 @@ dependencies:
   - openpyxl>=3.1.2
   - odfpy>=1.4.1
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyreadstat>=1.2.6

pandas/_testing/__init__.py

@@ -18,7 +18,7 @@
     set_locale,
 )
 
-from pandas.compat import pa_version_under10p1
+from pandas.compat import HAS_PYARROW
 
 import pandas as pd
 from pandas import (
@@ -183,7 +183,7 @@
     ]
 ]
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
 
     UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

pandas/compat/__init__.py

@@ -26,8 +26,7 @@
 from pandas.compat.numpy import is_numpy_dev
 from pandas.compat.pyarrow import (
     HAS_PYARROW,
-    pa_version_under10p1,
-    pa_version_under11p0,
+    pa_version_under12p1,
     pa_version_under13p0,
     pa_version_under14p0,
     pa_version_under14p1,
@@ -160,8 +159,7 @@ def is_ci_environment() -> bool:
     "PYPY",
     "WASM",
     "is_numpy_dev",
-    "pa_version_under10p1",
-    "pa_version_under11p0",
+    "pa_version_under12p1",
     "pa_version_under13p0",
     "pa_version_under14p0",
     "pa_version_under14p1",

pandas/compat/_optional.py

@@ -38,7 +38,7 @@
     "openpyxl": "3.1.2",
     "psycopg2": "2.9.6",  # (dt dec pq3 ext lo64)
     "pymysql": "1.1.0",
-    "pyarrow": "10.0.1",
+    "pyarrow": "12.0.1",
     "pyiceberg": "0.7.1",
     "pyreadstat": "1.2.6",
     "pytest": "7.3.2",

pandas/compat/pyarrow.py

@@ -8,9 +8,7 @@
     import pyarrow as pa
 
     _palv = Version(Version(pa.__version__).base_version)
-    pa_version_under10p1 = _palv < Version("10.0.1")
-    pa_version_under11p0 = _palv < Version("11.0.0")
-    pa_version_under12p0 = _palv < Version("12.0.0")
+    pa_version_under12p1 = _palv < Version("12.0.1")
     pa_version_under13p0 = _palv < Version("13.0.0")
     pa_version_under14p0 = _palv < Version("14.0.0")
     pa_version_under14p1 = _palv < Version("14.0.1")
@@ -20,11 +18,9 @@
     pa_version_under18p0 = _palv < Version("18.0.0")
     pa_version_under19p0 = _palv < Version("19.0.0")
     pa_version_under20p0 = _palv < Version("20.0.0")
-    HAS_PYARROW = True
+    HAS_PYARROW = _palv >= Version("12.0.1")
 except ImportError:
-    pa_version_under10p1 = True
-    pa_version_under11p0 = True
-    pa_version_under12p0 = True
+    pa_version_under12p1 = True
     pa_version_under13p0 = True
     pa_version_under14p0 = True
     pa_version_under14p1 = True

pandas/core/arrays/_arrow_string_mixins.py

@@ -12,13 +12,12 @@
 
 from pandas._libs import lib
 from pandas.compat import (
-    pa_version_under10p1,
-    pa_version_under11p0,
+    HAS_PYARROW,
     pa_version_under13p0,
     pa_version_under17p0,
 )
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -132,7 +131,7 @@ def _str_get(self, i: int) -> Self:
     def _str_slice(
         self, start: int | None = None, stop: int | None = None, step: int | None = None
     ) -> Self:
-        if pa_version_under11p0:
+        if pa_version_under13p0:
             # GH#59724
             result = self._apply_elementwise(lambda val: val[start:stop:step])
             return type(self)(pa.chunked_array(result, type=self._pa_array.type))

pandas/core/arrays/arrow/accessors.py

@@ -11,14 +11,11 @@
     cast,
 )
 
-from pandas.compat import (
-    pa_version_under10p1,
-    pa_version_under11p0,
-)
+from pandas.compat import HAS_PYARROW
 
 from pandas.core.dtypes.common import is_list_like
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -46,7 +43,7 @@ def _is_valid_pyarrow_dtype(self, pyarrow_dtype) -> bool:
 
     def _validate(self, data) -> None:
         dtype = data.dtype
-        if pa_version_under10p1 or not isinstance(dtype, ArrowDtype):
+        if not HAS_PYARROW or not isinstance(dtype, ArrowDtype):
             # Raise AttributeError so that inspect can handle non-struct Series.
             raise AttributeError(self._validation_msg.format(dtype=dtype))
 
@@ -171,11 +168,6 @@ def __getitem__(self, key: int | slice) -> Series:
                 name=self._data.name,
             )
         elif isinstance(key, slice):
-            if pa_version_under11p0:
-                raise NotImplementedError(
-                    f"List slice not supported by pyarrow {pa.__version__}."
-                )
-
             # TODO: Support negative start/stop/step, ideally this would be added
             # upstream in pyarrow.
             start, stop, step = key.start, key.stop, key.step

pandas/core/arrays/arrow/array.py

@@ -22,8 +22,8 @@
     timezones,
 )
 from pandas.compat import (
-    pa_version_under10p1,
-    pa_version_under11p0,
+    HAS_PYARROW,
+    pa_version_under12p1,
     pa_version_under13p0,
 )
 from pandas.util._decorators import doc
@@ -74,7 +74,7 @@
 from pandas.io._util import _arrow_dtype_mapping
 from pandas.tseries.frequencies import to_offset
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -208,16 +208,6 @@ def floordiv_compat(
     from pandas.core.arrays.timedeltas import TimedeltaArray
 
 
-def get_unit_from_pa_dtype(pa_dtype) -> str:
-    # https://github.com/pandas-dev/pandas/pull/50998#discussion_r1100344804
-    if pa_version_under11p0:
-        unit = str(pa_dtype).split("[", 1)[-1][:-1]
-        if unit not in ["s", "ms", "us", "ns"]:
-            raise ValueError(pa_dtype)
-        return unit
-    return pa_dtype.unit
-
-
 def to_pyarrow_type(
     dtype: ArrowDtype | pa.DataType | Dtype | None,
 ) -> pa.DataType | None:
@@ -300,7 +290,7 @@ class ArrowExtensionArray(
     _dtype: ArrowDtype
 
     def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
-        if pa_version_under10p1:
+        if pa_version_under12p1:
             msg = "pyarrow>=10.0.1 is required for PyArrow backed ArrowExtensionArray."
             raise ImportError(msg)
         if isinstance(values, pa.Array):
@@ -1199,10 +1189,6 @@ def factorize(
         null_encoding = "mask" if use_na_sentinel else "encode"
 
         data = self._pa_array
-        pa_type = data.type
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = data.cast(pa.int64())
 
         if pa.types.is_dictionary(data.type):
             if null_encoding == "encode":
@@ -1227,8 +1213,6 @@ def factorize(
             )
             uniques = type(self)(combined.dictionary)
 
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            uniques = cast(ArrowExtensionArray, uniques.astype(self.dtype))
         return indices, uniques
 
     def reshape(self, *args, **kwargs):
@@ -1515,19 +1499,7 @@ def unique(self) -> Self:
         -------
         ArrowExtensionArray
         """
-        pa_type = self._pa_array.type
-
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = self._pa_array.cast(pa.int64())
-        else:
-            data = self._pa_array
-
-        pa_result = pc.unique(data)
-
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            pa_result = pa_result.cast(pa_type)
-
+        pa_result = pc.unique(self._pa_array)
         return type(self)(pa_result)
 
     def value_counts(self, dropna: bool = True) -> Series:
@@ -1547,18 +1519,12 @@ def value_counts(self, dropna: bool = True) -> Series:
         --------
         Series.value_counts
         """
-        pa_type = self._pa_array.type
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            # https://github.com/apache/arrow/issues/15226#issuecomment-1376578323
-            data = self._pa_array.cast(pa.int64())
-        else:
-            data = self._pa_array
-
         from pandas import (
             Index,
             Series,
         )
 
+        data = self._pa_array
         vc = data.value_counts()
 
         values = vc.field(0)
@@ -1568,9 +1534,6 @@ def value_counts(self, dropna: bool = True) -> Series:
             values = values.filter(mask)
             counts = counts.filter(mask)
 
-        if pa_version_under11p0 and pa.types.is_duration(pa_type):
-            values = values.cast(pa_type)
-
         counts = ArrowExtensionArray(counts)
 
         index = Index(type(self)(values))
@@ -1864,8 +1827,7 @@ def pyarrow_meth(data, skip_nulls, min_count=0):  # type: ignore[misc]
             if pa.types.is_duration(pa_type):
                 result = result.cast(pa_type)
             elif pa.types.is_time(pa_type):
-                unit = get_unit_from_pa_dtype(pa_type)
-                result = result.cast(pa.duration(unit))
+                result = result.cast(pa.duration(pa_type.unit))
             elif pa.types.is_date(pa_type):
                 # go with closest available unit, i.e. "s"
                 result = result.cast(pa.duration("s"))

pandas/core/arrays/string_.py

@@ -25,7 +25,7 @@
 from pandas._libs.lib import ensure_string_array
 from pandas.compat import (
     HAS_PYARROW,
-    pa_version_under10p1,
+    pa_version_under12p1,
 )
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import (
@@ -182,9 +182,9 @@ def __init__(
             raise ValueError(
                 f"Storage must be 'python' or 'pyarrow'. Got {storage} instead."
             )
-        if storage == "pyarrow" and pa_version_under10p1:
+        if storage == "pyarrow" and pa_version_under12p1:
             raise ImportError(
-                "pyarrow>=10.0.1 is required for PyArrow backed StringArray."
+                "pyarrow>=12.0.1 is required for PyArrow backed StringArray."
             )
 
         if isinstance(na_value, float) and np.isnan(na_value):

pandas/core/arrays/string_arrow.py

@@ -14,7 +14,8 @@
     missing as libmissing,
 )
 from pandas.compat import (
-    pa_version_under10p1,
+    HAS_PYARROW,
+    pa_version_under12p1,
     pa_version_under13p0,
     pa_version_under16p0,
 )
@@ -38,7 +39,7 @@
 )
 from pandas.core.strings.object_array import ObjectStringArrayMixin
 
-if not pa_version_under10p1:
+if HAS_PYARROW:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -63,8 +64,8 @@
 
 
 def _chk_pyarrow_available() -> None:
-    if pa_version_under10p1:
-        msg = "pyarrow>=10.0.1 is required for PyArrow backed ArrowExtensionArray."
+    if pa_version_under12p1:
+        msg = "pyarrow>=12.0.1 is required for PyArrow backed ArrowExtensionArray."
         raise ImportError(msg)