Merge branch 'main' into frame_1

Author: zhangbowen-coder

doc/source/whatsnew/v3.0.0.rst

@@ -820,6 +820,7 @@ Other API changes
   :meth:`~DataFrame.ffill`, :meth:`~DataFrame.bfill`, :meth:`~DataFrame.interpolate`,
   :meth:`~DataFrame.where`, :meth:`~DataFrame.mask`, :meth:`~DataFrame.clip`) now return
   the modified DataFrame or Series (``self``) instead of ``None`` when ``inplace=True`` (:issue:`63207`)
+- All Index constructors now copy ``numpy.ndarray`` and ``ExtensionArray`` inputs by default when ``copy=None``, consistent with :class:`Series` behavior (:issue:`63388`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.deprecations:

pandas/core/algorithms.py

@@ -898,18 +898,15 @@ def value_counts_internal(
             result = result.iloc[0:0]
 
         # normalizing is by len of all (regardless of dropna)
-        counts = np.array([len(ii)])
+        normalize_denominator = len(ii)
 
     else:
+        normalize_denominator = None
         if is_extension_array_dtype(values):
             # handle Categorical and sparse,
             result = Series(values, copy=False)._values.value_counts(dropna=dropna)
             result.name = name
             result.index.name = index_name
-            counts = result._values
-            if not isinstance(counts, np.ndarray):
-                # e.g. ArrowExtensionArray
-                counts = np.asarray(counts)
 
         elif isinstance(values, ABCMultiIndex):
             # GH49558
@@ -920,10 +917,6 @@ def value_counts_internal(
                 .size()
             )
             result.index.names = values.names
-            # error: Incompatible types in assignment (expression has type
-            # "ndarray[Any, Any] | DatetimeArray | TimedeltaArray | PeriodArray | Any",
-            # variable has type "ndarray[tuple[int, ...], dtype[Any]]")
-            counts = result._values  # type: ignore[assignment]
 
         else:
             values = _ensure_arraylike(values, func_name="value_counts")
@@ -936,8 +929,7 @@ def value_counts_internal(
             idx = Index(keys, dtype=keys.dtype, name=index_name)
 
             if (
-                bins is None
-                and not sort
+                not sort
                 and isinstance(values, (DatetimeIndex, TimedeltaIndex))
                 and idx.equals(values)
                 and values.inferred_freq is not None
@@ -951,7 +943,10 @@ def value_counts_internal(
         result = result.sort_values(ascending=ascending, kind="stable")
 
     if normalize:
-        result = result / counts.sum()
+        if normalize_denominator is not None:
+            result = result / normalize_denominator
+        else:
+            result = result / result.sum()
 
     return result
 

pandas/core/arrays/datetimes.py

@@ -226,13 +226,16 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
     """
 
     _typ = "datetimearray"
-    _internal_fill_value = np.datetime64("NaT", "ns")
     _recognized_scalars = (datetime, np.datetime64)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(
         x, "M"
     ) or isinstance(x, DatetimeTZDtype)
     _infer_matches = ("datetime", "datetime64", "date")
 
+    @property
+    def _internal_fill_value(self) -> np.datetime64:
+        return np.datetime64("NaT", self.unit)
+
     @property
     def _scalar_type(self) -> type[Timestamp]:
         return Timestamp

pandas/core/arrays/timedeltas.py

@@ -154,11 +154,14 @@ class TimedeltaArray(dtl.TimelikeOps):
     """
 
     _typ = "timedeltaarray"
-    _internal_fill_value = np.timedelta64("NaT", "ns")
     _recognized_scalars = (timedelta, np.timedelta64, Tick)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(x, "m")
     _infer_matches = ("timedelta", "timedelta64")
 
+    @property
+    def _internal_fill_value(self) -> np.timedelta64:
+        return np.timedelta64("NaT", self.unit)
+
     @property
     def _scalar_type(self) -> type[Timedelta]:
         return Timedelta

pandas/core/indexes/base.py

@@ -505,12 +505,8 @@ def __new__(
         if not copy and isinstance(data, (ABCSeries, Index)):
             refs = data._references
 
-        if isinstance(data, (ExtensionArray, np.ndarray)):
-            # GH 63306
-            if copy is not False:
-                if dtype is None or astype_is_view(data.dtype, dtype):
-                    data = data.copy()
-                    copy = False
+        # GH 63306, GH 63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
 
         # range
         if isinstance(data, (range, RangeIndex)):
@@ -5197,6 +5193,21 @@ def _raise_scalar_data_error(cls, data):
             "was passed"
         )
 
+    @classmethod
+    def _maybe_copy_array_input(
+        cls, data, copy: bool | None, dtype
+    ) -> tuple[Any, bool]:
+        """
+        Ensure that the input data is copied if necessary.
+        GH#63388
+        """
+        if isinstance(data, (ExtensionArray, np.ndarray)):
+            if copy is not False:
+                if dtype is None or astype_is_view(data.dtype, pandas_dtype(dtype)):
+                    data = data.copy()
+                    copy = False
+        return data, bool(copy)
+
     def _validate_fill_value(self, value):
         """
         Check if the value can be inserted into our array without casting,
@@ -6865,12 +6876,15 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
         # we need to look up the label
         try:
             slc = self.get_loc(label)
-        except KeyError as err:
+        except KeyError:
             try:
                 return self._searchsorted_monotonic(label, side)
             except ValueError:
-                # raise the original KeyError
-                raise err from None
+                raise KeyError(
+                    f"Cannot get {side} slice bound for non-monotonic index "
+                    f"with a missing label {original_label!r}. "
+                    "Either sort the index or specify an existing label."
+                ) from None
 
         if isinstance(slc, np.ndarray):
             # get_loc may return a boolean array, which

pandas/core/indexes/datetimes.py

@@ -181,8 +181,13 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
         If True parse dates in `data` with the year first order.
     dtype : numpy.dtype or DatetimeTZDtype or str, default None
         Note that the only NumPy dtype allowed is `datetime64[ns]`.
-    copy : bool, default False
-        Make a copy of input ndarray.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index up front.
     name : label, default None
         Name to be stored in the index.
 
@@ -669,7 +674,7 @@ def __new__(
         dayfirst: bool = False,
         yearfirst: bool = False,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
     ) -> Self:
         if is_scalar(data):
@@ -679,6 +684,9 @@ def __new__(
 
         name = maybe_extract_name(name, data, cls)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         if (
             isinstance(data, DatetimeArray)
             and freq is lib.no_default

pandas/core/indexes/interval.py

@@ -169,8 +169,13 @@ class IntervalIndex(ExtensionIndex):
         neither.
     dtype : dtype or None, default None
         If None, dtype will be inferred.
-    copy : bool, default False
-        Copy the input data.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object, optional
          Name to be stored in the index.
     verify_integrity : bool, default True
@@ -252,12 +257,15 @@ def __new__(
         data,
         closed: IntervalClosedType | None = None,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
         verify_integrity: bool = True,
     ) -> Self:
         name = maybe_extract_name(name, data, cls)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         with rewrite_exception("IntervalArray", cls.__name__):
             array = IntervalArray(
                 data,

pandas/core/indexes/period.py

@@ -101,8 +101,13 @@ class PeriodIndex(DatetimeIndexOpsMixin):
         One of pandas period strings or corresponding objects.
     dtype : str or PeriodDtype, default None
         A dtype from which to extract a freq.
-    copy : bool
-        Make a copy of input ndarray.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : str, default None
         Name of the resulting PeriodIndex.
 
@@ -220,7 +225,7 @@ def __new__(
         data=None,
         freq=None,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
     ) -> Self:
         refs = None
@@ -231,6 +236,9 @@ def __new__(
 
         freq = validate_dtype_freq(dtype, freq)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         # PeriodIndex allow PeriodIndex(period_index, freq=different)
         # Let's not encourage that kind of behavior in PeriodArray.
 

pandas/core/indexes/timedeltas.py

@@ -81,8 +81,13 @@ class TimedeltaIndex(DatetimeTimedeltaMixin):
     dtype : numpy.dtype or str, default None
         Valid ``numpy`` dtypes are ``timedelta64[ns]``, ``timedelta64[us]``,
         ``timedelta64[ms]``, and ``timedelta64[s]``.
-    copy : bool
-        Make a copy of input array.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object
         Name to be stored in the index.
 
@@ -158,11 +163,14 @@ def __new__(
         data=None,
         freq=lib.no_default,
         dtype=None,
-        copy: bool = False,
+        copy: bool | None = None,
         name=None,
     ):
         name = maybe_extract_name(name, data, cls)
 
+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         if is_scalar(data):
             cls._raise_scalar_data_error(data)
 

pandas/io/html.py

@@ -24,10 +24,7 @@
     AbstractMethodError,
     EmptyDataError,
 )
-from pandas.util._decorators import (
-    doc,
-    set_module,
-)
+from pandas.util._decorators import set_module
 from pandas.util._validators import check_dtype_backend
 
 from pandas.core.dtypes.common import is_list_like
@@ -36,7 +33,6 @@
 from pandas.core.indexes.base import Index
 from pandas.core.indexes.multi import MultiIndex
 from pandas.core.series import Series
-from pandas.core.shared_docs import _shared_docs
 
 from pandas.io.common import (
     get_handle,
@@ -1024,7 +1020,6 @@ def _parse(
 
 
 @set_module("pandas")
-@doc(storage_options=_shared_docs["storage_options"])
 def read_html(
     io: FilePath | ReadBuffer[str],
     *,
@@ -1155,7 +1150,15 @@ def read_html(
 
         .. versionadded:: 2.0
 
-    {storage_options}
+    storage_options : dict, optional
+        Extra options that make sense for a particular storage connection, e.g.
+        host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
+        are forwarded to ``urllib.request.Request`` as header options. For other
+        URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
+        forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
+        details, and for more examples on storage options refer `here
+        <https://pandas.pydata.org/docs/user_guide/io.html?
+        highlight=storage_options#reading-writing-remote-files>`_.
 
         .. versionadded:: 2.1.0