Skip to content

fix(DataFrame): #799 to_dict #1283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix(DataFrame): #799 to_dict #1283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d041ff2
fix(DataFrame): to_dict("index") and typevar
cmp0xff Jul 19, 2025
7da086e
feat: https://github.com/pandas-dev/pandas/blob/v2.3.1/pandas/core/co…
cmp0xff Jul 19, 2025
afb5729
fix: https://github.com/pandas-dev/pandas-stubs/pull/1283#discussion_…
cmp0xff Jul 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 49 additions & 18 deletions pandas-stubs/core/frame.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ from builtins import (
bool as _bool,
str as _str,
)
from collections import (
OrderedDict,
defaultdict,
)
from collections.abc import (
Callable,
Hashable,
Expand All @@ -19,6 +23,7 @@ from typing import (
Generic,
Literal,
NoReturn,
TypeVar,
final,
overload,
)
Expand Down Expand Up @@ -165,6 +170,8 @@ from pandas._typing import (
from pandas.io.formats.style import Styler
from pandas.plotting import PlotAccessor

_T_MUTABLE_MAPPING = TypeVar("_T_MUTABLE_MAPPING", bound=MutableMapping, covariant=True)

class _iLocIndexerFrame(_iLocIndexer, Generic[_T]):
@overload
def __getitem__(self, idx: tuple[int, int]) -> Scalar: ...
Expand Down Expand Up @@ -392,13 +399,21 @@ class DataFrame(NDFrame, OpsMixin, _GetItemHack):
na_value: Scalar = ...,
) -> np.ndarray: ...
@overload
def to_dict(
self,
orient: str = ...,
*,
into: type[defaultdict],
index: Literal[True] = ...,
) -> Never: ...
@overload
def to_dict(
self,
orient: Literal["records"],
*,
into: MutableMapping | type[MutableMapping],
into: _T_MUTABLE_MAPPING | type[_T_MUTABLE_MAPPING],
index: Literal[True] = ...,
) -> list[MutableMapping[Hashable, Any]]: ...
) -> list[_T_MUTABLE_MAPPING]: ...
@overload
def to_dict(
self,
Expand All @@ -410,51 +425,67 @@ class DataFrame(NDFrame, OpsMixin, _GetItemHack):
@overload
def to_dict(
self,
orient: Literal["dict", "list", "series", "index"],
orient: Literal["index"],
*,
into: MutableMapping | type[MutableMapping],
into: defaultdict,
index: Literal[True] = ...,
) -> MutableMapping[Hashable, Any]: ...
) -> defaultdict[Hashable, dict[Hashable, Any]]: ...
@overload
def to_dict(
self,
orient: Literal["split", "tight"],
orient: Literal["index"],
*,
into: MutableMapping | type[MutableMapping],
index: bool = ...,
) -> MutableMapping[Hashable, Any]: ...
into: OrderedDict | type[OrderedDict],
index: Literal[True] = ...,
) -> OrderedDict[Hashable, dict[Hashable, Any]]: ...
@overload
def to_dict(
self,
orient: Literal["dict", "list", "series", "index"] = ...,
orient: Literal["index"],
*,
into: MutableMapping | type[MutableMapping],
index: Literal[True] = ...,
) -> MutableMapping[Hashable, Any]: ...
) -> MutableMapping[Hashable, dict[Hashable, Any]]: ...
@overload
def to_dict(
self,
orient: Literal["index"],
*,
into: type[dict] = ...,
index: Literal[True] = ...,
) -> dict[Hashable, dict[Hashable, Any]]: ...
@overload
def to_dict(
self,
orient: Literal["split", "tight"] = ...,
orient: Literal["dict", "list", "series"] = ...,
*,
into: MutableMapping | type[MutableMapping],
index: bool = ...,
) -> MutableMapping[Hashable, Any]: ...
into: _T_MUTABLE_MAPPING | type[_T_MUTABLE_MAPPING],
index: Literal[True] = ...,
) -> _T_MUTABLE_MAPPING: ...
@overload
def to_dict(
self,
orient: Literal["dict", "list", "series", "index"] = ...,
orient: Literal["dict", "list", "series"] = ...,
*,
into: type[dict] = ...,
index: Literal[True] = ...,
) -> dict[Hashable, Any]: ...
@overload
def to_dict(
self,
orient: Literal["split", "tight"] = ...,
orient: Literal["split", "tight"],
*,
into: MutableMapping | type[MutableMapping],
index: bool = ...,
) -> MutableMapping[str, list]: ...
@overload
def to_dict(
self,
orient: Literal["split", "tight"],
*,
into: type[dict] = ...,
index: bool = ...,
) -> dict[Hashable, Any]: ...
) -> dict[str, list]: ...
def to_gbq(
self,
destination_table: str,
Expand Down
104 changes: 82 additions & 22 deletions tests/test_frame.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
from __future__ import annotations

from collections import defaultdict
from collections import (
OrderedDict,
defaultdict,
)
from collections.abc import (
Callable,
Hashable,
Iterable,
Iterator,
Expand All @@ -20,7 +24,6 @@
from typing import (
TYPE_CHECKING,
Any,
Callable,
Generic,
TypedDict,
TypeVar,
Expand All @@ -39,6 +42,7 @@
)
import pytest
from typing_extensions import (
Never,
TypeAlias,
assert_never,
assert_type,
Expand Down Expand Up @@ -3638,33 +3642,86 @@ def test_to_records() -> None:
)


def test_to_dict() -> None:
def test_to_dict_simple() -> None:
check(assert_type(DF.to_dict(), dict[Hashable, Any]), dict)
check(assert_type(DF.to_dict("split"), dict[Hashable, Any]), dict)
check(assert_type(DF.to_dict("records"), list[dict[Hashable, Any]]), list)
check(assert_type(DF.to_dict("index"), dict[Hashable, dict[Hashable, Any]]), dict)
check(assert_type(DF.to_dict("dict"), dict[Hashable, Any]), dict)
check(assert_type(DF.to_dict("list"), dict[Hashable, Any]), dict)
check(assert_type(DF.to_dict("series"), dict[Hashable, Any]), dict)
check(assert_type(DF.to_dict("split"), dict[str, list]), dict, str)
check(assert_type(DF.to_dict("tight"), dict[str, list]), dict, str)

if TYPE_CHECKING_INVALID_USAGE:

def test(mapping: Mapping) -> None: # pyright: ignore[reportUnusedFunction]
DF.to_dict( # type: ignore[call-overload]
into=mapping # pyright: ignore[reportArgumentType,reportCallIssue]
)

assert_type(DF.to_dict(into=defaultdict), Never)
assert_type(DF.to_dict("records", into=defaultdict), Never)
assert_type(DF.to_dict("index", into=defaultdict), Never)
assert_type(DF.to_dict("dict", into=defaultdict), Never)
assert_type(DF.to_dict("list", into=defaultdict), Never)
assert_type(DF.to_dict("series", into=defaultdict), Never)
assert_type(DF.to_dict("split", into=defaultdict), Never)
assert_type(DF.to_dict("tight", into=defaultdict), Never)


def test_to_dict_into_defaultdict() -> None:
"""Test DataFrame.to_dict with `into` is an instance of defaultdict[Any, list]"""

data = pd.DataFrame({("str", "rts"): [[1, 2, 4], [2, 3], [3]]})
target: defaultdict[Any, list] = defaultdict(list)

target: MutableMapping = defaultdict(list)
check(
assert_type(DF.to_dict(into=target), MutableMapping[Hashable, Any]), defaultdict
assert_type(data.to_dict(into=target), defaultdict[Any, list]),
defaultdict,
tuple,
)
target = defaultdict(list)
check(
assert_type(DF.to_dict("tight", into=target), MutableMapping[Hashable, Any]),
assert_type( # type: ignore[assert-type]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't want to see ignore in test files (unless we are testing invalid usage). So can you fix this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The overload

    def to_dict(
        self,
        orient: Literal["index"],
        *,
        into: defaultdict,
        index: Literal[True] = ...,
    ) -> defaultdict[Hashable, dict[Hashable, Any]]: ...

should give the correct typing. However, with the following testing script

import pandas as pd
from collections import defaultdict
from typing import Any, reveal_type

data = pd.DataFrame({("str", "rts"): [[1, 2, 4], [2, 3], [3]]})
target: defaultdict[Any, list] = defaultdict(list)

reveal_type(data.to_dict("index", into=target))

mypy 1.17.0 gives attempt.py:8: note: Revealed type is "Any". In contract, pyright gives attempt.py:8:13 - information: Type of "data.to_dict("index", into=target)" is "defaultdict[Hashable, dict[Hashable, Any]]".

I think it is a mypy bug.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, was able to recreate at python/mypy#19525

So in this case, can you put a comment in the test with a link to that issue to indicate why the # type: ignore is there?

data.to_dict("index", into=target),
defaultdict[Hashable, dict[Hashable, Any]],
),
defaultdict,
)
target = defaultdict(list)
check(assert_type(DF.to_dict("records"), list[dict[Hashable, Any]]), list)
check(
assert_type(data.to_dict("tight", into=target), MutableMapping[str, list]),
defaultdict,
str,
)
check(
assert_type(data.to_dict("records", into=target), list[defaultdict[Any, list]]),
list,
defaultdict,
)


def test_to_dict_into_ordered_dict() -> None:
"""Test DataFrame.to_dict with `into=OrderedDict`"""

data = pd.DataFrame({("str", "rts"): [[1, 2, 4], [2, 3], [3]]})

check(assert_type(data.to_dict(into=OrderedDict), OrderedDict), OrderedDict, tuple)
check(
assert_type(
DF.to_dict("records", into=target), list[MutableMapping[Hashable, Any]]
data.to_dict("index", into=OrderedDict),
OrderedDict[Hashable, dict[Hashable, Any]],
),
OrderedDict,
)
check(
assert_type(data.to_dict("tight", into=OrderedDict), MutableMapping[str, list]),
OrderedDict,
str,
)
check(
assert_type(data.to_dict("records", into=OrderedDict), list[OrderedDict]),
list,
OrderedDict,
)
if TYPE_CHECKING_INVALID_USAGE:

def test(mapping: Mapping) -> None: # pyright: ignore[reportUnusedFunction]
DF.to_dict( # type: ignore[call-overload]
into=mapping # pyright: ignore[reportArgumentType,reportCallIssue]
)


def test_neg() -> None:
Expand Down Expand Up @@ -4111,19 +4168,22 @@ def test_to_dict_index() -> None:
assert_type(df.to_dict(orient="series", index=True), dict[Hashable, Any]), dict
)
check(
assert_type(df.to_dict(orient="index", index=True), dict[Hashable, Any]), dict
assert_type(
df.to_dict(orient="index", index=True), dict[Hashable, dict[Hashable, Any]]
),
dict,
)
check(
assert_type(df.to_dict(orient="split", index=True), dict[Hashable, Any]), dict
assert_type(df.to_dict(orient="split", index=True), dict[str, list]), dict, str
)
check(
assert_type(df.to_dict(orient="tight", index=True), dict[Hashable, Any]), dict
assert_type(df.to_dict(orient="tight", index=True), dict[str, list]), dict, str
)
check(
assert_type(df.to_dict(orient="tight", index=False), dict[Hashable, Any]), dict
assert_type(df.to_dict(orient="tight", index=False), dict[str, list]), dict, str
)
check(
assert_type(df.to_dict(orient="split", index=False), dict[Hashable, Any]), dict
assert_type(df.to_dict(orient="split", index=False), dict[str, list]), dict, str
)
if TYPE_CHECKING_INVALID_USAGE:
check(assert_type(df.to_dict(orient="records", index=False), list[dict[Hashable, Any]]), list) # type: ignore[assert-type, call-overload] # pyright: ignore[reportArgumentType,reportAssertTypeFailure,reportCallIssue]
Expand Down
Loading