Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: python/cpython
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: b22290e4abb986d101aa1d83f986b7aa39fa22d3
Choose a base ref
..
head repository: python/cpython
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 0a59d09acebe3672e8d1c9dc0e7a7154693df42c
Choose a head ref
Showing with 1,296 additions and 351 deletions.
  1. +6 −6 Doc/howto/enum.rst
  2. +4 −1 Doc/library/_thread.rst
  3. +8 −0 Doc/library/mmap.rst
  4. +4 −1 Doc/library/threading.rst
  5. +2 −2 Doc/tools/extensions/pyspecific.py
  6. +1 −1 Doc/tutorial/stdlib.rst
  7. +8 −0 Doc/whatsnew/3.13.rst
  8. +4 −0 Include/cpython/optimizer.h
  9. +7 −0 Include/cpython/pystate.h
  10. +242 −0 Include/internal/pycore_critical_section.h
  11. +1 −0 Include/internal/pycore_frame.h
  12. +17 −3 Include/internal/pycore_lock.h
  13. +233 −224 Include/internal/pycore_opcode_metadata.h
  14. +6 −2 Include/object.h
  15. +2 −1 Include/pythread.h
  16. +4 −4 Lib/idlelib/idle_test/test_config.py
  17. +4 −4 Lib/logging/config.py
  18. +15 −11 Lib/random.py
  19. 0 Lib/test/{ → configdata}/cfgparser.1
  20. 0 Lib/test/{ → configdata}/cfgparser.2
  21. 0 Lib/test/{ → configdata}/cfgparser.3
  22. +8 −5 Lib/test/libregrtest/single.py
  23. +1 −1 Lib/test/support/__init__.py
  24. +12 −7 Lib/test/support/os_helper.py
  25. +8 −8 Lib/test/test_configparser.py
  26. +43 −2 Lib/test/test_listcomps.py
  27. +37 −1 Lib/test/test_logging.py
  28. +6 −5 Lib/test/test_mmap.py
  29. +1 −1 Lib/test/test_module/__init__.py
  30. +2 −2 Lib/test/{ → test_module}/final_a.py
  31. +2 −2 Lib/test/{ → test_module}/final_b.py
  32. +25 −0 Lib/test/test_monitoring.py
  33. +2 −0 Makefile.pre.in
  34. +3 −0 Misc/NEWS.d/next/C API/2023-10-31-14-58-17.gh-issue-111569._V8iu4.rst
  35. +3 −0 Misc/NEWS.d/next/Core and Builtins/2023-10-09-19-54-33.gh-issue-110543.1wrxO8.rst
  36. +1 −0 Misc/NEWS.d/next/Core and Builtins/2023-11-03-22-48-29.gh-issue-109369.ELYaxJ.rst
  37. +2 −0 Misc/NEWS.d/next/Core and Builtins/2023-11-05-06-40-35.gh-issue-111843.c045cB.rst
  38. +1 −0 Misc/NEWS.d/next/Core and Builtins/2023-11-05-20-59-10.gh-issue-81925.wKHLSS.rst
  39. +2 −0 Misc/NEWS.d/next/Library/2023-11-08-15-58-57.gh-issue-111804.uAXTOL.rst
  40. +4 −0 Misc/NEWS.d/next/Library/2023-11-08-23-32-03.gh-issue-111835.ufFiuW.rst
  41. +1 −1 Modules/Setup.stdlib.in
  42. +3 −0 Modules/_testinternalcapi.c
  43. +1 −0 Modules/_testinternalcapi/parts.h
  44. +213 −0 Modules/_testinternalcapi/test_critical_sections.c
  45. +3 −3 Modules/clinic/posixmodule.c.h
  46. +14 −7 Modules/mmapmodule.c
  47. +5 −2 Modules/posixmodule.c
  48. +29 −0 Objects/codeobject.c
  49. +1 −1 Objects/floatobject.c
  50. +14 −22 Objects/genobject.c
  51. +1 −1 Objects/object.c
  52. +3 −0 PCbuild/_freeze_module.vcxproj
  53. +9 −0 PCbuild/_freeze_module.vcxproj.filters
  54. +1 −0 PCbuild/_testinternalcapi.vcxproj
  55. +3 −0 PCbuild/_testinternalcapi.vcxproj.filters
  56. +1 −0 PCbuild/pyproject.props
  57. +2 −0 PCbuild/pythoncore.vcxproj
  58. +6 −0 PCbuild/pythoncore.vcxproj.filters
  59. +4 −0 Python/abstract_interp_cases.c.h
  60. +23 −5 Python/bytecodes.c
  61. +11 −1 Python/ceval.c
  62. +100 −0 Python/critical_section.c
  63. +6 −0 Python/executor_cases.c.h
  64. +18 −5 Python/generated_cases.c.h
  65. +9 −7 Python/optimizer.c
  66. +14 −1 Python/optimizer_analysis.c
  67. +10 −0 Python/pystate.c
  68. +5 −0 Python/thread_pthread.h
  69. +1 −1 Tools/cases_generator/analysis.py
  70. +54 −0 Tools/cases_generator/flags.py
12 changes: 6 additions & 6 deletions Doc/howto/enum.rst
Original file line number Diff line number Diff line change
@@ -607,9 +607,9 @@ The complete signature is::
start=1,
)

:value: What the new enum class will record as its name.
* *value*: What the new enum class will record as its name.

:names: The enum members. This can be a whitespace- or comma-separated string
* *names*: The enum members. This can be a whitespace- or comma-separated string
(values will start at 1 unless otherwise specified)::

'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE'
@@ -626,13 +626,13 @@ The complete signature is::

{'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42}

:module: name of module where new enum class can be found.
* *module*: name of module where new enum class can be found.

:qualname: where in module new enum class can be found.
* *qualname*: where in module new enum class can be found.

:type: type to mix in to new enum class.
* *type*: type to mix in to new enum class.

:start: number to start counting at if only names are passed in.
* *start*: number to start counting at if only names are passed in.

.. versionchanged:: 3.5
The *start* parameter was added.
5 changes: 4 additions & 1 deletion Doc/library/_thread.rst
Original file line number Diff line number Diff line change
@@ -120,10 +120,13 @@ This module defines the following constants and functions:
Its value may be used to uniquely identify this particular thread system-wide
(until the thread terminates, after which the value may be recycled by the OS).

.. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD.
.. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD.

.. versionadded:: 3.8

.. versionchanged:: 3.13
Added support for GNU/kFreeBSD.


.. function:: stack_size([size])

8 changes: 8 additions & 0 deletions Doc/library/mmap.rst
Original file line number Diff line number Diff line change
@@ -285,6 +285,14 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
values are ``os.SEEK_CUR`` or ``1`` (seek relative to the current
position) and ``os.SEEK_END`` or ``2`` (seek relative to the file's end).

.. versionchanged:: 3.13
Return the new absolute position instead of ``None``.

.. method:: seekable()

Return whether the file supports seeking, and the return value is always ``True``.

.. versionadded:: 3.13

.. method:: size()

5 changes: 4 additions & 1 deletion Doc/library/threading.rst
Original file line number Diff line number Diff line change
@@ -127,10 +127,13 @@ This module defines the following functions:
Its value may be used to uniquely identify this particular thread system-wide
(until the thread terminates, after which the value may be recycled by the OS).

.. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD.
.. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD.

.. versionadded:: 3.8

.. versionchanged:: 3.13
Added support for GNU/kFreeBSD.


.. function:: enumerate()

4 changes: 2 additions & 2 deletions Doc/tools/extensions/pyspecific.py
Original file line number Diff line number Diff line change
@@ -127,8 +127,8 @@ class Availability(SphinxDirective):
# known platform, libc, and threading implementations
known_platforms = frozenset({
"AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD",
"Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", "Unix", "VxWorks",
"WASI", "Windows", "macOS",
"GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris",
"Unix", "VxWorks", "WASI", "Windows", "macOS",
# libc
"BSD libc", "glibc", "musl",
# POSIX platforms with pthreads
2 changes: 1 addition & 1 deletion Doc/tutorial/stdlib.rst
Original file line number Diff line number Diff line change
@@ -153,7 +153,7 @@ The :mod:`random` module provides tools for making random selections::
'apple'
>>> random.sample(range(100), 10) # sampling without replacement
[30, 83, 16, 4, 8, 81, 41, 50, 18, 33]
>>> random.random() # random float
>>> random.random() # random float from the interval [0.0, 1.0)
0.17970987693706186
>>> random.randrange(6) # random integer chosen from range(6)
4
8 changes: 8 additions & 0 deletions Doc/whatsnew/3.13.rst
Original file line number Diff line number Diff line change
@@ -198,6 +198,14 @@ ipaddress
* Add the :attr:`ipaddress.IPv4Address.ipv6_mapped` property, which returns the IPv4-mapped IPv6 address.
(Contributed by Charles Machalow in :gh:`109466`.)

mmap
----

* The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.seekable` method
that can be used where it requires a file-like object with seekable and
the :meth:`~mmap.mmap.seek` method return the new absolute position.
(Contributed by Donghee Na and Sylvie Liberman in :gh:`111835`.)

opcode
------

4 changes: 4 additions & 0 deletions Include/cpython/optimizer.h
Original file line number Diff line number Diff line change
@@ -45,6 +45,8 @@ typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_C
typedef struct _PyOptimizerObject {
PyObject_HEAD
optimize_func optimize;
/* These thresholds are treated as signed so do not exceed INT16_MAX
* Use INT16_MAX to indicate that the optimizer should never be called */
uint16_t resume_threshold;
uint16_t backedge_threshold;
/* Data needed by the optimizer goes here, but is opaque to the VM */
@@ -76,6 +78,8 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);

#define OPTIMIZER_BITS_IN_COUNTER 4
/* Minimum of 16 additional executions before retry */
#define MINIMUM_TIER2_BACKOFF 4

#ifdef __cplusplus
}
7 changes: 7 additions & 0 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
@@ -149,6 +149,13 @@ struct _ts {

struct _py_trashcan trash;

/* Tagged pointer to top-most critical section, or zero if there is no
* active critical section. Critical sections are only used in
* `--disable-gil` builds (i.e., when Py_NOGIL is defined to 1). In the
* default build, this field is always zero.
*/
uintptr_t critical_section;

/* Called when a thread state is deleted normally, but not when it
* is destroyed after fork().
* Pain: to prevent rare but fatal shutdown errors (issue 18808),
242 changes: 242 additions & 0 deletions Include/internal/pycore_critical_section.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,242 @@
#ifndef Py_INTERNAL_CRITICAL_SECTION_H
#define Py_INTERNAL_CRITICAL_SECTION_H

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#include "pycore_lock.h" // PyMutex
#include "pycore_pystate.h" // _PyThreadState_GET()
#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

// Implementation of Python critical sections
//
// Conceptually, critical sections are a deadlock avoidance layer on top of
// per-object locks. These helpers, in combination with those locks, replace
// our usage of the global interpreter lock to provide thread-safety for
// otherwise thread-unsafe objects, such as dict.
//
// NOTE: These APIs are no-ops in non-free-threaded builds.
//
// Straightforward per-object locking could introduce deadlocks that were not
// present when running with the GIL. Threads may hold locks for multiple
// objects simultaneously because Python operations can nest. If threads were
// to acquire the same locks in different orders, they would deadlock.
//
// One way to avoid deadlocks is to allow threads to hold only the lock (or
// locks) for a single operation at a time (typically a single lock, but some
// operations involve two locks). When a thread begins a nested operation it
// could suspend the locks for any outer operation: before beginning the nested
// operation, the locks for the outer operation are released and when the
// nested operation completes, the locks for the outer operation are
// reacquired.
//
// To improve performance, this API uses a variation of the above scheme.
// Instead of immediately suspending locks any time a nested operation begins,
// locks are only suspended if the thread would block. This reduces the number
// of lock acquisitions and releases for nested operations, while still
// avoiding deadlocks.
//
// Additionally, the locks for any active operation are suspended around
// other potentially blocking operations, such as I/O. This is because the
// interaction between locks and blocking operations can lead to deadlocks in
// the same way as the interaction between multiple locks.
//
// Each thread's critical sections and their corresponding locks are tracked in
// a stack in `PyThreadState.critical_section`. When a thread calls
// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when
// waiting to acquire a lock, the thread suspends all of its active critical
// sections, temporarily releasing the associated locks. When the thread calls
// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent)
// critical section by reacquiring the associated lock or locks. See
// `_PyCriticalSection_Resume()`.
//
// NOTE: Only the top-most critical section is guaranteed to be active.
// Operations that need to lock two objects at once must use
// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections
// to lock more than one object at once, because the inner critical section
// may suspend the outer critical sections. This API does not provide a way
// to lock more than two objects at once (though it could be added later
// if actually needed).
//
// NOTE: Critical sections implicitly behave like reentrant locks because
// attempting to acquire the same lock will suspend any outer (earlier)
// critical sections. However, they are less efficient for this use case than
// purposefully designed reentrant locks.
//
// Example usage:
// Py_BEGIN_CRITICAL_SECTION(op);
// ...
// Py_END_CRITICAL_SECTION();
//
// To lock two objects at once:
// Py_BEGIN_CRITICAL_SECTION2(op1, op2);
// ...
// Py_END_CRITICAL_SECTION2();


// Tagged pointers to critical sections use the two least significant bits to
// mark if the pointed-to critical section is inactive and whether it is a
// _PyCriticalSection2 object.
#define _Py_CRITICAL_SECTION_INACTIVE 0x1
#define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2
#define _Py_CRITICAL_SECTION_MASK 0x3

#ifdef Py_NOGIL
# define Py_BEGIN_CRITICAL_SECTION(op) \
{ \
_PyCriticalSection _cs; \
_PyCriticalSection_Begin(&_cs, &_PyObject_CAST(op)->ob_mutex)

# define Py_END_CRITICAL_SECTION() \
_PyCriticalSection_End(&_cs); \
}

# define Py_BEGIN_CRITICAL_SECTION2(a, b) \
{ \
_PyCriticalSection2 _cs2; \
_PyCriticalSection2_Begin(&_cs2, &_PyObject_CAST(a)->ob_mutex, &_PyObject_CAST(b)->ob_mutex)

# define Py_END_CRITICAL_SECTION2() \
_PyCriticalSection2_End(&_cs2); \
}
#else /* !Py_NOGIL */
// The critical section APIs are no-ops with the GIL.
# define Py_BEGIN_CRITICAL_SECTION(op)
# define Py_END_CRITICAL_SECTION()
# define Py_BEGIN_CRITICAL_SECTION2(a, b)
# define Py_END_CRITICAL_SECTION2()
#endif /* !Py_NOGIL */

typedef struct {
// Tagged pointer to an outer active critical section (or 0).
// The two least-significant-bits indicate whether the pointed-to critical
// section is inactive and whether it is a _PyCriticalSection2 object.
uintptr_t prev;

// Mutex used to protect critical section
PyMutex *mutex;
} _PyCriticalSection;

// A critical section protected by two mutexes. Use
// _PyCriticalSection2_Begin and _PyCriticalSection2_End.
typedef struct {
_PyCriticalSection base;

PyMutex *mutex2;
} _PyCriticalSection2;

static inline int
_PyCriticalSection_IsActive(uintptr_t tag)
{
return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0;
}

// Resumes the top-most critical section.
PyAPI_FUNC(void)
_PyCriticalSection_Resume(PyThreadState *tstate);

// (private) slow path for locking the mutex
PyAPI_FUNC(void)
_PyCriticalSection_BeginSlow(_PyCriticalSection *c, PyMutex *m);

PyAPI_FUNC(void)
_PyCriticalSection2_BeginSlow(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
int is_m1_locked);

static inline void
_PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m)
{
if (PyMutex_LockFast(&m->v)) {
PyThreadState *tstate = _PyThreadState_GET();
c->mutex = m;
c->prev = tstate->critical_section;
tstate->critical_section = (uintptr_t)c;
}
else {
_PyCriticalSection_BeginSlow(c, m);
}
}

// Removes the top-most critical section from the thread's stack of critical
// sections. If the new top-most critical section is inactive, then it is
// resumed.
static inline void
_PyCriticalSection_Pop(_PyCriticalSection *c)
{
PyThreadState *tstate = _PyThreadState_GET();
uintptr_t prev = c->prev;
tstate->critical_section = prev;

if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) {
_PyCriticalSection_Resume(tstate);
}
}

static inline void
_PyCriticalSection_End(_PyCriticalSection *c)
{
PyMutex_Unlock(c->mutex);
_PyCriticalSection_Pop(c);
}

static inline void
_PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
{
if (m1 == m2) {
// If the two mutex arguments are the same, treat this as a critical
// section with a single mutex.
c->mutex2 = NULL;
_PyCriticalSection_Begin(&c->base, m1);
return;
}

if ((uintptr_t)m2 < (uintptr_t)m1) {
// Sort the mutexes so that the lower address is locked first.
// The exact order does not matter, but we need to acquire the mutexes
// in a consistent order to avoid lock ordering deadlocks.
PyMutex *tmp = m1;
m1 = m2;
m2 = tmp;
}

if (PyMutex_LockFast(&m1->v)) {
if (PyMutex_LockFast(&m2->v)) {
PyThreadState *tstate = _PyThreadState_GET();
c->base.mutex = m1;
c->mutex2 = m2;
c->base.prev = tstate->critical_section;

uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES;
tstate->critical_section = p;
}
else {
_PyCriticalSection2_BeginSlow(c, m1, m2, 1);
}
}
else {
_PyCriticalSection2_BeginSlow(c, m1, m2, 0);
}
}

static inline void
_PyCriticalSection2_End(_PyCriticalSection2 *c)
{
if (c->mutex2) {
PyMutex_Unlock(c->mutex2);
}
PyMutex_Unlock(c->base.mutex);
_PyCriticalSection_Pop(&c->base);
}

PyAPI_FUNC(void)
_PyCriticalSection_SuspendAll(PyThreadState *tstate);

#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_CRITICAL_SECTION_H */
1 change: 1 addition & 0 deletions Include/internal/pycore_frame.h
Original file line number Diff line number Diff line change
@@ -45,6 +45,7 @@ typedef enum _framestate {
} PyFrameState;

#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM)
#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED)

enum _frameowner {
FRAME_OWNED_BY_THREAD = 0,
Loading