pythongh-115999: Disable the specializing adaptive interpreter in free-threaded builds (python#116013)

For now, disable all specialization when the GIL might be disabled.

Author: swtaarrs

Include/internal/pycore_code.h

@@ -245,7 +245,12 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);
 /** API for executors */
 extern void _PyCode_Clear_Executors(PyCodeObject *code);
 
+#ifdef Py_GIL_DISABLED
+// gh-115999 tracks progress on addressing this.
+#define ENABLE_SPECIALIZATION 0
+#else
 #define ENABLE_SPECIALIZATION 1
+#endif
 
 /* Specialization functions */
 

Lib/test/test_capi/test_opt.py

@@ -8,7 +8,7 @@
 
 import _testinternalcapi
 
-from test.support import script_helper
+from test.support import script_helper, requires_specialization
 
 
 @contextlib.contextmanager
@@ -31,6 +31,7 @@ def clear_executors(func):
         func.__code__ = func.__code__.replace()
 
 
+@requires_specialization
 class TestOptimizerAPI(unittest.TestCase):
 
     def test_new_counter_optimizer_dealloc(self):
@@ -133,6 +134,7 @@ def get_opnames(ex):
     return set(iter_opnames(ex))
 
 
+@requires_specialization
 class TestExecutorInvalidation(unittest.TestCase):
 
     def setUp(self):
@@ -211,6 +213,7 @@ def f():
         self.assertIsNone(exe)
 
 
+@requires_specialization
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUops(unittest.TestCase):
 
@@ -572,6 +575,7 @@ def testfunc(n):
         self.assertLessEqual(count, 2)
 
 
+@requires_specialization
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUopsOptimization(unittest.TestCase):
 

Lib/test/test_generated_cases.py

@@ -350,12 +350,15 @@ def test_cache_effect(self):
         output = """
         TARGET(OP) {
             _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            (void)this_instr;
             next_instr += 4;
             INSTRUCTION_STATS(OP);
             PyObject *value;
             value = stack_pointer[-1];
             uint16_t counter = read_u16(&this_instr[1].cache);
+            (void)counter;
             uint32_t extra = read_u32(&this_instr[2].cache);
+            (void)extra;
             stack_pointer += -1;
             DISPATCH();
         }
@@ -399,6 +402,7 @@ def test_macro_instruction(self):
             INSTRUCTION_STATS(OP);
             PREDICTED(OP);
             _Py_CODEUNIT *this_instr = next_instr - 6;
+            (void)this_instr;
             PyObject *right;
             PyObject *left;
             PyObject *arg2;
@@ -408,13 +412,15 @@ def test_macro_instruction(self):
             left = stack_pointer[-2];
             {
                 uint16_t counter = read_u16(&this_instr[1].cache);
+                (void)counter;
                 op1(left, right);
             }
             /* Skip 2 cache entries */
             // OP2
             arg2 = stack_pointer[-3];
             {
                 uint32_t extra = read_u32(&this_instr[4].cache);
+                (void)extra;
                 res = op2(arg2, left, right);
             }
             stack_pointer[-3] = res;
@@ -424,13 +430,15 @@ def test_macro_instruction(self):
 
         TARGET(OP1) {
             _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+            (void)this_instr;
             next_instr += 2;
             INSTRUCTION_STATS(OP1);
             PyObject *right;
             PyObject *left;
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             uint16_t counter = read_u16(&this_instr[1].cache);
+            (void)counter;
             op1(left, right);
             DISPATCH();
         }

Lib/test/test_monitoring.py

@@ -9,6 +9,7 @@
 import types
 import unittest
 import asyncio
+from test.support import requires_specialization
 
 PAIR = (0,1)
 
@@ -815,6 +816,9 @@ def func1():
 
         self.check_events(func1, [("raise", KeyError)])
 
+    # gh-116090: This test doesn't really require specialization, but running
+    # it without specialization exposes a monitoring bug.
+    @requires_specialization
     def test_implicit_stop_iteration(self):
 
         def gen():
@@ -963,6 +967,7 @@ def func():
         )
         self.assertEqual(events[0], ("throw", IndexError))
 
+    @requires_specialization
     def test_no_unwind_for_shim_frame(self):
 
         class B:

Lib/test/test_opcache.py

@@ -4,7 +4,7 @@
 import threading
 import types
 import unittest
-from test.support import threading_helper, check_impl_detail
+from test.support import threading_helper, check_impl_detail, requires_specialization
 
 # Skip this module on other interpreters, it is cpython specific:
 if check_impl_detail(cpython=False):
@@ -506,6 +506,7 @@ def f(x, y):
 
 
 @threading_helper.requires_working_threading()
+@requires_specialization
 class TestRacesDoNotCrash(unittest.TestCase):
     # Careful with these. Bigger numbers have a higher chance of catching bugs,
     # but you can also burn through a *ton* of type/dict/function versions:
@@ -1021,6 +1022,7 @@ def write(items):
 class C:
     pass
 
+@requires_specialization
 class TestInstanceDict(unittest.TestCase):
 
     def setUp(self):

Lib/test/test_type_cache.py

@@ -2,7 +2,7 @@
 import unittest
 import dis
 from test import support
-from test.support import import_helper
+from test.support import import_helper, requires_specialization
 try:
     from sys import _clear_type_cache
 except ImportError:
@@ -94,6 +94,7 @@ class C:
 
 
 @support.cpython_only
+@requires_specialization
 class TypeCacheWithSpecializationTests(unittest.TestCase):
     def tearDown(self):
         _clear_type_cache()

Python/ceval_macros.h

@@ -296,11 +296,19 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 #define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \
     (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1))
 
+#ifdef Py_GIL_DISABLED
+#define DECREMENT_ADAPTIVE_COUNTER(COUNTER)                             \
+    do {                                                                \
+        /* gh-115999 tracks progress on addressing this. */             \
+        static_assert(0, "The specializing interpreter is not yet thread-safe"); \
+    } while (0);
+#else
 #define DECREMENT_ADAPTIVE_COUNTER(COUNTER)           \
     do {                                              \
         assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \
         (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS);    \
     } while (0);
+#endif
 
 #define INCREMENT_ADAPTIVE_COUNTER(COUNTER)          \
     do {                                             \