bpo-44170: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList

Author: junnplus

Lib/multiprocessing/shared_memory.py

@@ -293,14 +293,21 @@ def _extract_recreation_code(value):
         else:
             return 3  # NoneType
 
+    @staticmethod
+    def _encode_value(value):
+        if not isinstance(value, str):
+            return value
+        else:
+            return value.encode(_encoding)
+
     def __init__(self, sequence=None, *, name=None):
         if name is None or sequence is not None:
             sequence = sequence or ()
             _formats = [
                 self._types_mapping[type(item)]
                     if not isinstance(item, (str, bytes))
                     else self._types_mapping[type(item)] % (
-                        self._alignment * (len(item) // self._alignment + 1),
+                        self._alignment * (len(self._encode_value(item)) // self._alignment + 1),
                     )
                 for item in sequence
             ]
@@ -341,7 +348,7 @@ def __init__(self, sequence=None, *, name=None):
                 "".join(_formats),
                 self.shm.buf,
                 self._offset_data_start,
-                *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
+                *(self._encode_value(v) for v in sequence)
             )
             struct.pack_into(
                 self._format_packing_metainfo,
@@ -451,9 +458,8 @@ def __setitem__(self, position, value):
         else:
             allocated_length = self._allocated_offsets[position + 1] - item_offset
 
-            encoded_value = (value.encode(_encoding)
-                             if isinstance(value, str) else value)
-            if len(encoded_value) > allocated_length:
+            encoded_value = self._encode_value(value)
+            if len(encoded_value) >= allocated_length:
                 raise ValueError("bytes/str item exceeds available storage")
             if current_format[-1] == "s":
                 new_format = current_format

Lib/test/_test_multiprocessing.py

@@ -3997,7 +3997,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):
 
     def test_shared_memory_ShareableList_basics(self):
         sl = shared_memory.ShareableList(
-            ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
+            ['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥']
         )
         self.addCleanup(sl.shm.unlink)
 
@@ -4007,22 +4007,22 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Index Out of Range (get)
         with self.assertRaises(IndexError):
-            sl[7]
+            sl[8]
 
         # Index Out of Range (set)
         with self.assertRaises(IndexError):
-            sl[7] = 2
+            sl[8] = 2
 
         # Assign value without format change (str -> str)
         current_format = sl._get_packing_format(0)
         sl[0] = 'howdy'
         self.assertEqual(current_format, sl._get_packing_format(0))
 
         # Verify attributes are readable.
-        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s')
 
         # Exercise len().
-        self.assertEqual(len(sl), 7)
+        self.assertEqual(len(sl), 8)
 
         # Exercise index().
         with warnings.catch_warnings():
@@ -4034,30 +4034,30 @@ def test_shared_memory_ShareableList_basics(self):
 
         # Exercise retrieving individual values.
         self.assertEqual(sl[0], 'howdy')
-        self.assertEqual(sl[-2], True)
+        self.assertEqual(sl[-3], True)
+        self.assertEqual(sl[-1], '💥 💥')
 
         # Exercise iterability.
         self.assertEqual(
             tuple(sl),
-            ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
+            ('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥')
         )
 
         # Exercise modifying individual values.
         sl[3] = 42
         self.assertEqual(sl[3], 42)
         sl[4] = 'some'  # Change type at a given position.
         self.assertEqual(sl[4], 'some')
-        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
+        self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
             sl[4] = 'far too many'
         self.assertEqual(sl[4], 'some')
-        sl[0] = 'encodés'  # Exactly 8 bytes of UTF-8 data
-        self.assertEqual(sl[0], 'encodés')
-        self.assertEqual(sl[1], b'HoWdY')  # no spillage
+        sl[0] = 'encodé'
+        self.assertEqual(sl[0], 'encodé')  # no spillage
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):
-            sl[0] = 'encodées'  # Exactly 9 bytes of UTF-8 data
+            sl[0] = 'encodés'  # Exactly 8 bytes of UTF-8 data
         self.assertEqual(sl[1], b'HoWdY')
         with self.assertRaisesRegex(ValueError,
                                     "exceeds available storage"):