[Andrew Polyakov] Add support for NDArray in python package

Author: AndrewPolyakov1

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/deep_serialization.py

@@ -50,6 +50,7 @@ def serialize_objects_dump(objs: List[Any], clear_visited: bool = False) -> Tupl
         serializer.write_object_to_memory(obj)
         for obj in objs
     ]
+
     return ids, serializer.memory, serialize_memory_dump(serializer.memory)
 
 

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py

@@ -11,9 +11,10 @@
     ListMemoryObject,
     DictMemoryObject,
     ReduceMemoryObject,
-    MemoryDump, IteratorMemoryObject,
+    MemoryDump, IteratorMemoryObject, NdarrayMemoryObject,
 )
 from utbot_executor.deep_serialization.utils import PythonId, TypeInfo
+import numpy as np
 
 
 class MemoryObjectEncoder(json.JSONEncoder):
@@ -27,6 +28,10 @@ def default(self, o):
             }
             if isinstance(o, ReprMemoryObject):
                 base_json["value"] = o.value
+            elif isinstance(o, NdarrayMemoryObject):
+                base_json["items"] = o.items
+                base_json["comparable"] = True
+                # raise AttributeError(base_json)
             elif isinstance(o, (ListMemoryObject, DictMemoryObject)):
                 base_json["items"] = o.items
             elif isinstance(o, IteratorMemoryObject):
@@ -53,6 +58,9 @@ def default(self, o):
                 "kind": o.kind,
                 "module": o.module,
             }
+        if isinstance(o, np.ndarray):
+            # raise TypeError(f'Object {o.tolist()}, type: {type(o)}')
+            return all(o.tolist())
         return json.JSONEncoder.default(self, o)
 
 
@@ -75,6 +83,16 @@ def as_reduce_object(dct: Dict) -> Union[MemoryObject, Dict]:
             )
             obj.comparable = dct["comparable"]
             return obj
+
+        if dct["strategy"] == "ndarray":
+            obj = NdarrayMemoryObject.__new__(NdarrayMemoryObject)
+            obj.items = dct["items"]
+            obj.typeinfo = TypeInfo(
+                kind=dct["typeinfo"]["kind"], module=dct["typeinfo"]["module"]
+            )
+            obj.comparable = dct["comparable"]
+            return obj
+
         if dct["strategy"] == "dict":
             obj = DictMemoryObject.__new__(DictMemoryObject)
             obj.items = dct["items"]
@@ -138,6 +156,10 @@ def reload_id(self) -> MemoryDump:
                 new_memory_object.items = [
                     self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
                 ]
+            elif isinstance(new_memory_object, NdarrayMemoryObject):
+                new_memory_object.items = [
+                    self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
+                ]
             elif isinstance(new_memory_object, IteratorMemoryObject):
                 new_memory_object.items = [
                     self.dump_id_to_real_id[id_] for id_ in new_memory_object.items
@@ -198,6 +220,17 @@ def load_object(self, python_id: PythonId) -> object:
 
                 for item in dump_object.items:
                     real_object.append(self.load_object(item))
+        elif isinstance(dump_object, NdarrayMemoryObject):
+            # print(f"Hi", file=sys.stderr)
+            real_object = []
+
+            id_ = PythonId(str(id(real_object)))
+            self.dump_id_to_real_id[python_id] = id_
+            self.memory[id_] = real_object
+
+            for item in dump_object.items:
+                real_object = np.append(real_object, self.load_object(item))
+                # real_object.append()
         elif isinstance(dump_object, DictMemoryObject):
             real_object = {}
 
@@ -250,7 +283,7 @@ def load_object(self, python_id: PythonId) -> object:
                     for key, dictitem in dictitems.items():
                         real_object[key] = dictitem
         else:
-            raise TypeError(f"Invalid type {dump_object}")
+            raise TypeError(f"Invalid type {dump_object}, type: {type(dump_object)}")
 
         id_ = PythonId(str(id(real_object)))
         self.dump_id_to_real_id[python_id] = id_
@@ -279,6 +312,10 @@ def main():
             "builtins.tuple",
             "builtins.bytes",
             "builtins.type",
+            "numpy.ndarray"
         ]
     )
     print(loader.load_object(PythonId("140239390887040")))
+
+if __name__ == '__main__':
+    main()

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py

@@ -8,6 +8,7 @@
 import typing
 from itertools import zip_longest
 from typing import Any, Callable, Dict, List, Optional, Set, Type, Iterable
+import numpy as np
 
 from utbot_executor.deep_serialization.config import PICKLE_PROTO
 from utbot_executor.deep_serialization.iterator_wrapper import IteratorWrapper
@@ -41,7 +42,7 @@ def __init__(self, obj: object) -> None:
         self.id_ = PythonId(str(id(self.obj)))
 
     def _initialize(
-        self, deserialized_obj: object = None, comparable: bool = True
+            self, deserialized_obj: object = None, comparable: bool = True
     ) -> None:
         self.deserialized_obj = deserialized_obj
         self.comparable = comparable
@@ -115,6 +116,30 @@ def initialize(self) -> None:
 
         super()._initialize(deserialized_obj, comparable)
 
+class NdarrayMemoryObject(MemoryObject):
+    strategy: str = "ndarray"
+    items: List[PythonId] = []
+
+    def __init__(self, ndarray_object: object) -> None:
+        self.items: List[PythonId] = []
+        super().__init__(ndarray_object)
+
+    def initialize(self) -> None:
+        serializer = PythonSerializer()
+        self.deserialized_obj = []  # for recursive collections
+        self.comparable = False  # for recursive collections
+
+        for elem in self.obj:
+            elem_id = serializer.write_object_to_memory(elem)
+            self.items.append(elem_id)
+            self.deserialized_obj.append(serializer[elem_id])
+
+        deserialized_obj = self.deserialized_obj
+        comparable = all(serializer.get_by_id(elem).comparable for elem in self.items)
+        # comparable = True
+
+        super()._initialize(deserialized_obj, comparable)
+
     def __repr__(self) -> str:
         if hasattr(self, "obj"):
             return str(self.obj)
@@ -264,10 +289,10 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
 
         is_reconstructor = constructor_kind.qualname == "copyreg._reconstructor"
         is_reduce_user_type = (
-            len(self.reduce_value[1]) == 3
-            and isinstance(self.reduce_value[1][0], type(self.obj))
-            and self.reduce_value[1][1] is object
-            and self.reduce_value[1][2] is None
+                len(self.reduce_value[1]) == 3
+                and isinstance(self.reduce_value[1][0], type(self.obj))
+                and self.reduce_value[1][1] is object
+                and self.reduce_value[1][2] is None
         )
         is_reduce_ex_user_type = len(self.reduce_value[1]) == 1 and isinstance(
             self.reduce_value[1][0], type(self.obj)
@@ -294,8 +319,8 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
                 len(inspect.signature(init_method).parameters),
             )
             if (
-                not init_from_object
-                and len(inspect.signature(init_method).parameters) == 1
+                    not init_from_object
+                    and len(inspect.signature(init_method).parameters) == 1
             ) or init_from_object:
                 logging.debug("init with one argument! %s", init_method)
                 constructor_arguments = []
@@ -317,9 +342,9 @@ def constructor_builder(self) -> typing.Tuple[typing.Any, typing.Callable]:
         if is_reconstructor and is_user_type:
             constructor_arguments = self.reduce_value[1]
             if (
-                len(constructor_arguments) == 3
-                and constructor_arguments[-1] is None
-                and constructor_arguments[-2] == object
+                    len(constructor_arguments) == 3
+                    and constructor_arguments[-1] is None
+                    and constructor_arguments[-2] == object
             ):
                 del constructor_arguments[1:]
             callable_constructor = object.__new__
@@ -392,6 +417,12 @@ def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
             return ListMemoryObject
         return None
 
+class NdarrayMemoryObjectProvider(MemoryObjectProvider):
+    @staticmethod
+    def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
+        if type(obj) == np.ndarray:
+            return NdarrayMemoryObject
+        return None
 
 class DictMemoryObjectProvider(MemoryObjectProvider):
     @staticmethod
@@ -425,6 +456,7 @@ def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
         return None
 
 
+
 class ReprMemoryObjectProvider(MemoryObjectProvider):
     @staticmethod
     def get_serializer(obj: object) -> Optional[Type[MemoryObject]]:
@@ -456,6 +488,7 @@ class PythonSerializer:
         ReduceMemoryObjectProvider,
         ReprMemoryObjectProvider,
         ReduceExMemoryObjectProvider,
+        NdarrayMemoryObjectProvider
     ]
 
     def __new__(cls):

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py

@@ -44,7 +44,7 @@
 
 
 def _update_states(
-    init_memory_dump: MemoryDump, before_memory_dump: MemoryDump
+        init_memory_dump: MemoryDump, before_memory_dump: MemoryDump
 ) -> MemoryDump:
     for id_, obj in before_memory_dump.objects.items():
         if id_ in init_memory_dump.objects:
@@ -246,9 +246,9 @@ def run_pickle_function(self, request: ExecutionRequest) -> ExecutionResponse:
 
 
 def _serialize_state(
-    args: List[Any],
-    kwargs: Dict[str, Any],
-    result: Any = None,
+        args: List[Any],
+        kwargs: Dict[str, Any],
+        result: Any = None,
 ) -> Tuple[List[PythonId], Dict[str, PythonId], PythonId, MemoryDump, str]:
     """Serialize objects from args, kwargs and result.
 
@@ -267,13 +267,13 @@ def _serialize_state(
 
 
 def _run_calculate_function_value(
-    function: types.FunctionType,
-    args: List[Any],
-    kwargs: Dict[str, Any],
-    fullpath: str,
-    state_init: str,
-    tracer: UtTracer,
-    state_assertions: bool,
+        function: types.FunctionType,
+        args: List[Any],
+        kwargs: Dict[str, Any],
+        fullpath: str,
+        state_init: str,
+        tracer: UtTracer,
+        state_assertions: bool,
 ) -> ExecutionResponse:
     """Calculate function evaluation result.
 

utbot-python-executor/src/main/python/utbot_executor/utbot_executor/memory_compressor.py

@@ -3,6 +3,8 @@
 from utbot_executor.deep_serialization.memory_objects import MemoryDump
 from utbot_executor.deep_serialization.utils import PythonId
 
+import numpy as np
+
 
 def compress_memory(
         ids: typing.List[PythonId],
@@ -13,7 +15,10 @@ def compress_memory(
     for id_ in ids:
         if id_ in state_before.objects and id_ in state_after.objects:
             try:
-                if state_before.objects[id_].obj != state_after.objects[id_].obj:
+                if isinstance(state_before.objects[id_].obj, np.ndarray) or isinstance(state_after.objects[id_].obj, np.ndarray):
+                    if (state_before.objects[id_].obj != state_after.objects[id_].obj).all():
+                        diff_ids.append(id_)
+                elif state_before.objects[id_].obj != state_after.objects[id_].obj:
                     diff_ids.append(id_)
             except AttributeError as _:
                 pass