Broadcast Version Conversion Adapters (Add, Mul, Gemm) (onnx#1284)

* Squashing all commits into one

* Fixed supported adapters list

* Long Int troubles

* Will int64s save the day?

* Remove comments in test_backend_test

* Updated broadcastable methods to accurately reflect purpose

* Attempt to resolve unsigned int issue

* Fixed trans bug in gemm

* New Gemm_6_7

* Fixed axis index

* Changed name of CompatibleAdapter and added description in BroadcastBackwardCompatible

* Attempt to resolve int64_t conversion issues in helper

* Unsigned int issue

* New approach to int64_t issue and novel numpy_multibroadcastable approach

* Unsqueeze instead of Reshape for broadcast_forward_compatibility, more assertions of sizes not being params

* Made unallowed_types a constructor parameter for TypeRestriction

* Changed long int back to int64_t in broadcast_forward_compatibility

* renamed opset7_requires_broadcasting

* Abstracted input validation

* Switched num_inputs in assertInputsAvailable from int to int64_t

* switch int64_t to uint64_t

* Removed test_mvn

* One more cast to int from int64_t in assertInputsAvailable

* Addressed code review except numpy_unibroadcastable

* insertBefore instead of moveBefore, changing BackwardsCompatibleTest, setting sizes of unsqueeze output

* fixed naming of broadcasting assertions

* Changed assert_numpy_unibroadcastable_and_require_broadcast to check_numpy_unibroadcastable_and_require_broadcast

Author: Ac2zoom

.gitignore

@@ -86,6 +86,7 @@ compile_commands.json
 .coverage
 onnx/examples/.coverage.nbval
 .pytest_cache
+test_report
 
 # autocomplete
 .ycm_extra_conf.py

docs/VersionConverter.md

@@ -32,19 +32,21 @@ ModelProto ConvertVersion(
     const OpSetID& target_version);
 ```
 
-which accepts an input `ModelProto`, the initial opset version of the model, 
-and the target opset verison, and which returns a new `ModelProto` which 
+which accepts an input `ModelProto`, the initial opset version of the model,
+and the target opset verison, and which returns a new `ModelProto` which
 is the result of apply all relevant adapters between initial_version and
-target_version. For a list of available passes, see 
+target_version. For a list of available passes, see
 [convert.h](onnx/version_converter/convert.h).
 
 Implementing Adapters
 
-You can implement a new adapter by subclassing `Adapter`, and registering 
-your new adapter with `VersionConverter::registerAdapter()`. Adapters operate 
-on an in-memory graph representation defined in [ir.h](onnx/common/ir.h). 
-There are a number of examples in the [adapters](onnx/version_converter/adapters) 
-directory.
+You can implement a new adapter by subclassing `Adapter`, and registering
+your new adapter with `VersionConverter::registerAdapter()`. Adapters operate
+on an in-memory graph representation defined in [ir.h](onnx/common/ir.h).
+There are a number of examples in the [adapters](onnx/version_converter/adapters)
+directory.  Please ensure that all adapters convert from opset version i to i + 1 
+or i - 1, i.e. from Version 6 to Version 5 or vice versa, even if the 2 versions
+being converted between are Version 1 and Version 6.
 
 If your adapter applies in the default domain, please consider adding it
 to the core ONNX repository

onnx/common/interned_strings.h

@@ -136,7 +136,10 @@ _(body) \
 _(then_branch) \
 _(else_branch) \
 _(Captured) \
-_(__control_inputs)
+_(__control_inputs) \
+_(count_include_pad) \
+_(storage_order) \
+_(Unsqueeze)
 
 enum BuiltinSymbol {
   #define DEFINE_SYMBOL(s) \

onnx/common/ir.h

@@ -816,7 +816,6 @@ class OpSetID final {
       }
     }
 
-
     const std::string& domain() const {
       return domain_;
     }

onnx/common/ir_pb_converter.cc

@@ -507,18 +507,17 @@ void encodeGraph(GraphProto * p_g, const std::shared_ptr<Graph> & g) {
     }
     for(auto output : node->outputs()) {
       p_n->add_output(value_name(output));
-
       // only save it if
       //  - it has actual information worth saving
       //  - it's not already saved in the graph outputs value info
       if (graph_outputs.find(output) != graph_outputs.end()) {
         continue;
       }
-      if (output->elemType() == ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED &&
+      if (output->elemType() == TensorProto_DataType_UNDEFINED &&
           output->sizes().empty()) {
         continue;
       }
-      ONNX_NAMESPACE::ValueInfoProto* v = p_g->add_value_info();
+      ValueInfoProto* v = p_g->add_value_info();
       encodeValueInfo(v, output);
     }
     p_n->set_op_type(node->kind().toString());

onnx/cpp2py_export.cc

@@ -268,6 +268,7 @@ PYBIND11_MODULE(onnx_cpp2py_export, onnx_cpp2py_export) {
       [](const py::bytes& bytes, const py::int_ target) {
         ModelProto proto{};
         ParseProtoFromPyBytes(&proto, bytes);
+        shape_inference::InferShapes(proto);
         auto const result = version_conversion::ConvertVersion(std::move(proto),
           target);
         std::string out;

onnx/optimizer/optimize.cc

@@ -5,6 +5,7 @@
 
 namespace ONNX_NAMESPACE { namespace optimization {
 
+// TODO: Remove this static reference
 static Optimizer _optimizer;
 
 ModelProto Optimize(

onnx/test/test_backend_test.py

@@ -12,6 +12,7 @@
 from onnx.backend.base import Device, DeviceType
 from onnx.backend.test.runner import BackendIsNotSupposedToImplementIt
 import onnx.shape_inference
+import onnx.version_converter
 from typing import Optional, Text, Any, Tuple, Sequence
 from onnx import NodeProto, ModelProto, TensorProto
 import numpy  # type: ignore
@@ -41,7 +42,7 @@ def prepare(cls,
         model = onnx.shape_inference.infer_shapes(model)
         value_infos = {vi.name: vi for vi in itertools.chain(model.graph.value_info, model.graph.output)}
 
-        if do_enforce_shape_inference_coverage(model):
+        if do_enforce_test_coverage_whitelist(model):
             for node in model.graph.node:
                 for i, output in enumerate(node.output):
                     if node.op_type == 'Dropout' and i != 0:
@@ -75,13 +76,13 @@ def supports_device(cls, device):  # type: (Text) -> bool
         return False
 
 
-shape_coverage_whitelist = set(
+test_coverage_whitelist = set(
     ['bvlc_alexnet', 'densenet121', 'inception_v1', 'inception_v2',
      'resnet50', 'shufflenet', 'SingleRelu', 'squeezenet_old', 'vgg19', 'zfnet'])
 
 
-def do_enforce_shape_inference_coverage(model):  # type: (ModelProto) -> bool
-    if model.graph.name not in shape_coverage_whitelist:
+def do_enforce_test_coverage_whitelist(model):  # type: (ModelProto) -> bool
+    if model.graph.name not in test_coverage_whitelist:
         return False
     for node in model.graph.node:
         if node.op_type in set(['RNN', 'LSTM', 'GRU']):

onnx/test/version_converter_test.py

@@ -31,17 +31,20 @@ def _converted(
     # Test 1: Backwards Incompatible Conversion: Reshape: 8 -> 2
     def test_backwards_incompatible(self):  # type: () -> None
         def test():  # type: () -> None
-            nodes = [helper.make_node('Reshape', ["X", "shape"], ["Y"])]
+            nodes = [helper.make_node('Add', ["W", "Z"], ["shape"]),
+                        helper.make_node('Reshape', ["X", "shape"], ["A"]),
+                        helper.make_node('Add', ["A", "W"], ["Y"])]
             graph = helper.make_graph(
                 nodes,
                 "test",
                 [helper.make_tensor_value_info("X", TensorProto.FLOAT, (5,)),
-                    helper.make_tensor_value_info("shape", TensorProto.FLOAT, (1,))],
+                    helper.make_tensor_value_info("W", TensorProto.FLOAT, (1,)),
+                    helper.make_tensor_value_info("Z", TensorProto.FLOAT, (1,))],
                 [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
             self._converted(graph, helper.make_operatorsetid("", 8), 2)
         self.assertRaises(RuntimeError, test)
 
-    # Test 2: Backwards Compatible Conversion: Add: 8 -> 7
+    # Test 2: Backwards Compatible Conversion (No Adaptations): Add: 3 -> 2
     def test_backwards_compatible(self):  # type: () -> None
         nodes = [helper.make_node('Add', ["X1", "X2"], ["Y"])]
         graph = helper.make_graph(
@@ -51,10 +54,10 @@ def test_backwards_compatible(self):  # type: () -> None
                 helper.make_tensor_value_info("X2", TensorProto.FLOAT, (5,))],
             [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
         converted_model = self._converted(graph, helper.make_operatorsetid(
-            "", 8), 7)
+            "", 3), 2)
         # Assert equality of graph and converted_model
         assert converted_model.graph.node[0].op_type == "Add"
-        assert converted_model.opset_import[0].version == 7
+        assert converted_model.opset_import[0].version == 2
 
     # Test 3: Non-Existent Op Conversion: Cos: 8 -> 6
     def test_non_existent_op(self):  # type: () -> None
@@ -68,6 +71,98 @@ def test():  # type: () -> None
             self._converted(graph, helper.make_operatorsetid("", 8), 6)
         self.assertRaises(RuntimeError, test)
 
+    # Test Add Adapter: 8 -> 5
+    def test_add_8_5(self):  # type: () -> None
+        nodes = [helper.make_node('Add', ["X1", "X2"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("X1", TensorProto.FLOAT, (5,)),
+                helper.make_tensor_value_info("X2", TensorProto.FLOAT, (1,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 8), 5)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Add"
+        assert converted_model.opset_import[0].version == 5
+
+    # Test Add Adapter: 5 -> 8
+    def test_add_5_8(self):  # type: () -> None
+        nodes = [helper.make_node('Add', ["X1", "X2"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("X1", TensorProto.FLOAT, (5,)),
+                helper.make_tensor_value_info("X2", TensorProto.FLOAT, (1,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 5), 8)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Add"
+        assert converted_model.opset_import[0].version == 8
+
+    # Test Mul Adapter: 8 -> 5
+    def test_mul_8_5(self):  # type: () -> None
+        nodes = [helper.make_node('Mul', ["X1", "X2"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("X1", TensorProto.FLOAT, (5,)),
+                helper.make_tensor_value_info("X2", TensorProto.FLOAT, (1,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 8), 5)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Mul"
+        assert converted_model.opset_import[0].version == 5
+
+    # Test Mul Adapter: 5 -> 8
+    def test_mul_5_8(self):  # type: () -> None
+        nodes = [helper.make_node('Mul', ["X1", "X2"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("X1", TensorProto.FLOAT, (5,)),
+                helper.make_tensor_value_info("X2", TensorProto.FLOAT, (1,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 5), 8)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Mul"
+        assert converted_model.opset_import[0].version == 8
+
+    # Test Gemm Adapter: 1 -> 8
+    def test_gemm_up(self):  # type: () -> None
+        nodes = [helper.make_node('Gemm', ["A", "B", "C"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("A", TensorProto.FLOAT, (5, 5,)),
+                helper.make_tensor_value_info("B", TensorProto.FLOAT, (5, 5,)),
+                helper.make_tensor_value_info("C", TensorProto.FLOAT, (5, 5,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5, 5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 1), 8)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Gemm"
+        assert converted_model.opset_import[0].version == 8
+
+    # Test Gemm Adapter: 8 -> 1
+    def test_gemm_down(self):  # type: () -> None
+        nodes = [helper.make_node('Gemm', ["A", "B", "C"], ["Y"])]
+        graph = helper.make_graph(
+            nodes,
+            "test",
+            [helper.make_tensor_value_info("A", TensorProto.FLOAT, (5, 5,)),
+                helper.make_tensor_value_info("B", TensorProto.FLOAT, (5, 5,)),
+                helper.make_tensor_value_info("C", TensorProto.FLOAT, (5, 5,))],
+            [helper.make_tensor_value_info("Y", TensorProto.FLOAT, (5, 5,))])
+        converted_model = self._converted(graph, helper.make_operatorsetid(
+            "", 8), 1)
+        # Assert equality of graph and converted_model
+        assert converted_model.graph.node[0].op_type == "Gemm"
+        assert converted_model.opset_import[0].version == 1
+
 
 if __name__ == '__main__':
     unittest.main()