Automatically skip unit tests of not implemented operators in onnx-tf

Author: winnietsang

doc/CLI.md

@@ -22,7 +22,7 @@ optional arguments:
 ### Convert:
 
 #### From ONNX to Tensorflow:
-`onnx-tf convert -t tf -i /path/to/input.onnx -o /path/to/output.pb`
+`onnx-tf convert -i /path/to/input.onnx -o /path/to/output.pb`
 
 More information: `onnx-tf convert -h`
 ```

doc/CLI_template.md

@@ -14,7 +14,7 @@ More information: `onnx-tf -h`
 ### Convert:
 
 #### From ONNX to Tensorflow:
-`onnx-tf convert -t tf -i /path/to/input.onnx -o /path/to/output.pb`
+`onnx-tf convert -i /path/to/input.onnx -o /path/to/output.pb`
 
 More information: `onnx-tf convert -h`
 ```

test/backend/test_onnx_backend.py

@@ -4,19 +4,54 @@
 from __future__ import unicode_literals
 
 import os
+import re
 import unittest
 
 import onnx.backend.test
 
+from onnx import defs
+
+from onnx_tf import opset_version
 from onnx_tf.backend import TensorflowBackend
 from onnx_tf.common.legacy import legacy_onnx_pre_ver
 from onnx_tf.common.legacy import legacy_opset_pre_ver
 
+def get_onnxtf_supported_ops():
+  return opset_version.backend_opset_version
+
+def get_onnx_supported_ops():
+  onnx_opset_dict = {}
+  for schema in defs.get_all_schemas():
+    op = schema.name
+    onnx_opset_dict[op] = schema.since_version
+  return onnx_opset_dict
+
+def skip_not_implemented_ops_test(test):
+  onnxtf_ops_list = get_onnxtf_supported_ops()
+  onnx_ops_list = get_onnx_supported_ops()
+  for op in onnx_ops_list:
+    if op in onnxtf_ops_list:
+      if onnx_ops_list[op] not in onnxtf_ops_list[op]:
+        test.exclude(r'[a-z,_]*' + op.lower() + '[a-z,_]*')
+    else:
+      test.exclude(r'[a-z,_]*' + op.lower() + '[a-z,_]*')
+  return test
+
 # This is a pytest magic variable to load extra plugins
 pytest_plugins = 'onnx.backend.test.report',
 
 backend_test = onnx.backend.test.BackendTest(TensorflowBackend, __name__)
 
+# exclude tests of not-implemented-ops
+backend_test = skip_not_implemented_ops_test(backend_test)
+
+# manually exclude tests of not-implemented-ops that are using "short name" in their testcase name
+# need to remove these lines once those ops support are added into onnx-tf
+# temporary exclude StringNormalizer test
+backend_test.exclude(r'[a-z,_]*strnorm[a-z,_]*')
+# temporary exclude MeanVarianceNormalization test
+backend_test.exclude(r'[a-z,_]*mvn[a-z,_]*')
+
 # https://github.com/onnx/onnx/issues/349
 backend_test.exclude(r'[a-z,_]*GLU[a-z,_]*')
 

test/test_cli.py

@@ -68,8 +68,6 @@ def test_convert_to_tf(self):
       subprocess.check_call([
           "onnx-tf",
           "convert",
-          "-t",
-          "tf",
           "-i",
           os.path.join(model_dir, '{}.onnx'.format(model_name)),
           "-o",