Merge pull request #13 from half-pie/fix-ratio

Fix ratio && update readme

Author: alingse

README.md

@@ -1,4 +1,4 @@
-## Usage
+# Usage
 
 ```python
 In [1]: from half_json.core import JSONFixer
@@ -10,37 +10,66 @@ Out[3]: FixResult(success=True, line='[{}]', origin=False)
 
 In [4]: f.fix('{"a')
 Out[4]: FixResult(success=True, line='{"a":null}', origin=False)
+
+In [5]: f.fix('{"a":}')
+Out[5]: FixResult(success=True, line='{"a":null}', origin=False)
 ```
 
 ## 目标
 
-修复残破的 json
+fix invalid/broken/truncated json
 
-## 修复原理
+# 修复原理
 
-1. 根据异常提示来做一些操作, json 预期啥给啥
-2. 根据文本前后,删除一些 BadCase
+1. JSONDecoderError
+2. line context
 
 ## HitRatio
 
+根据 checks 里面的工具来衡量修改效果
+
+ABC : autogen --> broken --> check
+TSR : run test.sh show.sh ratio.sh
+
+### FixRatio
+
+仅判断 result.success == True
+
 ```bash
-./runtest.sh
-# 查看准确率
-seq 1 10|xargs -I {} ./runtest.sh|grep ratio: |awk '{t += $3; h+= $6}{print h/t}'|tail -1
+./runratio.sh fix
 ```
-1. 0.4269, 0.4287, 0.4303   # 实现完 12 条规则
-2. 0.5037, 0.5084, 0.5077   # string 的 " 补充在末尾
-3. 0.5259, 0.5224, 0.5187   # Array 需要 pos - 2
-4. 0.5433, 0.5311, 0.5381   # Array 细化一下 [, 的情况
-5. 0.7192, 0.7216, 0.7265   # 大改进, FIX 之前的 Bug( parser 被冲掉了)
-6. 0.7732, 0.7686, 0.7701   # case: {"a":1 --> 补充 }
-7. 0.60  , 0.58             # 去掉了空行
-8. 0.6971, 0.6969, 0.6984   # 增加处理 StopIteration
-9. 0.7428, 0.7383, 0.7427   # 增加处理 half parse
-10. 0.7617,0.7631, 0.7558   # 细化处理 half parse
-11. 0.7608,0.7612, 0.7650   # 添加从左处理
-12. 0.9817,0.9827, 0.9819   # 增加对字符串的处理
-13. 0.8314,0.8302, 0.8312   # 去掉对字符串的额外处理
+```
+1.  0.4269, 0.4287, 0.4303   # 实现完 12 条规则
+2.  0.5037, 0.5084, 0.5077   # string 的 " 补充在末尾
+3.  0.5259, 0.5224, 0.5187   # Array 需要 pos - 2
+4.  0.5433, 0.5311, 0.5381   # Array 细化一下 [, 的情况
+5.  0.7192, 0.7216, 0.7265   # 大改进, FIX 之前的 Bug( parser 被冲掉了)
+6.  0.7732, 0.7686, 0.7701   # case: {"a":1 --> 补充 }
+7.  0.60  , 0.58             # 去掉了空行
+8.  0.6971, 0.6969, 0.6984   # 增加处理 StopIteration
+9.  0.7428, 0.7383, 0.7427   # 增加处理 half parse
+10. 0.7617, 0.7631, 0.7558   # 细化处理 half parse
+11. 0.7608, 0.7612, 0.7650   # 添加从左处理
+12. 0.9817, 0.9827, 0.9819   # 增加对字符串的处理
+13. 0.8314, 0.8302, 0.8312   # 去掉对字符串的额外处理
+14. 0.95X                    # 已不可参考
+```
+
+### Real HitRatio
+
+判断 result.success == True
+
+并且解析后的 json 大体和原来一致(equal && dictdiffer)
+
+```bash
+./runratio.sh hit
+```
+```
+1. 0.5610, 0.5563, 0.5529   # origin
+2. 0.5593, 0.5532, 0.5587   # fix :} --> :null}
+```
+
+# TODO
 
 ## 目前的缺点 && 发现
 
@@ -50,8 +79,6 @@ seq 1 10|xargs -I {} ./runtest.sh|grep ratio: |awk '{t += $3; h+= $6}{print h/t}
    应该明确 JSONFixer 的能力范围, 对 runratio.sh 也应该比较前后两个的 json 相似程度。
    (听起来像无能者的辩白?)
 
-## TODO
-
 1. 考虑分支回溯的方式来试探
 2. 解析缺失的 JSON 常量
 

checks/gen.py checks/autogen.py

@@ -6,6 +6,7 @@
 
 def borken(s):
     idx = random.randint(0, len(s) + 1)
+    # TODO add count
     return s[:idx] + s[idx + 1:]
 
 
@@ -23,8 +24,11 @@ def main(inflie, outfile):
             try:
                 json.loads(new_line)
             except Exception:
-                # only broken
-                outf.write(new_line)
+                out = {
+                    'origin': line,
+                    'broken': new_line
+                }
+                outf.write(json.dumps(out))
                 outf.write('\n')
 
     inf.close()

checks/broken.py

@@ -0,0 +1,48 @@
+# coding=utf8
+import sys
+import json
+
+from half_json.core import JSONFixer
+
+f = JSONFixer(100)
+
+
+def json_equal(line, origin):
+    return json.loads(line) == json.loads(origin)
+
+
+def main(inflie, outfile):
+    inf = open(inflie, 'r')
+    outf = open(outfile, 'w')
+
+    total = 0
+    hit = 0
+    fix = 0
+
+    for line in inf:
+        info = json.loads(line)
+        result = f.fix(info['broken'])
+        info['fixed'] = result.success
+        info['fix'] = result.line
+        info['hited'] = False
+        if info['fixed']:
+            info['hited'] = json_equal(result.line, info['origin'])
+
+        outf.write(json.dumps(info))
+        outf.write('\n')
+
+        if info['fixed']:
+            fix += 1
+        if info['hited']:
+            hit += 1
+        total += 1
+    print 'total: %d fix: %d hit: %d' % (total, fix, hit)
+    print 'fix ratio: %f' % (fix * 1.0 / total)
+    print 'hit ratio: %f' % (hit * 1.0 / total)
+
+    inf.close()
+    outf.close()
+
+
+if __name__ == '__main__':
+    main(sys.argv[1], sys.argv[2])

checks/check.py

@@ -1 +1 @@
-1-
+[,]

checks/oneline.json

@@ -1,2 +1,7 @@
 #!/bin/bash
-seq 1 20|xargs -I {} ./runtest.sh|grep ratio: |awk '{t += $3; h+= $6}{print h/t}'|tail -1
+mode=$1
+if [ ! $mode ]; then
+  mode=fix
+fi
+seq 1 20|xargs -P 4 -I {} ./runtest.sh {}|grep ratio:|grep $mode|awk -v mode="$mode" '{t += $3}END{printf("%s: %f \n", mode, t/NR)}'
+rm random.*

checks/runratio.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+id=$1
+if [ ! $id ]; then
+  id=1
+fi
+base_name=random.$id
+cat $base_name.broken.uniq.fix.json|jq -r 'select((.fixed == true) and (.hited == false))|("orgin: "+."origin", "broken:"+."broken","fix:   "+."fix")'
+# cat $base_name.broken.uniq.fix.json|jq -r 'select((.fixed == false))|("orgin: "+."origin", "broken:"+."broken","fix:   "+."fix")'

checks/runshow.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
-python gen.py > random.1.json
-python broken.py random.1.json random.1.broken.json
-jsonfixer random.1.broken.json random.1.broken.fix.json
+id=$1
+if [ ! $id ]; then
+  id=1
+fi
+base_name=random.$id
+python autogen.py > $base_name.json
+python broken.py $base_name.json $base_name.broken.json
+cat $base_name.broken.json|sort|uniq > $base_name.broken.uniq.json
+python check.py $base_name.broken.uniq.json $base_name.broken.uniq.fix.json

checks/runtest.sh

@@ -103,6 +103,9 @@ def patch_value_error(self, line, err_info):
                 if lastchar == "{":
                     return False, insert_line(line, "}", pos)
                 return False, insert_line(line, "null}", pos)
+            # :} --> :null}
+            if nextchar == "}":
+                return False, insert_line(line, "null", pos)
             # 08.2
             return False, insert_line(line, "\"", pos)
         # 09
@@ -112,11 +115,14 @@ def patch_value_error(self, line, err_info):
             return False, insert_line(line, ",", pos)
         # 11
         if error == errors.ArrayExceptObject:
-            # fix-error
+            # fix [, --> [
             if lastchar == "[" and nextchar == ",":
                 return False, remove_line(line, pos, pos + 1)
             if nextchar == ",":
                 return False, insert_line(line, "null", pos)
+            # ,] --> ]
+            if nextchar == "]":
+                return False, remove_line(line, pos - 1, pos)
             # 11.1
             if nextchar == "":
                 # quick

half_json/core.py

@@ -13,7 +13,7 @@
 
 setup(
     name='jsonfixer',
-    version='0.1.4b2',
+    version='0.1.4',
     url='https://github.com/half-pie/half-json',
     description='jsonfixer: fix invalid json: broken-json / truncated-json.',
     long_description_content_type='text/x-rst',

setup.py

@@ -84,3 +84,21 @@ def test_case_miss_key(self):
         ok, newline, _ = JSONFixer().fix(line)
         self.assertTrue(ok)
         self.assertEqual('{"":[]}', newline)
+
+    def test_object_miss_value(self):
+        line = '{"V":}'
+        ok, newline, _ = JSONFixer().fix(line)
+        self.assertTrue(ok)
+        self.assertEqual('{"V":null}', newline)
+
+    def test_array_miss_value(self):
+        line = '[,]'
+        ok, newline, _ = JSONFixer().fix(line)
+        self.assertTrue(ok)
+        self.assertEqual('[]', newline)
+
+    def test_array_miss_value_2(self):
+        line = '[null,]'
+        ok, newline, _ = JSONFixer().fix(line)
+        self.assertTrue(ok)
+        self.assertEqual('[null]', newline)