Merge pull request #18 from alingse/add-unicode-enhance

support unicode

.gitignore

@@ -9,6 +9,9 @@ __pycache__/
 # nosetests
 cover/
 
+# test fixture
+fixture/files/tmp.*
+
 # Distribution / packaging
 .Python
 env/

.travis.yml

@@ -13,7 +13,9 @@ script:
     - flake8 jsoncsv tests --max-line-length=200
     - nosetests --cover-package=jsoncsv --with-coverage
     - python setup.py install
-    - cat fixture/raw.json|jsoncsv -e > fixture/expand.test.json && diff fixture/expand.json fixture/expand.test.json
+    - cat fixture/files/raw.0.json|jsoncsv -e > fixture/files/tmp.expand.0.json && diff fixture/files/expand.0.json fixture/files/tmp.expand.0.json
+    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
+    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
 
 after_success:
   - coveralls

ChangeLog

@@ -0,0 +1,21 @@
+2018.05.04
+
+    2.0.8 support unicode
+
+2018.05.03
+
+    2.0.8b fix setup on windows(maybe)
+
+2018.04.09
+
+    2.0.8a clean code & add travis
+
+2018.03.30
+
+    2.0.7 support sort headers on dump csv
+
+2017.05.10
+
+    2.0.6 clean the mkexcel
+
+....... the earlier version is jsut... debug

fixture/files/expand.1.json

@@ -0,0 +1,4 @@
+{"detail.when": 2016, "detail.what": "eat", "title": "AAAA", "detail.where": "beijing", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one"}
+{"detail.when": 2016, "detail.what": "play", "title": "BBBB", "detail.where": "湖北", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one", "detail.how": "offline"}
+{"detail.when": 2015, "title": "CCCC", "detail.where": "chengdu", "detail.who.1": "three", "detail.who.0": "two"}
+{"detail.when": 2015, "detail.how": "online", "detail.what": "happy", "title": "DDDD"}

fixture/files/raw.2.json

@@ -0,0 +1 @@
+[{"河流名字": "长江", "河流长度": "6000千米"}, {"河流名字": "黄河", "河流长度": "5000千米"}]

jsoncsv/dumptool.py

@@ -20,9 +20,6 @@ def patch(self, value):
         if value is None:
             return ''
 
-        if isinstance(value, basestring):
-            return value.encode('utf-8')
-
         if value == {} or value == []:
             return ''
 
@@ -108,27 +105,27 @@ def initialize(self, **kwargs):
 
     def write_headers(self):
         header = self._separator.join(self._headers)
-        self.fout.write(header)
+        self.fout.write(header.encode('utf-8'))
         self.fout.write('\n')
 
     def patch(self, value):
         value = super(DumpCSV, self).patch(value)
-        return str(value)
+        return unicode(value)
 
     def write_obj(self, obj):
         values = [
             self.patch(obj.get(head))
             for head in self._headers
         ]
         content = self._separator.join(values)
-        self.fout.write(content)
+        self.fout.write(content.encode('utf-8'))
         self.fout.write('\n')
 
 
 class DumpXLS(DumpExcel):
 
     def initialize(self, **kwargs):
-        super(DumpCSV, self).initialize(**kwargs)
+        super(DumpXLS, self).initialize(**kwargs)
 
         self.sheet = kwargs.get('sheet', 'Sheet1')
         self.wb = xlwt.Workbook(encoding='utf-8')

jsoncsv/jsontool.py

@@ -41,7 +41,7 @@ def gen_leaf(root, path=None):
 
 
 int_digit = lambda x: isinstance(x, int)  # noqa
-str_digit = lambda x: isinstance(x, str) and x.isdigit()  # noqa
+str_digit = lambda x: isinstance(x, basestring) and x.isdigit()  # noqa
 
 
 def is_array(keys, ensure_str=True):
@@ -86,7 +86,7 @@ def expand(origin, separator='.', safe=False):
 
     expobj = {}
     for path, value in leafs:
-        path = map(str, path)
+        path = map(unicode, path)
         if safe:
             key = encode_safe_key(path, separator)
         else:

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.0.8b',
+    version='2.0.8',
     description='a command tool easily convert json file to csv or xlsx',
     long_description=readme,
     author='alingse',

tests/test_jsoncsv.py

@@ -12,6 +12,6 @@ class Testjsoncsv(unittest.TestCase):
 
     def test_jsoncsv_expand(self):
         runner = CliRunner()
-        args = ['-e', 'fixture/raw.json', 'fixture/expand.2.json']
+        args = ['-e', 'fixture/files/raw.0.json', 'fixture/files/tmp.expand.0.json']
         result = runner.invoke(jsoncsv, args=args)
         assert result.exit_code == 0

tests/test_jsontool.py

@@ -64,3 +64,12 @@ def test_is_array(self):
         assert is_array(['0', '1', '2', '3'])
         assert not is_array([1, 2, 3])
         assert not is_array(['0', 1, 2])
+
+    def test_unicode(self):
+        data = [
+            {u"河流名字": u"长江", u"河流长度": u"6000千米"},
+            {u"河流名字": u"黄河", u"河流长度": u"5000千米"}
+        ]
+
+        expobj = expand(data)
+        assert expobj

tests/test_mkexcel.py

@@ -9,32 +9,33 @@
 
 class Testdumptool(unittest.TestCase):
 
+    # FIXME (使用虚拟文件)
     def test_dumpexcel_csv(self):
-        fin = open('./fixture/expand.json', 'r')
-        fout = open('./fixture/output2.csv', 'w')
+        fin = open('./fixture/files/expand.1.json', 'r')
+        fout = open('./fixture/files/tmp.output.1.csv', 'w')
 
         dumpexcel(fin, fout, 'csv')
         fin.close()
         fout.close()
 
-        output = open('./fixture/output.csv', 'r')
-        fout = open('./fixture/output2.csv', 'r')
+        output = open('./fixture/files/output.1.csv', 'r')
+        fout = open('./fixture/files/tmp.output.1.csv', 'r')
 
         self.assertEqual(output.read(), fout.read())
 
         output.close()
         fout.close()
 
     def test_dumpexcel_csv_with_sort(self):
-        fin = open('./fixture/expand.json', 'r')
-        fout = open('./fixture/tmp.csv', 'w')
+        fin = open('./fixture/files/expand.1.json', 'r')
+        fout = open('./fixture/files/tmp.output.1.sort.csv', 'w')
 
         dumpexcel(fin, fout, 'csv', sort_type=True)
         fin.close()
         fout.close()
 
-        output = open('./fixture/output.sort.csv', 'r')
-        fout = open('./fixture/tmp.csv', 'r')
+        output = open('./fixture/files/output.1.sort.csv', 'r')
+        fout = open('./fixture/files/tmp.output.1.sort.csv', 'r')
 
         self.assertEqual(output.read(), fout.read())