[MIG] Migration to python 3 while keeping compatibility with python 2

[IMP] possibility to use file object or similar instead of file name for tranform, load and extract operation
[TEST] Launch test with python 2 and python 3 + generate coverage report
[MIG] migration odoo-client-lib a fork of openerp-client-lib that is now compatible with python 3
[IMP] change requirements and setu.py

Author: tfr-odoo

odoo_csv_tools/__init__.py

@@ -1,3 +1,3 @@
-import lib
-import export_threaded
-import import_threaded
+from . import lib
+from . import export_threaded
+from . import import_threaded

odoo_csv_tools/export_threaded.py

@@ -14,29 +14,25 @@
 You should have received a copy of the GNU Lesser General Public License
 along with this program. If not, see <http://www.gnu.org/licenses/>.
 '''
-
-from xmlrpclib import Fault
-from time import time
-from itertools import islice, chain
-
-
 import sys
 import csv
 
-from lib import conf_lib
-from lib.conf_lib import log_error, log_info
-from lib.internal.rpc_thread import RpcThread
-from lib.internal.csv_reader import UnicodeWriter
-from odoo_csv_tools.lib.internal.io import ListWriter
+from time import time
 
-csv.field_size_limit(sys.maxint)
+from . lib import conf_lib
+from . lib.conf_lib import log_error, log_info
+from . lib.internal.rpc_thread import RpcThread
+from . lib.internal.csv_reader import UnicodeWriter
+from . lib.internal.io import ListWriter, open_write
+from . lib.internal.tools import batch
 
+if sys.version_info >= (3, 0, 0):
+    from xmlrpc.client import Fault
+    csv.field_size_limit(sys.maxsize)
+else:
+    from xmlrpclib import Fault
+    csv.field_size_limit(sys.maxint)
 
-def batch(iterable, size):
-    sourceiter = iter(iterable)
-    while True:
-        batchiter = islice(sourceiter, size)
-        yield chain([batchiter.next()], batchiter)
 
 class RPCThreadExport(RpcThread):
 
@@ -80,7 +76,7 @@ def export_data(config_file, model, domain, header, context=None, output=None, m
     object_registry = conf_lib.get_server_connection(config_file).get_model(model)
 
     if output:
-        file_result = open(output, "wb")
+        file_result = open_write(output)
         writer = UnicodeWriter(file_result, delimiter=separator, encoding=encoding, quoting=csv.QUOTE_ALL)
     else:
         writer = ListWriter()
@@ -104,6 +100,3 @@ def export_data(config_file, model, domain, header, context=None, output=None, m
         return False, False
     else:
         return writer.header, writer.data
-
-
-

odoo_csv_tools/import_threaded.py

@@ -20,23 +20,22 @@
 import csv
 
 from time import time
-from itertools import islice, chain
-from xmlrpclib import Fault
 
-from lib import conf_lib
-from lib.conf_lib import log_error, log_info, log
-from lib.internal.rpc_thread import RpcThread
-from lib.internal.io import ListWriter
-from lib.internal.csv_reader import UnicodeReader, UnicodeWriter
+from . lib import conf_lib
+from . lib.conf_lib import log_error, log_info, log
+from . lib.internal.rpc_thread import RpcThread
+from . lib.internal.io import ListWriter, open_read, open_write
+from . lib.internal.csv_reader import UnicodeReader, UnicodeWriter
+from . lib.internal.tools import batch
+
+if sys.version_info >= (3, 0, 0):
+    from xmlrpc.client import Fault
+else:
+    from xmlrpclib import Fault
+    from builtins import range
 
-csv.field_size_limit(sys.maxint)
 
 
-def batch(iterable, size):
-    sourceiter = iter(iterable)
-    while True:
-        batchiter = islice(sourceiter, size)
-        yield chain([batchiter.next()], batchiter)
 
 class RPCThreadImport(RpcThread):
 
@@ -141,13 +140,13 @@ def check_id_column(header):
 
     def skip_line(reader):
         log_info("Skipping until line %s excluded" % skip)
-        for _ in xrange(1, skip):
+        for _ in range(1, skip):
             reader.next()
 
     log('open %s' % file_to_read)
-    file_ref = open(file_to_read, 'r')
+    file_ref = open_read(file_to_read, encoding='utf-8-sig')
     reader = UnicodeReader(file_ref, delimiter=delimiter, encoding='utf-8-sig')
-    header = reader.next()
+    header = next(reader)
     header = get_real_header(header)
     check_id_column(header)
     skip_line(reader)
@@ -176,7 +175,7 @@ def import_data(config_file, model, header=None, data=None, file_csv=None, conte
     if file_csv:
         header, data = read_file(file_csv, delimiter=separator, encoding=encoding, skip=skip)
         fail_file = fail_file or file_csv + ".fail"
-        file_result = open(fail_file, "wb")
+        file_result = open_write(fail_file, encoding=encoding)
 
     if not header or data == None:
         raise ValueError("Please provide either a data file or a header and data")
@@ -220,4 +219,3 @@ def import_data(config_file, model, header=None, data=None, file_csv=None, conte
         return False, False
     else:
         return writer.header, writer.data
-

odoo_csv_tools/lib/__init__.py

@@ -1,6 +1,6 @@
-import internal
-import conf_lib
-import workflow
-import checker
-import mapper
-import transform
+from . import internal
+from . import conf_lib
+from . import workflow
+from . import checker
+from . import mapper
+from . import transform

odoo_csv_tools/lib/checker.py

@@ -15,7 +15,7 @@ def check_id_validity(header, data):
             line = [s.strip() if s.strip() not in null_values else '' for s in line]
             line_dict = dict(zip(header, line))
             if not regular.match(line_dict[id_field]):
-                print "Check Failed Id Validity", i+1, line_dict[id_field]
+                print("Check Failed Id Validity", i+1, line_dict[id_field])
                 res = False
         return res
     return check_id_validity
@@ -27,15 +27,15 @@ def check_line_length(header, data):
         for line in data:
             i+=1
             if len(line) != length:
-                print "Check Failed", i, "Line Length", len(line)
+                print("Check Failed", i, "Line Length", len(line))
                 res = False
         return res
     return check_line_length
 
 def line_number_checker(line_number):
     def check_line_numner(header, data):
         if len(data) + 1 != line_number:
-            print "Check Line Number Failed %s instead of %s" % (len(data) + 1, line_number)
+            print("Check Line Number Failed %s instead of %s" % (len(data) + 1, line_number))
             return False
         else:
             return True
@@ -47,8 +47,8 @@ def check_max_cell_len(header, data):
         for i, line in enumerate(data):
             for ele in line:
                 if len(ele) > max_cell_len:
-                    print "Check Failed", i + 1, "Cell Length", len(ele)
-                    print line
+                    print("Check Failed", i + 1, "Cell Length", len(ele))
+                    print(line)
                     res = False
         return res
     return check_max_cell_len

odoo_csv_tools/lib/conf_lib.py

@@ -1,5 +1,9 @@
-import openerplib
-import ConfigParser
+import odoolib
+import sys
+if sys.version_info >= (3, 0, 0):
+    import configparser as ConfigParser
+else:
+    import ConfigParser
 import logging
 import sys
 
@@ -15,7 +19,7 @@ def get_server_connection(config_file):
     protocol = config.get('Connection', 'protocol')
     port = int(config.get('Connection', 'port'))
     uid = int(config.get('Connection', 'uid'))
-    return openerplib.get_connection(hostname=hostname, database=database, login=login, password=password, protocol=protocol, port=port, user_id=uid)
+    return odoolib.get_connection(hostname=hostname, database=database, login=login, password=password, protocol=protocol, port=port, user_id=uid)
 
 def init_logger():
     logger_err = logging.getLogger("error")

odoo_csv_tools/lib/internal/__init__.py

@@ -1,5 +1,5 @@
-import exceptions
-import tools
-import csv_reader
-import io
-import rpc_thread
+from . import exceptions
+from . import tools
+from . import csv_reader
+from . import io
+from . import rpc_thread

odoo_csv_tools/lib/internal/csv_reader.py

@@ -3,35 +3,32 @@
 
 @author: openerp
 '''
-import csv, codecs, cStringIO
+from __future__ import absolute_import
+import sys
+#import csv, codecs
+if sys.version_info >= (3, 0, 0):
+    import csv
+else:
+    import unicodecsv as csv
+from io import StringIO
 import threading
 
-class UTF8Recoder:
-    """
-    Iterator that reads an encoded stream and reencodes the input to UTF-8
-    """
-    def __init__(self, f, encoding):
-        self.reader = codecs.getreader(encoding)(f)
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        return self.reader.next().encode("utf-8")
-
 class UnicodeReader:
     """
     A CSV reader which will iterate over lines in the CSV file "f",
     which is encoded in the given encoding.
     """
 
     def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
-        f = UTF8Recoder(f, encoding)
         self.reader = csv.reader(f, dialect=dialect, **kwds)
 
     def next(self):
-        row = self.reader.next()
-        return [unicode(s, "utf-8") for s in row]
+        #For python2
+        return self.reader.next()
+
+    def __next__(self):
+        #For python3
+        return self.reader.__next__()
 
     def __iter__(self):
         return self
@@ -45,29 +42,17 @@ class UnicodeWriter:
 
     def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
         # Redirect output to a queue
-        self.queue = cStringIO.StringIO()
-        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
         self.stream = f
-        self.encoder = codecs.getincrementalencoder(encoding)()
+        self.writer = writer = csv.writer(f, dialect=dialect, **kwds)
         self.lock = threading.RLock()
 
     def writerow(self, row):
         self.lock.acquire()
-        self.writer.writerow([str(s) if isinstance(s, bool) else s.encode("utf-8") for s in row])
-        # Fetch UTF-8 output from the queue ...
-        data = self.queue.getvalue()
-        data = data.decode("utf-8")
-        # ... and reencode it into the target encoding
-        data = self.encoder.encode(data)
-        # write to the target stream
-        self.stream.write(data)
-        # empty queue
-        self.queue.truncate(0)
+        self.writer.writerow(row)
         self.lock.release()
 
     def writerows(self, rows):
         self.lock.acquire()
-        for row in rows:
-            self.writerow(row)
+        self.writer.writerows(rows)
         self.stream.flush()
         self.lock.release()

odoo_csv_tools/lib/internal/exceptions.py

@@ -5,4 +5,5 @@
 '''
 
 class SkippingException(Exception):
-    pass
+    def __init__(self, message):
+        self.message = message

odoo_csv_tools/lib/internal/io.py

@@ -3,14 +3,40 @@
 
 @author: mythrys
 '''
+from __future__ import absolute_import
+
 import csv
 import os
-
-from csv_reader import UnicodeWriter, UnicodeReader
-
+import sys
+from . csv_reader import UnicodeWriter, UnicodeReader
+
+"""
+    Compatibility layer between python 2.7 and python 3
+"""
+def is_string(f):
+    if sys.version_info >= (3, 0, 0):
+        return isinstance(f, str)
+    else:
+        return isinstance(f, basestring)
+
+def open_read(f, encoding='utf-8-sig'):
+    if not is_string(f):
+        return f
+    if sys.version_info >= (3, 0, 0):
+        return open(f, 'r', newline='', encoding=encoding)
+    else:
+        return open(f, 'r')
+
+def open_write(f, encoding='utf-8-sig'):
+    if not is_string(f):
+        return f
+    if sys.version_info >= (3, 0, 0):
+        return open(f, "w", newline='', encoding=encoding)
+    else:
+        return open(f, "w")
 
 def write_csv(filename, header, data):
-    file_result = open(filename, "wb")
+    file_result = open_write(filename)
     c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
     c.writerow(header)
     for d in data:

odoo_csv_tools/lib/internal/tools.py

@@ -3,12 +3,18 @@
 
 @author: Thibault Francois <[email protected]>
 '''
+from itertools import islice, chain
 
+def batch(iterable, size):
+    sourceiter = iter(iterable)
+    while True:
+        batchiter = islice(sourceiter, size)
+        yield chain([next(batchiter)], batchiter)
 """
     Data formatting tools
 """
 def to_xmlid(name):
-    return name.replace('.', '_').replace(',', '_').strip()
+    return name.replace('.', '_').replace(',', '_').replace('\n', '_').strip()
 
 def list_to_xml_id(names):
     return '_'.join([to_xmlid(name) for name in names])
@@ -77,10 +83,10 @@ def add_line(self, line, header):
     def generate_line(self):
         lines_header = ['id', 'product_tmpl_id/id', 'attribute_id/id', 'value_ids/id']
         lines_out = []
-        for template_id, attributes in self.data.iteritems():
+        for template_id, attributes in self.data.items():
             if not template_id:
                 continue
-            for attribute, values in attributes.iteritems():
+            for attribute, values in attributes.items():
                 line = [self.id_gen(template_id, attributes), template_id, attribute, ','.join(values)]
                 lines_out.append(line)
         return lines_header, lines_out