[FIX] Encoding is now consistent utf-8 by default, is a parameter each time is needed and get propagate properly

Author: tfr-odoo

odoo_csv_tools/export_threaded.py

@@ -70,11 +70,11 @@ def write_file(self, file_writer):
 
 
 def export_data(config_file, model, domain, header, context=None, output=None, max_connection=1, batch_size=100,
-                separator=';', encoding='utf-8-sig'):
+                separator=';', encoding='utf-8'):
     object_registry = conf_lib.get_server_connection(config_file).get_model(model)
 
     if output:
-        file_result = open_write(output)
+        file_result = open_write(output, encoding=encoding)
         writer = UnicodeWriter(file_result, delimiter=separator, encoding=encoding, quoting=csv.QUOTE_ALL)
     else:
         writer = ListWriter()

odoo_csv_tools/import_threaded.py

@@ -112,7 +112,7 @@ def filter_header_ignore(ignore, header):
     return new_header
 
 
-def read_file(file_to_read, delimiter=';', encoding='utf-8-sig', skip=0):
+def read_file(file_to_read, delimiter=';', encoding='utf-8', skip=0):
     def get_real_header(header):
         """ Get real header cut at the first empty column """
         new_header = []
@@ -136,8 +136,8 @@ def skip_line(reader):
             reader.next()
 
     log('open %s' % file_to_read)
-    file_ref = open_read(file_to_read, encoding='utf-8-sig')
-    reader = UnicodeReader(file_ref, delimiter=delimiter, encoding='utf-8-sig')
+    file_ref = open_read(file_to_read, encoding=encoding)
+    reader = UnicodeReader(file_ref, delimiter=delimiter, encoding=encoding)
     header = next(reader)
     header = get_real_header(header)
     check_id_column(header)
@@ -180,7 +180,7 @@ def do_not_split(split, previous_split_value, split_index, line, o2m=False, id_i
 
 
 def import_data(config_file, model, header=None, data=None, file_csv=None, context=None, fail_file=False,
-                encoding='utf-8-sig', separator=";", ignore=False, split=False, check=True, max_connection=1,
+                encoding='utf-8', separator=";", ignore=False, split=False, check=True, max_connection=1,
                 batch_size=10, skip=0, o2m=False):
     """
         header and data mandatory in file_csv is not provided

odoo_csv_tools/lib/internal/io.py

@@ -19,32 +19,32 @@ def is_string(f):
     else:
         return isinstance(f, basestring)
 
-def open_read(f, encoding='utf-8-sig'):
+def open_read(f, encoding='utf-8'):
     if not is_string(f):
         return f
     if sys.version_info >= (3, 0, 0):
         return open(f, 'r', newline='', encoding=encoding)
     else:
         return open(f, 'r')
 
-def open_write(f, encoding='utf-8-sig'):
+def open_write(f, encoding='utf-8'):
     if not is_string(f):
         return f
     if sys.version_info >= (3, 0, 0):
         return open(f, "w", newline='', encoding=encoding)
     else:
         return open(f, "w")
 
-def write_csv(filename, header, data):
-    file_result = open_write(filename)
-    c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL)
+def write_csv(filename, header, data, encoding="utf-8"):
+    file_result = open_write(filename, encoding=encoding)
+    c = UnicodeWriter(file_result, delimiter=';', quoting=csv.QUOTE_ALL, encoding=encoding)
     c.writerow(header)
     for d in data:
         c.writerow(d)
     file_result.close()
 
 def write_file(filename=None, header=None, data=None, fail=False, model="auto",
-               launchfile="import_auto.sh", worker=1, batch_size=10, init=False,
+               launchfile="import_auto.sh", worker=1, batch_size=10, init=False, encoding="utf-8",
                conf_file=False, groupby='', sep=";", python_exe='', path='', context=None, ignore=""):
     def get_model():
         if model == "auto":
@@ -54,17 +54,17 @@ def get_model():
 
     context = '--context="%s"' % str(context) if context else ''
     conf_file = conf_file or "%s%s%s" % ('conf', os.sep, 'connection.conf')
-    write_csv(filename, header, data)
+    write_csv(filename, header, data, encoding=encoding)
     if not launchfile:
         return
 
     mode = init and 'w' or 'a'
     with open(launchfile, mode) as myfile:
-        myfile.write("%s %sodoo_import_thread.py -c %s --file=%s --model=%s --worker=%s --size=%s --groupby=%s --ignore=%s --sep=\"%s\" %s\n" %
-                    (python_exe, path, conf_file, filename, get_model(), worker, batch_size, groupby, ignore, sep, context))
+        myfile.write("%s %sodoo_import_thread.py -c %s --file=%s --model=%s --encoding=%s --worker=%s --size=%s --groupby=%s --ignore=%s --sep=\"%s\" %s\n" %
+                    (python_exe, path, conf_file, filename, get_model(), encoding, worker, batch_size, groupby, ignore, sep, context))
         if fail:
-            myfile.write("%s %sodoo_import_thread.py -c %s --fail --file=%s --model=%s --ignore=%s --sep=\"%s\" %s\n" %
-                         (python_exe, path, conf_file, filename, get_model(), ignore, sep, context))
+            myfile.write("%s %sodoo_import_thread.py -c %s --fail --file=%s --model=%s --encoding=%s --ignore=%s --sep=\"%s\" %s\n" %
+                         (python_exe, path, conf_file, filename, get_model(), encoding, ignore, sep, context))
 
 class ListWriter(object):
     def __init__(self):

odoo_csv_tools/lib/mapper.py

@@ -138,7 +138,7 @@ def binary_val(line):
             return ''
 
         with open(path, "rb") as image_file:
-                encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
+                encoded_string = base64.b64encode(image_file.read()).decode(encoding)
                 image_file.close()
         return encoded_string
     return binary_val
@@ -148,7 +148,7 @@ def binary(field, path_prefix, skip=False, encoding="utf-8"):
 
 
 
-def binary_url_map(mapper, skip=False, verbose=False):
+def binary_url_map(mapper, skip=False, verbose=False, encoding="utf-8"):
     def binary_url_fun(line):
         url = mapper(line)
         if verbose:
@@ -159,7 +159,7 @@ def binary_url_fun(line):
                 raise SkippingException("Cannot fetch file at url %s" % url)
             return ''
 
-        return base64.b64encode(res.content).decode("utf-8")
+        return base64.b64encode(res.content).decode(encoding)
     return binary_url_fun
 
 def binary_url(field, skip=False, verbose=False):

odoo_csv_tools/lib/transform.py

@@ -16,7 +16,7 @@
 
 
 class Processor(object):
-    def __init__(self, filename=None, delimiter=";", encoding='utf-8-sig', header=None, data=None, preprocess=lambda header, data: (header, data), conf_file=False):
+    def __init__(self, filename=None, delimiter=";", encoding='utf-8', header=None, data=None, preprocess=lambda header, data: (header, data), conf_file=False):
         self.file_to_write = OrderedDict()
         if header and data:
             self.header = header
@@ -74,7 +74,7 @@ def process(self, mapping, filename_out, import_args, t='list', null_values=['NU
         self._add_data(head, data, filename_out, import_args)
         return head, data
 
-    def write_to_file(self, script_filename, fail=True, append=False, python_exe='', path=''):
+    def write_to_file(self, script_filename, fail=True, append=False, python_exe='', path='', encoding='utf-8'):
         init = not append
         for _, info in self.file_to_write.items():
             info_copy = dict(info)
@@ -86,6 +86,7 @@ def write_to_file(self, script_filename, fail=True, append=False, python_exe='',
                 'python_exe' : python_exe,
                 'path' : path,
                 'conf_file' : self.conf_file,
+                'encoding': encoding,
             })
 
             write_file(**info_copy)
@@ -94,7 +95,7 @@ def write_to_file(self, script_filename, fail=True, append=False, python_exe='',
     def get_processed_data(self, filename_out):
         return self.file_to_write[filename_out]
 
-    def join_file(self, filename, master_key, child_key, header_prefix="child", delimiter=";", encoding='utf-8-sig'):
+    def join_file(self, filename, master_key, child_key, header_prefix="child", delimiter=";", encoding='utf-8'):
         """
             Join another file with the main file defined in the constructor.
             Need a key (column name) on the master file  and on the file to join

odoo_export_thread.py

@@ -33,6 +33,7 @@
     parser.add_argument('--context', dest='context',
                         help='context that will be passed to the load function, need to be a valid python dict',
                         default="{'tracking_disable' : True}")
+    parser.add_argument('--encoding', dest='encoding', default="utf-8", help='Encoding of the data file')
     # TODO args : encoding
     # {'update_many2many': True,'tracking_disable' : True, 'create_product_variant' : True, 'check_move_validity' : False}
     args = parser.parse_args()
@@ -43,7 +44,7 @@
     model = args.model
     max_connection = int(args.worker)
     separator = args.separator
-    encoding = 'utf-8-sig'
+    encoding = args.encoding
     context = eval(args.context)
     domain = eval(args.domain)
     header = args.fields.split(',')

odoo_import_thread.py

@@ -34,6 +34,7 @@
     parser.add_argument('--check', dest='check', action='store_true', help='Check if record are imported after each batch.')
     parser.add_argument('--context', dest='context', help='context that will be passed to the load function, need to be a valid python dict', default="{'tracking_disable' : True}")
     parser.add_argument('--o2m', action='store_true', dest="o2m", help="When you want to import o2m field, don't cut the batch until we find a new id")
+    parser.add_argument('--encoding', dest='encoding', default="utf-8", help='Encoding of the data file')
     #TODO args : encoding
     #{'update_many2many': True,'tracking_disable' : True, 'create_product_variant' : True, 'check_move_validity' : False}
     args = parser.parse_args()
@@ -43,7 +44,7 @@
     fail_file = file_csv + ".fail"
     max_connection = int(args.worker)
     split = False
-    encoding='utf-8-sig'
+    encoding= args.encoding
     context= eval(args.context)
     ignore = False
     if args.ignore:

tests/5_partner_export.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-$1 ../odoo_export_thread.py -c conf/connection.conf --file=data/res.partner.exported.csv --model=res.partner --worker=4 --size=200 --domain="[]" --field="id,name,phone,website,street,city,country_id/id" --sep=";"
+$1 ../odoo_export_thread.py -c conf/connection.conf --file=data/res.partner.exported.csv --model=res.partner --worker=4 --size=200 --domain="[]" --field="id,name,phone,website,street,city,country_id/id" --sep=";" --encoding=utf-8-sig

tests/test_import.py

@@ -61,4 +61,4 @@
     'batch_size': 100,
     'model': 'res.partner',
 })
-processor.write_to_file(script, python_exe=EXEC, path='../')
+processor.write_to_file(script, python_exe=EXEC, path='../', encoding="utf-8-sig")