[ADD] Wrap import and export tools in method in order to use them directly with python script

[ADD] Listwriter to allow direct manupilation of list in import/export script instead of using csv file, however csv manipulation is working and remains unchange
[RENAME] Add a more specific name for the top level package of the library

Author: tfrancoi

README renamed to README.rst

@@ -0,0 +1,3 @@
+import lib
+import export_threaded
+import import_threaded

__init__.py

@@ -0,0 +1,109 @@
+#-*- coding: utf-8 -*-
+'''
+Copyright (C) Thibault Francois
+
+This program is free software: you can redistribute it and/or modify  
+it under the terms of the GNU Lesser General Public License as   
+published by the Free Software Foundation, version 3.
+
+This program is distributed in the hope that it will be useful, but 
+WITHOUT ANY WARRANTY; without even the implied warranty of 
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+Lesser General Lesser Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+'''
+
+from xmlrpclib import Fault
+from time import time
+from itertools import islice, chain
+
+
+import sys
+import csv
+
+from lib import conf_lib
+from lib.conf_lib import log_error, log_info
+from lib.internal.rpc_thread import RpcThread
+from lib.internal.csv_reader import UnicodeWriter
+from odoo_csv_tools.lib.internal.io import ListWriter
+
+csv.field_size_limit(sys.maxsize)
+
+
+def batch(iterable, size):
+    sourceiter = iter(iterable)
+    while True:
+        batchiter = islice(sourceiter, size)
+        yield chain([batchiter.next()], batchiter)
+
+class RPCThreadExport(RpcThread):
+
+    def __init__(self, max_connection, model, header, writer, batch_size=20, context=None):
+        super(RPCThreadExport, self).__init__(max_connection)
+        self.model = model
+        self.header = header
+        self.batch_size = batch_size
+        self.writer = writer
+        self.context = context
+        self.result = {}
+
+
+    def launch_batch(self, data_ids, batch_number):
+        def launch_batch_fun(data_ids, batch_number, check=False):
+            st = time()
+            try:
+                self.result[batch_number] = self.model.export_data(data_ids, self.header, context=self.context)['datas']
+            except Fault as e:
+                log_error("export %s failed" % batch_number)
+                log_error(e.faultString)
+            except Exception as e:
+                log_info("Unknown Problem")
+                exc_type, exc_value, _ = sys.exc_info()
+                #traceback.print_tb(exc_traceback, file=sys.stdout)
+                log_error(exc_type)
+                log_error(exc_value)
+            log_info("time for batch %s: %s" % (batch_number, time() - st))
+
+        self.spawn_thread(launch_batch_fun, [data_ids, batch_number], {})
+
+    def write_file(self, file_writer):
+        file_writer.writerow(self.header)
+        for key in self.result:
+            file_writer.writerows(self.result[key])
+
+
+
+def export_data(config_file, model, domain, header, context=None, output=None, max_connection=1, batch_size=100, separator=';', encoding='utf-8-sig'):
+
+    object_registry = conf_lib.get_server_connection(config_file).get_model(model)
+
+    if output:
+        file_result = open(output, "wb")
+        writer = UnicodeWriter(file_result, delimiter=separator, encoding=encoding, quoting=csv.QUOTE_ALL)
+    else:
+        writer = ListWriter()
+
+    rpc_thread = RPCThreadExport(int(max_connection), object_registry, header, writer, batch_size, context)
+    st = time()
+
+    ids = object_registry.search(domain, context=context)
+    i = 0
+    for b in batch(ids,batch_size):
+        batch_ids = [l for l in b]
+        rpc_thread.launch_batch(batch_ids, i)
+        i += 1
+
+    rpc_thread.wait()
+    log_info("%s %s exported, total time %s second(s)" % (len(ids), model, (time() - st)))
+    log_info("Writing file")
+    rpc_thread.write_file(writer)
+    if output:
+        file_result.close()
+        return False, False
+    else:
+        return writer.header, writer.data
+
+
+

lib/__init__.py

@@ -0,0 +1,223 @@
+#-*- coding: utf-8 -*-
+'''
+Copyright (C) Thibault Francois
+
+This program is free software: you can redistribute it and/or modify  
+it under the terms of the GNU Lesser General Public License as   
+published by the Free Software Foundation, version 3.
+
+This program is distributed in the hope that it will be useful, but 
+WITHOUT ANY WARRANTY; without even the implied warranty of 
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
+Lesser General Lesser Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+'''
+
+
+import sys
+import csv
+
+from time import time
+from itertools import islice, chain
+from xmlrpclib import Fault
+
+from lib import conf_lib
+from lib.conf_lib import log_error, log_info, log
+from lib.internal.rpc_thread import RpcThread
+from lib.internal.io import ListWriter
+from lib.internal.csv_reader import UnicodeReader, UnicodeWriter
+
+csv.field_size_limit(sys.maxsize)
+
+
+def batch(iterable, size):
+    sourceiter = iter(iterable)
+    while True:
+        batchiter = islice(sourceiter, size)
+        yield chain([batchiter.next()], batchiter)
+
+class RPCThreadImport(RpcThread):
+
+    def __init__(self, max_connection, model, header, writer, batch_size=20, context=None):
+        super(RPCThreadImport, self).__init__(max_connection)
+        self.model = model
+        self.header = header
+        self.batch_size = batch_size
+        self.writer = writer
+        self.context = context
+
+
+    def launch_batch(self, data_lines, batch_number, check=False):
+        def launch_batch_fun(lines, batch_number, check=False):
+            i = 0
+            for lines_batch in batch(lines, self.batch_size):
+                lines_batch = [l for l in lines_batch]
+                self.sub_batch_run(lines_batch, batch_number, i, len(lines), check=check)
+                i += 1
+
+        self.spawn_thread(launch_batch_fun, [data_lines, batch_number], {'check' : check})
+
+    def sub_batch_run(self, lines, batch_number, sub_batch_number, total_line_nb, check=False):
+        success = False
+
+        st = time()
+        try:
+            success = self._send_rpc(lines, batch_number, sub_batch_number, check=check)
+        except Fault as e:
+            log_error("Line %s %s failed" % (batch_number, sub_batch_number))
+            log_error(e.faultString)
+        except ValueError as e:
+            log_error("Line %s %s failed value error" % (batch_number, sub_batch_number))
+        except Exception as e:
+            log_info("Unknown Problem")
+            exc_type, exc_value, _ = sys.exc_info()
+            #traceback.print_tb(exc_traceback, file=sys.stdout)
+            log_error(exc_type)
+            log_error(exc_value)
+
+        if not success:
+            self.writer.writerows(lines)
+
+        log_info("time for batch %s - %s of %s : %s" % (batch_number, (sub_batch_number + 1) * self.batch_size, total_line_nb, time() - st))
+
+
+    def _send_rpc(self, lines, batch_number, sub_batch_number, check=False):
+        res = self.model.load(self.header, lines, context=self.context)
+        if res['messages']:
+            for msg in res['messages']:
+                log_error('batch %s, %s' % (batch_number, sub_batch_number))
+                log_error(msg)
+                log_error(lines[msg['record']])
+            return False
+        if len(res['ids']) != len(lines) and check:
+            log_error("number of record import is different from the record to import, probably duplicate xml_id")
+            return False
+
+        return True
+
+def do_not_split(split, previous_split_value, split_index, line):
+    if not split: # If no split no need to continue
+        return False
+
+    split_value = line[split_index]
+    if split_value != previous_split_value: #Different Value no need to not split
+        return False
+
+    return True
+
+def filter_line_ignore(ignore, header, line):
+    new_line = []
+    for k, val in zip(header, line):
+        if k not in ignore:
+            new_line.append(val)
+    return new_line
+
+def filter_header_ignore(ignore, header):
+    new_header = []
+    for val in header:
+        if val not in ignore:
+            new_header.append(val)
+    return new_header
+
+def read_file(file_to_read, delimiter=';', encoding='utf-8-sig', skip=0):
+    def get_real_header(header):
+        """ Get real header cut at the first empty column """
+        new_header = []
+        for head in header:
+            if head:
+                new_header.append(head)
+            else:
+                break
+        return new_header
+
+    def check_id_column(header):
+        try:
+            header.index('id')
+        except ValueError as ve:
+            log_error("No External Id (id) column defined, please add one")
+            raise ve
+
+    def skip_line(reader):
+        log_info("Skipping until line %s excluded" % skip)
+        for _ in xrange(1, skip):
+            reader.next()
+
+    log('open %s' % file_to_read)
+    file_ref = open(file_to_read, 'r')
+    reader = UnicodeReader(file_ref, delimiter=delimiter, encoding='utf-8-sig')
+    header = reader.next()
+    header = get_real_header(header)
+    check_id_column(header)
+    skip_line(reader)
+    data = [l for l in reader]
+    return header, data
+
+def split_sort(split, header, data):
+    split_index = 0
+    if split:
+        try:
+            split_index = header.index(split)
+        except ValueError as ve:
+            log("column %s not defined" % split)
+            raise ve
+        data = sorted(data, key=lambda d: d[split_index])
+    return data, split_index
+
+def import_data(config_file, model, header=None, data=None, file_csv=None, context=None, fail_file=False, encoding='utf-8-sig', separator=";", ignore=False, split=False, check=True, max_connection=1, batch_size=10, skip=0):
+    """
+        header and data mandatory in file_csv is not provided
+
+    """
+    ignore = ignore or []
+    context = context or {}
+
+    if file_csv:
+        header, data = read_file(file_csv, delimiter=separator, encoding=encoding, skip=skip)
+        fail_file = fail_file or file_csv + ".fail"
+        file_result = open(fail_file, "wb")
+
+    if not header or data == None:
+        raise ValueError("Please provide either a data file or a header and data")
+
+    object_registry = conf_lib.get_server_connection(config_file).get_model(model)
+
+    if file_csv:
+        writer = UnicodeWriter(file_result, delimiter=separator, encoding=encoding, quoting=csv.QUOTE_ALL)
+    else:
+        writer = ListWriter()
+
+    writer.writerow(filter_header_ignore(ignore, header))
+    if file_csv:
+        file_result.flush()
+    rpc_thread = RPCThreadImport(int(max_connection), object_registry, filter_header_ignore(ignore, header), writer, batch_size, context)
+    st = time()
+
+
+    data, split_index = split_sort(split, header, data)
+
+    i = 0
+    previous_split_value = False
+    while  i < len(data):
+        lines = []
+        j = 0
+        while i < len(data) and (j < batch_size or do_not_split(split, previous_split_value, split_index, data[i])):
+            line = data[i][:len(header)]
+            lines.append(filter_line_ignore(ignore, header, line))
+            previous_split_value = line[split_index]
+            j += 1
+            i += 1
+        batch_number = split and "[%s] - [%s]" % (rpc_thread.thread_number(), previous_split_value) or "[%s]" % rpc_thread.thread_number()
+        rpc_thread.launch_batch(lines, batch_number, check)
+
+    rpc_thread.wait()
+    if file_csv:
+        file_result.close()
+
+    log_info("%s %s imported, total time %s second(s)" % (len(data), model, (time() - st)))
+    if file_csv:
+        return False, False
+    else:
+        return writer.header, writer.data
+

lib/internal/__init__.py

@@ -0,0 +1,6 @@
+import internal
+import conf_lib
+import workflow
+import checker
+import mapper
+import transform

odoo_csv_tools/__init__.py

@@ -0,0 +1,5 @@
+import exceptions
+import tools
+import csv_reader
+import io
+import rpc_thread