From 9cbc1b7aa1f2dade872465bd98dde378afbe54f7 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 16 Oct 2023 11:45:54 +0200 Subject: [PATCH 1/3] connector_importer: allow rename keys via conf You can now configure mapper options' key source_key_rename to specify a mapping of keys to be renamed to match odoo field names. --- connector_importer/components/dynamicmapper.py | 17 +++++++++++++++++ connector_importer/tests/test_mapper.py | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/connector_importer/components/dynamicmapper.py b/connector_importer/components/dynamicmapper.py index ea4be5d9..4d118b6f 100644 --- a/connector_importer/components/dynamicmapper.py +++ b/connector_importer/components/dynamicmapper.py @@ -42,6 +42,11 @@ def dynamic_fields(self, record): # Eg: prefix all supplier fields w/ `supplier.` fname = fname[len(prefix) :] clean_record[fname] = clean_record.pop(prefix + fname) + final_fname = self._get_field_name(fname, clean_record) + if final_fname != fname: + clean_record[final_fname] = clean_record.pop(fname) + fname = final_fname + if available_fields.get(fname): fspec = available_fields.get(fname) ftype = fspec["type"] @@ -113,6 +118,18 @@ def _source_key_empty_skip(self): def _source_key_prefix(self): return self.work.options.mapper.get("source_key_prefix", "") + @property + def _source_key_rename(self): + return self.work.options.mapper.get("source_key_rename", {}) + + def _get_field_name(self, fname, clean_record): + """Return final field name. + + Field names can be manipulated via mapper option `source_key_rename` + which must be a dictionary w/ source name -> destination name. + """ + return self._source_key_rename.get(fname, fname) + def _is_xmlid_key(self, fname, ftype): return fname.startswith("xid::") and ftype in ( "many2one", diff --git a/connector_importer/tests/test_mapper.py b/connector_importer/tests/test_mapper.py index 3b9b97ed..2bf5db2f 100644 --- a/connector_importer/tests/test_mapper.py +++ b/connector_importer/tests/test_mapper.py @@ -181,3 +181,16 @@ def test_rel_create_if_missing(self): self.assertEqual(cat.name, "New category") self.assertEqual(res["parent_id"], parent.id) self.assertEqual(res["category_id"], [(6, 0, [cat.id])]) + + def test_dynamic_mapper_rename_keys(self): + rec = { + "another_name": "John Doe", + } + # Whitelist + expected = { + "name": "John Doe", + } + mapper = self._get_dynamyc_mapper( + options=dict(source_key_rename={"another_name": "name"}) + ) + self.assertEqual(mapper.dynamic_fields(rec), expected) From 3c8dd6f50850b41b044c40c335721c024eceb293 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 6 Mar 2023 17:30:01 +0100 Subject: [PATCH 2/3] connector_importer: improve job details --- connector_importer/models/record.py | 30 +++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/connector_importer/models/record.py b/connector_importer/models/record.py index 7ee2d5c9..eac11471 100644 --- a/connector_importer/models/record.py +++ b/connector_importer/models/record.py @@ -72,6 +72,16 @@ def debug_mode(self): self.ensure_one() return self.backend_id.debug_mode or os.environ.get("IMPORTER_DEBUG_MODE") + def _should_use_jobs(self): + self.ensure_one() + debug_mode = self.debug_mode() + if debug_mode: + logger.warning("### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###") + use_job = self.recordset_id.import_type_id.use_job + if debug_mode: + use_job = False + return use_job + def import_record(self, importer_config): """This job will import a record. @@ -88,13 +98,7 @@ def import_record(self, importer_config): def run_import(self): """Queue a job for importing data stored in to self""" self.ensure_one() - debug_mode = self.debug_mode() - if debug_mode: - logger.warning("### DEBUG MODE ACTIVE: WILL NOT USE QUEUE ###") - use_job = self.recordset_id.import_type_id.use_job - if debug_mode: - use_job = False - result = self._run_import(use_job=use_job) + result = self._run_import(use_job=self._should_use_jobs()) return result def _run_import(self, use_job=True): @@ -103,7 +107,9 @@ def _run_import(self, use_job=True): # that needs to be imported new_self = self.with_context(queue_job__no_delay=not use_job) for config in self.recordset_id.available_importers(): - result = new_self.with_delay().import_record(config) + result = new_self.with_delay( + **self._run_import_job_params(config) + ).import_record(config) res[config.model] = result if self.debug_mode() or not use_job: # debug mode, no job here: reset it! @@ -114,3 +120,11 @@ def _run_import(self, use_job=True): # we keep the reference on w/ the last job. self.write({"job_id": result.db_record().id}) return res + + def _run_import_job_params(self, config): + params = { + "description": ( + f"recordset {self.recordset_id.name}: import {config['model']}" + ) + } + return params From 7a818d15495ccfa658e8c7fa3078ff5abf964592 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 16 Oct 2023 09:59:38 +0000 Subject: [PATCH 3/3] [BOT] post-merge updates --- README.md | 2 +- connector_importer/README.rst | 2 +- connector_importer/__manifest__.py | 2 +- connector_importer/static/description/index.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 985e9f02..fd0307ce 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Available addons ---------------- addon | version | maintainers | summary --- | --- | --- | --- -[connector_importer](connector_importer/) | 16.0.1.0.0 | [![simahawk](https://github.com/simahawk.png?size=30px)](https://github.com/simahawk) | This module takes care of import sessions. +[connector_importer](connector_importer/) | 16.0.1.1.0 | [![simahawk](https://github.com/simahawk.png?size=30px)](https://github.com/simahawk) | This module takes care of import sessions. Unported addons diff --git a/connector_importer/README.rst b/connector_importer/README.rst index 30e003eb..bf0e6680 100644 --- a/connector_importer/README.rst +++ b/connector_importer/README.rst @@ -7,7 +7,7 @@ Connector Importer !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:b2899192f47ba6f99e95eb4ca437e990cdc23bf30a924c6795dae3660b129fdc + !! source digest: sha256:7680b9cbb3329f4f4588d69d8487b9c03cae24d89c215adb8246c96b06f950fd !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/connector_importer/__manifest__.py b/connector_importer/__manifest__.py index db8ed651..1fa2e8cb 100644 --- a/connector_importer/__manifest__.py +++ b/connector_importer/__manifest__.py @@ -5,7 +5,7 @@ { "name": "Connector Importer", "summary": """This module takes care of import sessions.""", - "version": "16.0.1.0.0", + "version": "16.0.1.1.0", "depends": ["connector", "queue_job"], "author": "Camptocamp, Odoo Community Association (OCA)", "license": "AGPL-3", diff --git a/connector_importer/static/description/index.html b/connector_importer/static/description/index.html index 332496da..0e865e3b 100644 --- a/connector_importer/static/description/index.html +++ b/connector_importer/static/description/index.html @@ -367,7 +367,7 @@

Connector Importer

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:b2899192f47ba6f99e95eb4ca437e990cdc23bf30a924c6795dae3660b129fdc +!! source digest: sha256:7680b9cbb3329f4f4588d69d8487b9c03cae24d89c215adb8246c96b06f950fd !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: AGPL-3 OCA/connector-interfaces Translate me on Weblate Try me on Runboat

This module allows to import / update records from files using the connector