diff --git a/chunk_processing/__init__.py b/chunk_processing/__init__.py new file mode 100644 index 00000000..0f00a673 --- /dev/null +++ b/chunk_processing/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import components diff --git a/chunk_processing/__manifest__.py b/chunk_processing/__manifest__.py new file mode 100644 index 00000000..f85d2855 --- /dev/null +++ b/chunk_processing/__manifest__.py @@ -0,0 +1,31 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +{ + "name": "Chunk Processing", + "summary": "Base module for processing chunk", + "version": "14.0.1.0.0", + "category": "Uncategorized", + "website": "https://github.com/shopinvader/pattern-import-export", + "author": " Akretion", + "license": "AGPL-3", + "application": False, + "installable": True, + "external_dependencies": { + "python": [], + "bin": [], + }, + "depends": [ + "queue_job", + "component", + "web_refresher", + ], + "data": [ + "views/chunk_item_view.xml", + "views/chunk_group_view.xml", + "views/templates.xml", + ], + "demo": [], +} diff --git a/chunk_processing/components/__init__.py b/chunk_processing/components/__init__.py new file mode 100644 index 00000000..c6e3896d --- /dev/null +++ b/chunk_processing/components/__init__.py @@ -0,0 +1,8 @@ +from . import processor +from . import processor_xml +from . import processor_json +from . import processor_txt +from . import splitter +from . import splitter_json +from . import splitter_xml +from . import splitter_txt diff --git a/chunk_processing/components/processor.py b/chunk_processing/components/processor.py new file mode 100644 index 00000000..6d27b829 --- /dev/null +++ b/chunk_processing/components/processor.py @@ -0,0 +1,33 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessor(AbstractComponent): + _name = "chunk.processor" + _collection = "chunk.item" + + def _import_item(self): + raise NotImplementedError + + def _prepare_error_message(self, idx, item, error): + return { + "rows": {"from": idx, "to": idx}, + "type": type(error).__name__, + "message": str(error), + } + + def run(self): + res = {"ids": [], "messages": []} + for idx, item in enumerate(self._parse_data()): + try: + with self.env.cr.savepoint(): + res["ids"] += self._import_item(item) + except Exception as e: + if self.env.context.get("chunk_raise_if_exception"): + raise + res["messages"].append(self._prepare_error_message(idx, item, e)) + return res diff --git a/chunk_processing/components/processor_json.py b/chunk_processing/components/processor_json.py new file mode 100644 index 00000000..6fb8f1d5 --- /dev/null +++ b/chunk_processing/components/processor_json.py @@ -0,0 +1,17 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 +import json + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorJson(AbstractComponent): + _name = "chunk.importer.json" + _inherit = "chunk.processor" + _collection = "chunk.item" + + def _parse_data(self): + return json.loads(base64.b64decode(self.collection.data)) diff --git a/chunk_processing/components/processor_txt.py b/chunk_processing/components/processor_txt.py new file mode 100644 index 00000000..a996e76c --- /dev/null +++ b/chunk_processing/components/processor_txt.py @@ -0,0 +1,17 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorTxt(AbstractComponent): + _name = "chunk.importer.txt" + _inherit = "chunk.processor" + _collection = "chunk.item" + _end_of_line = b"\n" + + def _parse_data(self): + return base64.b64decode(self.collection.data).split(self._end_of_line) diff --git a/chunk_processing/components/processor_xml.py b/chunk_processing/components/processor_xml.py new file mode 100644 index 00000000..95a25dd5 --- /dev/null +++ b/chunk_processing/components/processor_xml.py @@ -0,0 +1,20 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from lxml import objectify + +from odoo.addons.component.core import AbstractComponent + + +class ChunkProcessorXml(AbstractComponent): + _name = "chunk.importer.xml" + _inherit = "chunk.processor" + _collection = "chunk.item" + + def _parse_data(self): + return objectify.fromstring( + base64.b64decode(self.collection.data) + ).iterchildren() diff --git a/chunk_processing/components/splitter.py b/chunk_processing/components/splitter.py new file mode 100644 index 00000000..c939d6a8 --- /dev/null +++ b/chunk_processing/components/splitter.py @@ -0,0 +1,55 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from odoo.addons.component.core import AbstractComponent + + +class ChunkSplitter(AbstractComponent): + _name = "chunk.splitter" + _collection = "chunk.group" + + def _parse_data(self, data): + raise NotImplementedError + + def _convert_items_to_data(self, items): + raise NotImplementedError + + def _prepare_chunk(self, start_idx, stop_idx, items): + return { + "start_idx": start_idx, + "stop_idx": stop_idx, + "data": base64.b64encode(self._convert_items_to_data(items)), + "nbr_item": len(items), + "state": "pending", + "group_id": self.collection.id, + } + + def _should_create_chunk(self, items, next_item): + """Customise this code if you want to add some additionnal + item after reaching the limit""" + return len(items) > self.collection.chunk_size + + def _create_chunk(self, start_idx, stop_idx, data): + vals = self._prepare_chunk(start_idx, stop_idx, data) + chunk = self.env["chunk.item"].create(vals) + # we enqueue the chunk in case of multi process of if it's the first chunk + if self.collection.process_multi or len(self.collection.item_ids) == 1: + chunk.with_delay(priority=self.collection.job_priority).run() + return chunk + + def run(self, data): + items = [] + start_idx = 1 + previous_idx = None + for idx, item in self._parse_data(data): + if self._should_create_chunk(items, item): + self._create_chunk(start_idx, previous_idx, items) + items = [] + start_idx = idx + items.append((idx, item)) + previous_idx = idx + if items: + self._create_chunk(start_idx, idx, items) diff --git a/chunk_processing/components/splitter_json.py b/chunk_processing/components/splitter_json.py new file mode 100644 index 00000000..40f16d31 --- /dev/null +++ b/chunk_processing/components/splitter_json.py @@ -0,0 +1,21 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import json + +from odoo.addons.component.core import Component + + +class ChunkSplitterJson(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.json" + _usage = "json" + + def _convert_items_to_data(self, items): + return json.dumps(items, indent=2).encode("utf-8") + + def _parse_data(self, data): + items = json.loads(data.decode("utf-8")) + for idx, item in enumerate(items): + yield idx + 1, item diff --git a/chunk_processing/components/splitter_txt.py b/chunk_processing/components/splitter_txt.py new file mode 100644 index 00000000..27ac179d --- /dev/null +++ b/chunk_processing/components/splitter_txt.py @@ -0,0 +1,20 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class ChunkSplitterTxt(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.txt" + _usage = "txt" + _end_of_line = b"\n" + + def _parse_data(self, data): + for idx, item in enumerate(data.split(self._end_of_line)): + if item: + yield idx + 1, item + + def _convert_items_to_data(self, items): + return self._end_of_line.join([x[1] for x in items]) diff --git a/chunk_processing/components/splitter_xml.py b/chunk_processing/components/splitter_xml.py new file mode 100644 index 00000000..88df4890 --- /dev/null +++ b/chunk_processing/components/splitter_xml.py @@ -0,0 +1,25 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from lxml import etree + +from odoo.addons.component.core import Component + + +class ChunkSplitterXml(Component): + _inherit = "chunk.splitter" + _name = "chunk.splitter.xml" + _usage = "xml" + + def _parse_data(self, data): + tree = etree.fromstring(data) + items = tree.xpath(self.collection.xml_split_xpath) + for idx, item in enumerate(items): + yield idx + 1, item + + def _convert_items_to_data(self, items): + data = etree.Element("data") + for item in items: + data.append(item[1]) + return etree.tostring(data) diff --git a/chunk_processing/models/__init__.py b/chunk_processing/models/__init__.py new file mode 100644 index 00000000..23028c82 --- /dev/null +++ b/chunk_processing/models/__init__.py @@ -0,0 +1,2 @@ +from . import chunk_item +from . import chunk_group diff --git a/chunk_processing/models/chunk_group.py b/chunk_processing/models/chunk_group.py new file mode 100644 index 00000000..db182261 --- /dev/null +++ b/chunk_processing/models/chunk_group.py @@ -0,0 +1,81 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + + +from odoo import _, api, fields, models + + +class ChunkGroup(models.Model): + _inherit = "collection.base" + _name = "chunk.group" + + item_ids = fields.One2many("chunk.item", "group_id", "Item") + process_multi = fields.Boolean() + job_priority = fields.Integer(default=20) + chunk_size = fields.Integer(default=500, help="Define the size of the chunk") + progress = fields.Float(compute="_compute_stat") + date_done = fields.Datetime() + data_format = fields.Selection( + [ + ("json", "Json"), + ("xml", "XML"), + ("txt", "Txt"), + ] + ) + xml_split_xpath = fields.Char() + state = fields.Selection( + [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], + default="pending", + ) + info = fields.Char() + nbr_error = fields.Integer(compute="_compute_stat") + nbr_success = fields.Integer(compute="_compute_stat") + apply_on_model = fields.Char() + usage = fields.Char() + + @api.depends("item_ids.nbr_error", "item_ids.nbr_success") + def _compute_stat(self): + for record in self: + record.nbr_error = sum(record.mapped("item_ids.nbr_error")) + record.nbr_success = sum(record.mapped("item_ids.nbr_success")) + todo = sum(record.mapped("item_ids.nbr_item")) + if todo: + record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo + else: + record.progress = 0 + + def _get_data(self): + raise NotImplementedError + + def split_in_chunk(self): + """Split Group into Chunk""" + # purge chunk in case of retring a job + self.item_ids.unlink() + try: + data = self._get_data() + with self.work_on(self._name) as work: + splitter = work.component(usage=self.data_format) + splitter.run(data) + except Exception as e: + if self._context.get("chunk_raise_if_exception"): + raise + else: + self.state = "failed" + self.info = _("Failed to create the chunk: %s") % e + return True + + def set_done(self): + for record in self: + if record.nbr_error: + record.state = "failed" + else: + record.state = "done" + record.date_done = fields.Datetime.now() + + @api.model_create_multi + def create(self, vals_list): + records = super().create(vals_list) + for record in records: + record.with_delay(priority=self.job_priority).split_in_chunk() + return records diff --git a/pattern_import_export/models/pattern_chunk.py b/chunk_processing/models/chunk_item.py similarity index 55% rename from pattern_import_export/models/pattern_chunk.py rename to chunk_processing/models/chunk_item.py index e4cf6ed4..1fcd4e94 100644 --- a/pattern_import_export/models/pattern_chunk.py +++ b/chunk_processing/models/chunk_item.py @@ -5,18 +5,19 @@ from odoo import fields, models -class PatternChunk(models.Model): - _name = "pattern.chunk" - _description = "Pattern Chunk" +class ChunkItem(models.Model): + _inherit = "collection.base" + _name = "chunk.item" + _description = "Chunk Item" _order = "start_idx" _rec_name = "start_idx" - pattern_file_id = fields.Many2one( - "pattern.file", "Pattern File", required=True, ondelete="cascade" + group_id = fields.Many2one( + "chunk.group", "Chunk Group", required=True, ondelete="cascade" ) start_idx = fields.Integer() stop_idx = fields.Integer() - data = fields.Serialized() + data = fields.Binary() record_ids = fields.Serialized() messages = fields.Serialized() result_info = fields.Html() @@ -31,43 +32,55 @@ class PatternChunk(models.Model): ("failed", "Failed"), ] ) + filename = fields.Char(compute="_compute_filename") - def run_import(self): - model = self.pattern_file_id.pattern_config_id.model_id.model - res = ( - self.with_context(pattern_config={"model": model, "record_ids": []}) - .env[model] - .load([], self.data) - ) - self.write(self._prepare_chunk_result(res)) - config = self.pattern_file_id.pattern_config_id - priority = config.job_priority - if not config.process_multi: + def _compute_filename(self): + for record in self: + record.filename = ( + f"{record.start_idx}-{record.stop_idx}.{record.group_id.data_format}" + ) + + def manual_run(self): + """ Run the import without try/except, easier for debug """ + return self._run() + + def _run(self): + with self.work_on(self.group_id.apply_on_model) as work: + processor = work.component(usage=self.group_id.usage) + res = processor.run() + vals = self._prepare_chunk_result(res) + self.write(vals) + + if not self.group_id.process_multi: next_chunk = self.get_next_chunk() if next_chunk: - next_chunk.with_delay(priority=priority).run() + next_chunk.with_delay(priority=self.group_id.job_priority).run() else: self.with_delay(priority=5).check_last() else: self.with_delay(priority=5).check_last() + return True def run(self): - """Process Import of Pattern Chunk""" + """Process Chunk Item in a savepoint""" cr = self.env.cr try: self.state = "started" cr.commit() # pylint: disable=invalid-commit with cr.savepoint(): - self.run_import() + self._run() except Exception as e: - self.write( - { - "result_info": "Fail to process chunk %s" % e, - "nbr_error": self.nbr_item, - "state": "failed", - } - ) - self.with_delay().check_last() + if self._context.get("chunk_raise_if_exception"): + raise + else: + self.write( + { + "result_info": "Fail to process chunk %s" % e, + "nbr_error": self.nbr_item, + "state": "failed", + } + ) + self.with_delay().check_last() return "OK" def _prepare_chunk_result(self, res): @@ -75,6 +88,7 @@ def _prepare_chunk_result(self, res): nbr_error = len(res["messages"]) nbr_success = max(self.nbr_item - nbr_error, 0) + # TODO move this in pattern-import # case where error are not return and record are not imported nbr_imported = len(res.get("ids") or []) if nbr_success > nbr_imported: @@ -85,9 +99,7 @@ def _prepare_chunk_result(self, res): state = "failed" else: state = "done" - result = self.env["ir.qweb"]._render( - "pattern_import_export.format_message", res - ) + result = self.env["ir.qweb"]._render("chunk_processing.format_message", res) return { "record_ids": res.get("ids"), "messages": res.get("messages"), @@ -98,23 +110,19 @@ def _prepare_chunk_result(self, res): } def get_next_chunk(self): - return self.search( - [ - ("pattern_file_id", "=", self.pattern_file_id.id), - ("state", "=", "pending"), - ], - limit=1, + return fields.first( + self.group_id.item_ids.filtered(lambda s: s.state == "pending") ) def is_last_job(self): - return not self.pattern_file_id.chunk_ids.filtered( + return not self.group_id.item_ids.filtered( lambda s: s.state in ("pending", "started") ) def check_last(self): """Check if all chunk have been processed""" if self.is_last_job(): - self.pattern_file_id.set_import_done() - return "Pattern file is done" + self.group_id.set_done() + return "Chunk group is done" else: return "There is still some running chunk" diff --git a/chunk_processing/security/ir.model.access.csv b/chunk_processing/security/ir.model.access.csv new file mode 100644 index 00000000..a32970c3 --- /dev/null +++ b/chunk_processing/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_chunk_group,Chunk group System,model_chunk_group,base.group_system,1,1,1,1 +access_chunk_item,Chunk item System,model_chunk_item,base.group_system,1,1,1,1 diff --git a/chunk_processing/views/chunk_group_view.xml b/chunk_processing/views/chunk_group_view.xml new file mode 100644 index 00000000..cc9fd600 --- /dev/null +++ b/chunk_processing/views/chunk_group_view.xml @@ -0,0 +1,64 @@ + + + + + chunk.group + + + + + + + + + + chunk.group + +
+
+
+ + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + +
+ + + diff --git a/pattern_import_export/views/pattern_chunk.xml b/chunk_processing/views/chunk_item_view.xml similarity index 70% rename from pattern_import_export/views/pattern_chunk.xml rename to chunk_processing/views/chunk_item_view.xml index d9dbbd30..3b69ab4a 100644 --- a/pattern_import_export/views/pattern_chunk.xml +++ b/chunk_processing/views/chunk_item_view.xml @@ -1,8 +1,8 @@ - - pattern.chunk + + chunk.item @@ -14,15 +14,17 @@ - - pattern.chunk + + chunk.item
-
+ + diff --git a/pattern_import_export/views/templates.xml b/chunk_processing/views/templates.xml similarity index 100% rename from pattern_import_export/views/templates.xml rename to chunk_processing/views/templates.xml diff --git a/pattern_import_export/__init__.py b/pattern_import_export/__init__.py index 9b429614..d0c2ad41 100644 --- a/pattern_import_export/__init__.py +++ b/pattern_import_export/__init__.py @@ -1,2 +1,3 @@ from . import models from . import wizard +from . import components diff --git a/pattern_import_export/__manifest__.py b/pattern_import_export/__manifest__.py index 93a54145..2b943ec6 100644 --- a/pattern_import_export/__manifest__.py +++ b/pattern_import_export/__manifest__.py @@ -15,6 +15,7 @@ "web_notify", "base_sparse_field_list_support", "base_sparse_field", + "chunk_processing", ], "data": [ "security/pattern_security.xml", @@ -23,11 +24,9 @@ "wizard/import_pattern_wizard.xml", "views/pattern_config.xml", "views/pattern_file.xml", - "views/pattern_chunk.xml", "views/menuitems.xml", - "views/templates.xml", "data/queue_job_channel_data.xml", - "data/queue_job_function_data.xml", + # "data/queue_job_function_data.xml", ], "demo": ["demo/demo.xml"], "installable": True, diff --git a/pattern_import_export/components/__init__.py b/pattern_import_export/components/__init__.py new file mode 100644 index 00000000..ad6975d7 --- /dev/null +++ b/pattern_import_export/components/__init__.py @@ -0,0 +1 @@ +from . import processor diff --git a/pattern_import_export/components/processor.py b/pattern_import_export/components/processor.py new file mode 100644 index 00000000..5e9111d9 --- /dev/null +++ b/pattern_import_export/components/processor.py @@ -0,0 +1,47 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class ChunkProcessorPattern(Component): + _inherit = "chunk.processor" + _name = "chunk.processor.pattern" + _usage = "pattern.import" + + def run(self): + model = self.collection.group_id.apply_on_model + res = ( + self.env[model] + .with_context(pattern_config={"model": model, "record_ids": []}) + .load([], self.collection.data) + ) + self.collection.write(self._prepare_chunk_result(res)) + + def _prepare_chunk_result(self, res): + # TODO rework this part and add specific test case + nbr_error = len(res["messages"]) + nbr_success = max(self.collection.nbr_item - nbr_error, 0) + + # case where error are not return and record are not imported + nbr_imported = len(res.get("ids") or []) + if nbr_success > nbr_imported: + nbr_success = nbr_imported + nbr_error = self.collection.nbr_item - nbr_imported + + if nbr_error: + state = "failed" + else: + state = "done" + result = self.env["ir.qweb"]._render( + "pattern_import_export.format_message", res + ) + return { + "record_ids": res.get("ids"), + "messages": res.get("messages"), + "result_info": result, + "state": state, + "nbr_success": nbr_success, + "nbr_error": nbr_error, + } diff --git a/pattern_import_export/models/__init__.py b/pattern_import_export/models/__init__.py index 74322801..307fad64 100644 --- a/pattern_import_export/models/__init__.py +++ b/pattern_import_export/models/__init__.py @@ -1,9 +1,9 @@ from . import pattern_config +from . import chunk_group from . import ir_exports_line from . import ir_exports from . import ir_actions from . import base from . import ir_fields from . import pattern_file -from . import pattern_chunk from . import ir_attachment diff --git a/pattern_import_export/models/chunk_group.py b/pattern_import_export/models/chunk_group.py new file mode 100644 index 00000000..7bf0e521 --- /dev/null +++ b/pattern_import_export/models/chunk_group.py @@ -0,0 +1,22 @@ +# Copyright 2021 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import base64 + +from odoo import fields, models + + +class ChunkGroup(models.Model): + _inherit = "chunk.group" + + # We use a O2M but as there is a sql constraint we can have only + # one pattern file, this is why the fieldname end with "id" + pattern_file_id = fields.One2many("pattern.file", "chunk_group_id", "Pattern File") + + def _get_data(self): + self.ensure_one() + if self.pattern_file_id: + return base64.b64decode(self.pattern_file_id.datas.decode("utf-8")) + else: + return super()._get_data() diff --git a/pattern_import_export/models/pattern_file.py b/pattern_import_export/models/pattern_file.py index ccbe4879..8f6e4c2b 100644 --- a/pattern_import_export/models/pattern_file.py +++ b/pattern_import_export/models/pattern_file.py @@ -1,8 +1,6 @@ # Copyright (c) Akretion 2020 # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html) -import base64 -import json import urllib.parse from odoo import _, api, fields, models @@ -14,46 +12,61 @@ class PatternFile(models.Model): _description = "Attachment with pattern file metadata" attachment_id = fields.Many2one("ir.attachment", required=True, ondelete="cascade") - state = fields.Selection( - [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], - default="pending", - ) - info = fields.Char() kind = fields.Selection([("import", "import"), ("export", "export")], required=True) pattern_config_id = fields.Many2one( "pattern.config", required=True, string="Export pattern" ) - nbr_error = fields.Integer(compute="_compute_stat") - nbr_success = fields.Integer(compute="_compute_stat") - progress = fields.Float(compute="_compute_stat") - chunk_ids = fields.One2many("pattern.chunk", "pattern_file_id", "Chunk") - date_done = fields.Datetime() - - @api.depends("chunk_ids.nbr_error", "chunk_ids.nbr_success") - def _compute_stat(self): + chunk_group_id = fields.Many2one("chunk.group", string="Chunk Group") + chunk_item_ids = fields.One2many("chunk.item", related="chunk_group_id.item_ids") + state = fields.Selection( + [("pending", "Pending"), ("failed", "Failed"), ("done", "Done")], + compute="_compute_state", + ) + date_done = fields.Date(compute="_compute_date_done", store=True) + progress = fields.Float(related="chunk_group_id.progress") + nbr_error = fields.Integer(related="chunk_group_id.nbr_error") + nbr_success = fields.Integer(related="chunk_group_id.nbr_success") + info = fields.Char(related="chunk_group_id.info") + + _sql_constraints = [ + ("uniq_chunk_group_id", "unique(chunk_group_id)", "The Group must be unique!") + ] + + def _add_chunk_group(self): for record in self: - record.nbr_error = sum(record.mapped("chunk_ids.nbr_error")) - record.nbr_success = sum(record.mapped("chunk_ids.nbr_success")) - todo = sum(record.mapped("chunk_ids.nbr_item")) - if todo: - record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo - else: - record.progress = 0 + config = record.pattern_config_id + record.chunk_group_id = self.env["chunk.group"].create( + { + "job_priority": config.job_priority, + "process_multi": config.process_multi, + "data_format": "json", + "apply_on_model": config.resource, + "usage": "pattern.import", + } + ) @api.model_create_multi - def create(self, vals): - result = super().create(vals) - for record in result: - if record.state != "pending": - record._notify_user() - return result - - def write(self, vals): - result = super().write(vals) - if "state" in vals.keys() and vals["state"] != "pending": - for rec in self: - rec._notify_user() - return result + def create(self, vals_list): + records = super().create(vals_list) + records._add_chunk_group() + return records + + @api.depends("kind", "chunk_group_id.date_done") + def _compute_date_done(self): + for record in self: + if record.kind == "export" and not record.date_done: + record.date_done = fields.Date.today() + elif record.kind == "import": + record.date_done = record.chunk_group_id.date_done + + @api.depends("kind", "chunk_group_id.state") + def _compute_state(self): + for record in self: + if record.kind == "export": + record.state = "done" + elif record.kind == "import": + record.state = record.chunk_group_id.state + record._notify_user() def _notify_user(self): import_or_export = _("Import") if self.kind == "import" else _("Export") @@ -114,74 +127,8 @@ def _helper_build_content_link(self): link += "" + _("Download") + "" return link - def _parse_data(self): - data = base64.b64decode(self.datas.decode("utf-8")) - target_function = "_parse_data_{format}".format( - format=self.pattern_config_id.export_format or "" - ) - if not hasattr(self, target_function): - raise NotImplementedError() - return getattr(self, target_function)(data) - - def _parse_data_json(self, data): - items = json.loads(data.decode("utf-8")) - for idx, item in enumerate(items): - yield idx + 1, item - - def _prepare_chunk(self, start_idx, stop_idx, data): - return { - "start_idx": start_idx, - "stop_idx": stop_idx, - "data": data, - "nbr_item": len(data), - "state": "pending", - "pattern_file_id": self.id, - } - - def _should_create_chunk(self, items, next_item): - """Customise this code if you want to add some additionnal - item after reaching the limit""" - return len(items) > self.pattern_config_id.chunk_size - - def _create_chunk(self, start_idx, stop_idx, data): - vals = self._prepare_chunk(start_idx, stop_idx, data) - chunk = self.env["pattern.chunk"].create(vals) - # we enqueue the chunk in case of multi process of if it's the first chunk - if self.pattern_config_id.process_multi or len(self.chunk_ids) == 1: - chunk.with_delay(priority=self.pattern_config_id.job_priority).run() - return chunk - def split_in_chunk(self): - """Split Pattern File into Pattern Chunk""" - # purge chunk in case of retring a job - self.chunk_ids.unlink() - try: - items = [] - start_idx = 1 - previous_idx = None - # idx is the index position in the original file - # we can have empty line that can be skipped - for idx, item in self._parse_data(): - if self._should_create_chunk(items, item): - self._create_chunk(start_idx, previous_idx, items) - items = [] - start_idx = idx - items.append((idx, item)) - previous_idx = idx - if items: - self._create_chunk(start_idx, idx, items) - except Exception as e: - self.state = "failed" - self.info = _("Failed to create the chunk: %s") % e - return True - - def set_import_done(self): - for record in self: - if record.nbr_error: - record.state = "failed" - else: - record.state = "done" - record.date_done = fields.Datetime.now() + return self.chunk_group_id.split_in_chunk() def refresh(self): """Empty function to refresh view""" diff --git a/pattern_import_export/security/ir.model.access.csv b/pattern_import_export/security/ir.model.access.csv index c97bba1b..02566180 100644 --- a/pattern_import_export/security/ir.model.access.csv +++ b/pattern_import_export/security/ir.model.access.csv @@ -2,6 +2,5 @@ id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink access_export_pattern_wizard_user,export.pattern.wizard.user,model_export_pattern_wizard,group_pattern_user,1,1,1,1 access_import_pattern_wizard_user,import.pattern.wizard.user,model_import_pattern_wizard,group_pattern_user,1,1,1,1 access_pattern_file_user,pattern.file.user,model_pattern_file,group_pattern_user,1,1,1,1 -access_pattern_chunk_user,pattern.chunk.user,model_pattern_chunk,group_pattern_user,1,1,1,1 access_pattern_config_user,pattern.config.user,model_pattern_config,group_pattern_user,1,0,0,0 access_pattern_config_manager,pattern.config.manager,model_pattern_config,group_pattern_manager,1,1,1,1 diff --git a/pattern_import_export/tests/test_pattern_import.py b/pattern_import_export/tests/test_pattern_import.py index fa326c29..9776deb4 100644 --- a/pattern_import_export/tests/test_pattern_import.py +++ b/pattern_import_export/tests/test_pattern_import.py @@ -2,13 +2,14 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). from uuid import uuid4 -from odoo.tests.common import SavepointCase from odoo.tools import mute_logger +from odoo.addons.component.tests.common import SavepointComponentCase + from .common import PatternCommon -class TestPatternImport(PatternCommon, SavepointCase): +class TestPatternImport(PatternCommon, SavepointComponentCase): @classmethod def setUpClass(cls): super().setUpClass() @@ -58,7 +59,7 @@ def test_update_with_external_id_bad_data_1(self): self.assertEqual(pattern_file.state, "failed") # TODO it will be better to retour a better exception # but it's not that easy - chunk = pattern_file.chunk_ids + chunk = pattern_file.chunk_item_ids self.assertEqual( chunk.result_info, "

Fail to process chunk 'int' object has no attribute 'split'

", @@ -76,7 +77,7 @@ def test_update_with_external_id_bad_data_2(self): records = self.run_pattern_file(pattern_file) self.assertFalse(records) self.assertEqual(pattern_file.state, "failed") - chunk = pattern_file.chunk_ids + chunk = pattern_file.chunk_item_ids self.assertIn("Invalid database identifier", chunk.result_info) def test_create_new_record(self): @@ -254,7 +255,7 @@ def test_wrong_import(self): self.run_pattern_file(pattern_file) self.assertEqual(pattern_file.state, "failed") self.assertEqual(pattern_file.nbr_error, 1) - self.assertIn("res_partner_check_name", pattern_file.chunk_ids.result_info) + self.assertIn("res_partner_check_name", pattern_file.chunk_item_ids.result_info) def test_m2m_with_empty_columns(self): unique_name = str(uuid4()) @@ -334,7 +335,7 @@ def test_missing_record(self): "No value found for model 'Country' with the field 'code' " "and the value 'Fake'" ), - pattern_file.chunk_ids.result_info, + pattern_file.chunk_item_ids.result_info, ) def test_import_m2o_key(self): @@ -415,5 +416,9 @@ def test_partial_import_too_many_error(self): self.assertEqual(len(records), 2) self.assertEqual(pattern_file.state, "failed") self.assertEqual(pattern_file.nbr_error, 16) - self.assertIn("Contacts require a name", pattern_file.chunk_ids.result_info) - self.assertIn("Found more than 10 errors", pattern_file.chunk_ids.result_info) + self.assertIn( + "Contacts require a name", pattern_file.chunk_item_ids.result_info + ) + self.assertIn( + "Found more than 10 errors", pattern_file.chunk_item_ids.result_info + ) diff --git a/pattern_import_export/views/pattern_file.xml b/pattern_import_export/views/pattern_file.xml index 001e3acf..eafca5f3 100644 --- a/pattern_import_export/views/pattern_file.xml +++ b/pattern_import_export/views/pattern_file.xml @@ -12,9 +12,9 @@ - + @@ -76,7 +76,7 @@ readonly="1" attrs="{'invisible': [('info', '=', False)]}" /> - + diff --git a/setup/chunk_processing/odoo/addons/chunk_processing b/setup/chunk_processing/odoo/addons/chunk_processing new file mode 120000 index 00000000..d865efb9 --- /dev/null +++ b/setup/chunk_processing/odoo/addons/chunk_processing @@ -0,0 +1 @@ +../../../../chunk_processing \ No newline at end of file diff --git a/setup/chunk_processing/setup.py b/setup/chunk_processing/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/chunk_processing/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)