diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..a5e43536f3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# generated from manifests external_dependencies +fastexcel +polars diff --git a/sheet_dataframe_process/README.rst b/sheet_dataframe_process/README.rst new file mode 100644 index 0000000000..a1b5037779 --- /dev/null +++ b/sheet_dataframe_process/README.rst @@ -0,0 +1,115 @@ +======================= +Sheet Dataframe Process +======================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github + :target: https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process + :alt: OCA/reporting-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-sheet_dataframe_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +From an imported spreadsheet (xlsx), this module allows to transform +file data in Polars dataframe and process them according to rules in +order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database + but in memory: you don't need to iterate on each line to perform + operations +- the operations are powerful: filter, add column resulting from + calculation, select a subset of data + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/sheet_dataframe_process/__init__.py b/sheet_dataframe_process/__init__.py new file mode 100644 index 0000000000..aee8895e7a --- /dev/null +++ b/sheet_dataframe_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/sheet_dataframe_process/__manifest__.py b/sheet_dataframe_process/__manifest__.py new file mode 100644 index 0000000000..abed5a41c8 --- /dev/null +++ b/sheet_dataframe_process/__manifest__.py @@ -0,0 +1,35 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Sheet Dataframe Process", + "version": "18.0.1.0.0", + "summary": "Allow to create a Polars dataframe from a sheet file and " + "process it according to rules", + "category": "Reporting", + "license": "AGPL-3", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/reporting-engine", + "maintainers": ["bealdav"], + "depends": [ + "contacts", + ], + "external_dependencies": { + "python": [ + "polars", + "fastexcel", + ] + }, + "data": [ + "data/action.xml", + "data/demo.xml", + "security/ir.model.access.xml", + "wizards/sheet_dataframe.xml", + "views/file_config.xml", + "views/file_field.xml", + "views/file_partner_field.xml", + "views/try_file.xml", + "views/menu.xml", + ], + "installable": True, +} diff --git a/sheet_dataframe_process/data/action.xml b/sheet_dataframe_process/data/action.xml new file mode 100644 index 0000000000..1a0754e881 --- /dev/null +++ b/sheet_dataframe_process/data/action.xml @@ -0,0 +1,10 @@ + + + + 🐻‍❄️ Populate file polars example + + + code + env["try.file"]._populate() + + diff --git a/sheet_dataframe_process/data/demo.xml b/sheet_dataframe_process/data/demo.xml new file mode 100644 index 0000000000..dd3728ab26 --- /dev/null +++ b/sheet_dataframe_process/data/demo.xml @@ -0,0 +1,38 @@ + + + + stop + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sheet_dataframe_process/models/__init__.py b/sheet_dataframe_process/models/__init__.py new file mode 100644 index 0000000000..d7bb43d5ed --- /dev/null +++ b/sheet_dataframe_process/models/__init__.py @@ -0,0 +1,6 @@ +from . import file_config +from . import file_field +from . import file_partner_field +from . import try_file +from . import ir_model_fields +from . import partner diff --git a/sheet_dataframe_process/models/file_config.py b/sheet_dataframe_process/models/file_config.py new file mode 100644 index 0000000000..69a2d83411 --- /dev/null +++ b/sheet_dataframe_process/models/file_config.py @@ -0,0 +1,84 @@ +from odoo import Command, fields, models + +# Demo record +UISTRING = "sheet_dataframe_process.file_config_contact" + + +class FileConfig(models.Model): + _name = "file.config" + _inherit = "mail.thread" + _description = "File Configuration" + _rec_name = "model_id" + + model_id = fields.Many2one( + comodel_name="ir.model", + required=True, + copy=False, + ondelete="cascade", + tracking=True, + ) + code = fields.Char(help="Allow to browse between several identical models") + action = fields.Selection( + selection=[ + ("display", "Display"), + ("dataframe", "Dataframe"), + ], + default="display", + tracking=True, + help="Some other behaviors can be implemented", + ) + on_fail = fields.Selection( + selection=[("stop", "Stop"), ("skip", "Skip record (TODO)")], + default="stop", + tracking=True, + help="What should be the behavior in case of failure regarding constraint " + "fields (required, format, etc)\n\n" + " - Stop: stop the process by raising an exception\n" + " - Skip record: current line'll be ignored from the next process", + ) + partner_ids = fields.Many2many( + comodel_name="res.partner", + domain="[('active', 'in', (True, False))]", + tracking=True, + ) + field_ids = fields.One2many( + comodel_name="file.field", inverse_name="config_id", copy=True + ) + field_match_ids = fields.One2many( + comodel_name="file.partner.field", inverse_name="config_id", copy=True + ) + + def populate_match_lines(self): + # TODO use api depends instead of ui button ? + self.ensure_one() + for partner in self.partner_ids: + ffields = self.field_match_ids.filtered( + lambda s, partner=partner: s.partner_id == partner + ).mapped("field_id") + line_ids = self.field_ids.filtered( + lambda s, ffields=ffields: s.field_id not in ffields + ).mapped("id") + self.field_match_ids = [ + Command.create({"partner_id": partner.id, "line_id": x}) + for x in line_ids + ] + self.field_match_ids.filtered( + lambda s: s.partner_id not in s.config_id.partner_ids + ).unlink() + for rec in self: + if rec == self.env.ref(UISTRING): + rec._populate_demo_column_names() + + def _populate_match(self, mfield, mstring, uidstring): + record = self.field_match_ids.filtered( + lambda s, field=mfield, uidstring=uidstring: s.field_id.name == field + and s.partner_id == s.config_id.partner_ids[0] + ) + if record: + record.matching_column = mstring + + def _populate_demo_column_names(self): + self.field_match_ids.matching_column = False + self._populate_match("street", "myStreet", UISTRING) + self._populate_match("street2", "Second street", UISTRING) + self._populate_match("country_code", "Country", UISTRING) diff --git a/sheet_dataframe_process/models/file_field.py b/sheet_dataframe_process/models/file_field.py new file mode 100644 index 0000000000..ce1af8de92 --- /dev/null +++ b/sheet_dataframe_process/models/file_field.py @@ -0,0 +1,29 @@ +from odoo import fields, models + + +class FileField(models.Model): + _name = "file.field" + _inherit = ["mail.thread"] + _description = "Configuration de l'import de champ" + _order = "field_id ASC" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade" + ) + field_id = fields.Many2one( + comodel_name="ir.model.fields", + ondelete="cascade", + required=True, + domain="[('model_id', '=', model_id)]", + ) + model_id = fields.Many2one( + comodel_name="ir.model", + related="config_id.model_id", + readonly=True, + ) + required = fields.Boolean( + help="Prevent to import missing data if field is missing in some records", + ) + check_type = fields.Boolean( + help="Check data type is compatible", + ) diff --git a/sheet_dataframe_process/models/file_partner_field.py b/sheet_dataframe_process/models/file_partner_field.py new file mode 100644 index 0000000000..488fce5c16 --- /dev/null +++ b/sheet_dataframe_process/models/file_partner_field.py @@ -0,0 +1,16 @@ +from odoo import fields, models + + +class FilePartnerField(models.Model): + _name = "file.partner.field" + _inherits = {"file.field": "line_id"} + _description = "Configuration de l'import de champ" + _order = "partner_id ASC, field_id ASC" + + line_id = fields.Many2one( + comodel_name="file.field", required=True, ondelete="cascade" + ) + partner_id = fields.Many2one( + comodel_name="res.partner", + ) + matching_column = fields.Char(help="Field name in spreadsheet") diff --git a/sheet_dataframe_process/models/ir_model_fields.py b/sheet_dataframe_process/models/ir_model_fields.py new file mode 100644 index 0000000000..14d2039c81 --- /dev/null +++ b/sheet_dataframe_process/models/ir_model_fields.py @@ -0,0 +1,14 @@ +from odoo import api, models + + +class IrModelFields(models.Model): + _inherit = "ir.model.fields" + + @api.depends("field_description", "model") + def _compute_display_name(self): + super()._compute_display_name() + if self.env.context.get("technical_name"): + for field in self: + if self.env.context.get("technical_name"): + field.display_name = field.name + return diff --git a/sheet_dataframe_process/models/partner.py b/sheet_dataframe_process/models/partner.py new file mode 100644 index 0000000000..1a74eb89d1 --- /dev/null +++ b/sheet_dataframe_process/models/partner.py @@ -0,0 +1,19 @@ +from odoo import models + + +class ResPartner(models.Model): + _inherit = "res.partner" + + def sheet_dataframe_import(self): + self.ensure_one() + transient = self.env["sheet.dataframe.transient"].create( + { + "config_id": self.env.context.get("config_id")["id"], + "partner_id": self.id, + } + ) + action = self.env.ref( + "sheet_dataframe_process.sheet_dataframe_transient_action" + )._get_action_dict() + action["res_id"] = transient.id + return action diff --git a/sheet_dataframe_process/models/try_file.py b/sheet_dataframe_process/models/try_file.py new file mode 100644 index 0000000000..230fb4246c --- /dev/null +++ b/sheet_dataframe_process/models/try_file.py @@ -0,0 +1,85 @@ +import base64 +from pathlib import Path + +from odoo import fields, models +from odoo.modules.module import get_module_path + + +class TestPolarsFile(models.Model): + _name = "try.file" + _description = "Example files to ensure your configuration match with cases" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade", readonly=True + ) + name = fields.Char() + template = fields.Binary(string="Fichier", attachment=False) + + def _populate(self): + def create_attach(myfile, addon, idstring, relative_path): + with open(myfile, "rb") as f: + vals = { + "config_id": self.env.ref(idstring).id, + "name": f.name[f.name.find(addon) :], + } + self.env[self._name].sudo().create(vals) + + self.env[self._name].search([("template", "=", False)]).unlink() + paths = self._get_test_file_paths() + for addon, data in paths.items(): + relative_path = data["relative_path"] + idstring = f"{addon}.{data['xmlid']}" + if self.env.ref(idstring): + mpath = Path(get_module_path(addon)) / relative_path + for mfile in tuple(mpath.iterdir()): + create_attach(mfile, addon, idstring, relative_path) + action = self.env.ref( + "sheet_dataframe_process.try_file_action" + )._get_action_dict() + return action + + def try_import(self): + self.ensure_one() + transient = self.env["sheet.dataframe.transient"].create( + { + "filename": self.name, + "file": self._get_file(), + "config_id": self.config_id.id, + } + ) + action = self.env.ref( + "sheet_dataframe_process.sheet_dataframe_transient_action" + )._get_action_dict() + action["res_id"] = transient.id + return action + + def _get_file(self): + # TODO Clean + if self.template: + return self.template + module = self.name[: self.name.find("/")] + relative = self._get_test_file_paths().get(module) + relative = relative and relative.get("relative_path") + if relative: + path = Path(get_module_path(module)) + path = path / relative / self.name[self.name.rfind("/") + 1 :] + # myfile = path / self.name + with open(path, "rb") as f: + return base64.b64encode(f.read()) + + def _get_test_file_paths(self): + """ + You may override if you want populate files in your module + returns: + {"module_name": { + "relative_path": "tests/files", + "xmlid": "file_config_xml_id"} + } + } + """ + return { + "sheet_dataframe_process": { + "relative_path": "tests/files", + "xmlid": "file_config_contact", + } + } diff --git a/sheet_dataframe_process/pyproject.toml b/sheet_dataframe_process/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/sheet_dataframe_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/sheet_dataframe_process/readme/DESCRIPTION.md b/sheet_dataframe_process/readme/DESCRIPTION.md new file mode 100644 index 0000000000..b2eedffec0 --- /dev/null +++ b/sheet_dataframe_process/readme/DESCRIPTION.md @@ -0,0 +1,27 @@ +From an imported spreadsheet (xlsx), this module allows to transform file data in +Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference (column names, number of columns, dirty paging) but contains data related to same concepts. +Then you want apply them a common process to automate things. +For that you need to transform/arrange data to the same way + + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database but in memory: you don't need to iterate on each line to perform operations +- the operations are powerful: filter, add column resulting from calculation, select a subset of data + + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project diff --git a/sheet_dataframe_process/security/ir.model.access.xml b/sheet_dataframe_process/security/ir.model.access.xml new file mode 100644 index 0000000000..eabce61120 --- /dev/null +++ b/sheet_dataframe_process/security/ir.model.access.xml @@ -0,0 +1,51 @@ + + + File Config + + + + + + + + + + File Field + + + + + + + + + + File Partner Field + + + + + + + + + + sheet.dataframe.transient + + + + + + + + + + try.file + + + + + + + + diff --git a/sheet_dataframe_process/static/description/icon.png b/sheet_dataframe_process/static/description/icon.png new file mode 100644 index 0000000000..d1d2b57297 Binary files /dev/null and b/sheet_dataframe_process/static/description/icon.png differ diff --git a/sheet_dataframe_process/static/description/index.html b/sheet_dataframe_process/static/description/index.html new file mode 100644 index 0000000000..55197dcb66 --- /dev/null +++ b/sheet_dataframe_process/static/description/index.html @@ -0,0 +1,453 @@ + + + + + +Sheet Dataframe Process + + + +
+

Sheet Dataframe Process

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

From an imported spreadsheet (xlsx), this module allows to transform +file data in Polars dataframe and process them according to rules in +order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update or import

+

Typical use case:

+

You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way

+

Why dataframe ?

+
    +
  • a dataframe is a kind of in-memory dataset on which you can operate
    • +
  • you can operates on your entire dataset a bit like with a database +but in memory: you don’t need to iterate on each line to perform +operations
    • +
  • the operations are powerful: filter, add column resulting from +calculation, select a subset of data
    • +
+

Why Polars ?

+
    +
  • performance: code in rust
    • +
  • environment consideration
    • +
  • dynamic project
    • +
+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/sheet_dataframe_process/tests/__init__.py b/sheet_dataframe_process/tests/__init__.py new file mode 100644 index 0000000000..d9b96c4fa5 --- /dev/null +++ b/sheet_dataframe_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/sheet_dataframe_process/tests/files/4_fields.xlsx b/sheet_dataframe_process/tests/files/4_fields.xlsx new file mode 100644 index 0000000000..66c858ae68 Binary files /dev/null and b/sheet_dataframe_process/tests/files/4_fields.xlsx differ diff --git a/sheet_dataframe_process/tests/files/missing_required_column.xlsx b/sheet_dataframe_process/tests/files/missing_required_column.xlsx new file mode 100644 index 0000000000..b1ce314ed6 Binary files /dev/null and b/sheet_dataframe_process/tests/files/missing_required_column.xlsx differ diff --git a/sheet_dataframe_process/tests/files/wrong_date.xlsx b/sheet_dataframe_process/tests/files/wrong_date.xlsx new file mode 100644 index 0000000000..67808f7ee9 Binary files /dev/null and b/sheet_dataframe_process/tests/files/wrong_date.xlsx differ diff --git a/sheet_dataframe_process/tests/test_module.py b/sheet_dataframe_process/tests/test_module.py new file mode 100644 index 0000000000..51c5e6bf6e --- /dev/null +++ b/sheet_dataframe_process/tests/test_module.py @@ -0,0 +1,39 @@ +from lxml import etree + +from odoo.tests.common import TransactionCase + + +class TestModule(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env["try.file"]._populate() + cls.env.ref( + "sheet_dataframe_process.file_config_contact" + ).populate_match_lines() + cls.file_records = cls.env["try.file"].search([]) + + def test_missing(self): + wiz = self.get_wizard(self.file_records, "missing_required") + self.assertTrue(wiz.partner_id) + comment = sanitize(str(wiz.comment)) + root = etree.fromstring(comment) + self.assertEqual( + root.xpath('//div[@id="missing-name-values"]/div')[0].text, + "Missing 'name' values", + ) + + # def test_four_fields(self): + # wiz = self.get_wizard(self.file_records, "4_fields") + # self.assertFalse(wiz.missing_cols) + + def get_wizard(self, file_recs, file_str): + action = file_recs.filtered( + lambda s, file_str=file_str: file_str in s.name + ).try_import() + return self.env["sheet.dataframe.transient"].browse(action.get("res_id")) + + +def sanitize(string): + string = string.replace("
", "
") + return string diff --git a/sheet_dataframe_process/views/file_config.xml b/sheet_dataframe_process/views/file_config.xml new file mode 100644 index 0000000000..b1c9485507 --- /dev/null +++ b/sheet_dataframe_process/views/file_config.xml @@ -0,0 +1,95 @@ + + + file.config + +
+
+
+ + + + + + + + + + + +