From fd02a15729b60ddc27c35dd94b97d798929f7e1c Mon Sep 17 00:00:00 2001 From: David Beal Date: Fri, 18 Oct 2024 15:31:41 +0200 Subject: [PATCH 1/4] [ADD] sheet_to_dataframe: convert file to polars dataframe and process it --- requirements.txt | 3 + sheet_dataframe_process/README.rst | 115 +++++ sheet_dataframe_process/__init__.py | 2 + sheet_dataframe_process/__manifest__.py | 35 ++ sheet_dataframe_process/data/action.xml | 10 + sheet_dataframe_process/data/demo.xml | 33 ++ sheet_dataframe_process/models/__init__.py | 6 + sheet_dataframe_process/models/file_config.py | 76 +++ sheet_dataframe_process/models/file_field.py | 26 + .../models/file_partner_field.py | 16 + .../models/ir_model_fields.py | 14 + sheet_dataframe_process/models/partner.py | 19 + sheet_dataframe_process/models/try_file.py | 85 ++++ sheet_dataframe_process/pyproject.toml | 3 + sheet_dataframe_process/readme/DESCRIPTION.md | 27 ++ .../security/ir.model.access.xml | 51 ++ .../static/description/icon.png | Bin 0 -> 2605 bytes .../static/description/index.html | 453 ++++++++++++++++++ sheet_dataframe_process/tests/__init__.py | 1 + .../tests/files/4_fields.xlsx | Bin 0 -> 5391 bytes .../tests/files/missing_required_column.xlsx | Bin 0 -> 5330 bytes sheet_dataframe_process/tests/test_module.py | 23 + sheet_dataframe_process/views/file_config.xml | 94 ++++ sheet_dataframe_process/views/file_field.xml | 23 + .../views/file_partner_field.xml | 24 + sheet_dataframe_process/views/menu.xml | 33 ++ sheet_dataframe_process/views/try_file.xml | 24 + sheet_dataframe_process/wizards/__init__.py | 1 + .../wizards/sheet_dataframe.py | 89 ++++ .../wizards/sheet_dataframe.xml | 52 ++ 30 files changed, 1338 insertions(+) create mode 100644 requirements.txt create mode 100644 sheet_dataframe_process/README.rst create mode 100644 sheet_dataframe_process/__init__.py create mode 100644 sheet_dataframe_process/__manifest__.py create mode 100644 sheet_dataframe_process/data/action.xml create mode 100644 sheet_dataframe_process/data/demo.xml create mode 100644 sheet_dataframe_process/models/__init__.py create mode 100644 sheet_dataframe_process/models/file_config.py create mode 100644 sheet_dataframe_process/models/file_field.py create mode 100644 sheet_dataframe_process/models/file_partner_field.py create mode 100644 sheet_dataframe_process/models/ir_model_fields.py create mode 100644 sheet_dataframe_process/models/partner.py create mode 100644 sheet_dataframe_process/models/try_file.py create mode 100644 sheet_dataframe_process/pyproject.toml create mode 100644 sheet_dataframe_process/readme/DESCRIPTION.md create mode 100644 sheet_dataframe_process/security/ir.model.access.xml create mode 100644 sheet_dataframe_process/static/description/icon.png create mode 100644 sheet_dataframe_process/static/description/index.html create mode 100644 sheet_dataframe_process/tests/__init__.py create mode 100644 sheet_dataframe_process/tests/files/4_fields.xlsx create mode 100644 sheet_dataframe_process/tests/files/missing_required_column.xlsx create mode 100644 sheet_dataframe_process/tests/test_module.py create mode 100644 sheet_dataframe_process/views/file_config.xml create mode 100644 sheet_dataframe_process/views/file_field.xml create mode 100644 sheet_dataframe_process/views/file_partner_field.xml create mode 100644 sheet_dataframe_process/views/menu.xml create mode 100644 sheet_dataframe_process/views/try_file.xml create mode 100644 sheet_dataframe_process/wizards/__init__.py create mode 100644 sheet_dataframe_process/wizards/sheet_dataframe.py create mode 100644 sheet_dataframe_process/wizards/sheet_dataframe.xml diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..a5e43536f3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# generated from manifests external_dependencies +fastexcel +polars diff --git a/sheet_dataframe_process/README.rst b/sheet_dataframe_process/README.rst new file mode 100644 index 0000000000..a1b5037779 --- /dev/null +++ b/sheet_dataframe_process/README.rst @@ -0,0 +1,115 @@ +======================= +Sheet Dataframe Process +======================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github + :target: https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process + :alt: OCA/reporting-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-sheet_dataframe_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +From an imported spreadsheet (xlsx), this module allows to transform +file data in Polars dataframe and process them according to rules in +order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database + but in memory: you don't need to iterate on each line to perform + operations +- the operations are powerful: filter, add column resulting from + calculation, select a subset of data + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/sheet_dataframe_process/__init__.py b/sheet_dataframe_process/__init__.py new file mode 100644 index 0000000000..aee8895e7a --- /dev/null +++ b/sheet_dataframe_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/sheet_dataframe_process/__manifest__.py b/sheet_dataframe_process/__manifest__.py new file mode 100644 index 0000000000..abed5a41c8 --- /dev/null +++ b/sheet_dataframe_process/__manifest__.py @@ -0,0 +1,35 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Sheet Dataframe Process", + "version": "18.0.1.0.0", + "summary": "Allow to create a Polars dataframe from a sheet file and " + "process it according to rules", + "category": "Reporting", + "license": "AGPL-3", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/reporting-engine", + "maintainers": ["bealdav"], + "depends": [ + "contacts", + ], + "external_dependencies": { + "python": [ + "polars", + "fastexcel", + ] + }, + "data": [ + "data/action.xml", + "data/demo.xml", + "security/ir.model.access.xml", + "wizards/sheet_dataframe.xml", + "views/file_config.xml", + "views/file_field.xml", + "views/file_partner_field.xml", + "views/try_file.xml", + "views/menu.xml", + ], + "installable": True, +} diff --git a/sheet_dataframe_process/data/action.xml b/sheet_dataframe_process/data/action.xml new file mode 100644 index 0000000000..1a0754e881 --- /dev/null +++ b/sheet_dataframe_process/data/action.xml @@ -0,0 +1,10 @@ + + + + 🐻‍❄️ Populate file polars example + + + code + env["try.file"]._populate() + + diff --git a/sheet_dataframe_process/data/demo.xml b/sheet_dataframe_process/data/demo.xml new file mode 100644 index 0000000000..7a9009ce9c --- /dev/null +++ b/sheet_dataframe_process/data/demo.xml @@ -0,0 +1,33 @@ + + + + stop + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sheet_dataframe_process/models/__init__.py b/sheet_dataframe_process/models/__init__.py new file mode 100644 index 0000000000..d7bb43d5ed --- /dev/null +++ b/sheet_dataframe_process/models/__init__.py @@ -0,0 +1,6 @@ +from . import file_config +from . import file_field +from . import file_partner_field +from . import try_file +from . import ir_model_fields +from . import partner diff --git a/sheet_dataframe_process/models/file_config.py b/sheet_dataframe_process/models/file_config.py new file mode 100644 index 0000000000..0fd6ec32f0 --- /dev/null +++ b/sheet_dataframe_process/models/file_config.py @@ -0,0 +1,76 @@ +from odoo import Command, fields, models + +# Demo record +UISTRING = "sheet_dataframe_process.file_config_contact" + + +class FileConfig(models.Model): + _name = "file.config" + _inherit = "mail.thread" + _description = "File Configuration" + _rec_name = "model_id" + + model_id = fields.Many2one( + comodel_name="ir.model", required=True, ondelete="cascade" + ) + code = fields.Char(help="Allow to browse between several identical models") + action = fields.Selection( + selection=[ + ("display", "Display"), + ("dataframe", "Dataframe"), + ], + default="display", + help="Some other behaviors can be implemented", + ) + on_fail = fields.Selection( + selection=[("stop", "Stop"), ("skip", "Skip record (TODO)")], + default="stop", + help="What should be the behavior in case of failure regarding constraint " + "fields (required, format, etc)\n\n" + " - Stop: stop the process by raising an exception\n" + " - Skip record: current line'll be ignored from the next process", + ) + partner_ids = fields.Many2many( + comodel_name="res.partner", domain="[('active', 'in', (True, False))]" + ) + field_ids = fields.One2many( + comodel_name="file.field", inverse_name="config_id", copy=True + ) + field_match_ids = fields.One2many( + comodel_name="file.partner.field", inverse_name="config_id", copy=True + ) + + def populate_match_lines(self): + # TODO use api depends instead of ui button ? + self.ensure_one() + for partner in self.partner_ids: + ffields = self.field_match_ids.filtered( + lambda s, partner=partner: s.partner_id == partner + ).mapped("field_id") + line_ids = self.field_ids.filtered( + lambda s, ffields=ffields: s.field_id not in ffields + ).mapped("id") + self.field_match_ids = [ + Command.create({"partner_id": partner.id, "line_id": x}) + for x in line_ids + ] + self.field_match_ids.filtered( + lambda s: s.partner_id not in s.config_id.partner_ids + ).unlink() + for rec in self: + if rec == self.env.ref(UISTRING): + rec._populate_demo_column_names() + + def _populate_match(self, mfield, mstring, uidstring): + record = self.field_match_ids.filtered( + lambda s, field=mfield, uidstring=uidstring: s.field_id.name == field + and s.partner_id == s.config_id.partner_ids[0] + ) + if record: + record.matching_column = mstring + + def _populate_demo_column_names(self): + self.field_match_ids.matching_column = False + self._populate_match("street", "myStreet", UISTRING) + self._populate_match("street2", "Second street", UISTRING) + self._populate_match("country_code", "Country", UISTRING) diff --git a/sheet_dataframe_process/models/file_field.py b/sheet_dataframe_process/models/file_field.py new file mode 100644 index 0000000000..41cbb15798 --- /dev/null +++ b/sheet_dataframe_process/models/file_field.py @@ -0,0 +1,26 @@ +from odoo import fields, models + + +class FileField(models.Model): + _name = "file.field" + _description = "Configuration de l'import de champ" + _order = "field_id ASC" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade" + ) + field_id = fields.Many2one( + comodel_name="ir.model.fields", + ondelete="cascade", + required=True, + domain="[('model_id', '=', model_id)]", + ) + model_id = fields.Many2one( + comodel_name="ir.model", + related="config_id.model_id", + readonly=True, + ) + required = fields.Boolean( + tracking=True, + help="Prevent to import missing data if field is missing in some records", + ) diff --git a/sheet_dataframe_process/models/file_partner_field.py b/sheet_dataframe_process/models/file_partner_field.py new file mode 100644 index 0000000000..488fce5c16 --- /dev/null +++ b/sheet_dataframe_process/models/file_partner_field.py @@ -0,0 +1,16 @@ +from odoo import fields, models + + +class FilePartnerField(models.Model): + _name = "file.partner.field" + _inherits = {"file.field": "line_id"} + _description = "Configuration de l'import de champ" + _order = "partner_id ASC, field_id ASC" + + line_id = fields.Many2one( + comodel_name="file.field", required=True, ondelete="cascade" + ) + partner_id = fields.Many2one( + comodel_name="res.partner", + ) + matching_column = fields.Char(help="Field name in spreadsheet") diff --git a/sheet_dataframe_process/models/ir_model_fields.py b/sheet_dataframe_process/models/ir_model_fields.py new file mode 100644 index 0000000000..14d2039c81 --- /dev/null +++ b/sheet_dataframe_process/models/ir_model_fields.py @@ -0,0 +1,14 @@ +from odoo import api, models + + +class IrModelFields(models.Model): + _inherit = "ir.model.fields" + + @api.depends("field_description", "model") + def _compute_display_name(self): + super()._compute_display_name() + if self.env.context.get("technical_name"): + for field in self: + if self.env.context.get("technical_name"): + field.display_name = field.name + return diff --git a/sheet_dataframe_process/models/partner.py b/sheet_dataframe_process/models/partner.py new file mode 100644 index 0000000000..1a74eb89d1 --- /dev/null +++ b/sheet_dataframe_process/models/partner.py @@ -0,0 +1,19 @@ +from odoo import models + + +class ResPartner(models.Model): + _inherit = "res.partner" + + def sheet_dataframe_import(self): + self.ensure_one() + transient = self.env["sheet.dataframe.transient"].create( + { + "config_id": self.env.context.get("config_id")["id"], + "partner_id": self.id, + } + ) + action = self.env.ref( + "sheet_dataframe_process.sheet_dataframe_transient_action" + )._get_action_dict() + action["res_id"] = transient.id + return action diff --git a/sheet_dataframe_process/models/try_file.py b/sheet_dataframe_process/models/try_file.py new file mode 100644 index 0000000000..230fb4246c --- /dev/null +++ b/sheet_dataframe_process/models/try_file.py @@ -0,0 +1,85 @@ +import base64 +from pathlib import Path + +from odoo import fields, models +from odoo.modules.module import get_module_path + + +class TestPolarsFile(models.Model): + _name = "try.file" + _description = "Example files to ensure your configuration match with cases" + + config_id = fields.Many2one( + comodel_name="file.config", required=True, ondelete="cascade", readonly=True + ) + name = fields.Char() + template = fields.Binary(string="Fichier", attachment=False) + + def _populate(self): + def create_attach(myfile, addon, idstring, relative_path): + with open(myfile, "rb") as f: + vals = { + "config_id": self.env.ref(idstring).id, + "name": f.name[f.name.find(addon) :], + } + self.env[self._name].sudo().create(vals) + + self.env[self._name].search([("template", "=", False)]).unlink() + paths = self._get_test_file_paths() + for addon, data in paths.items(): + relative_path = data["relative_path"] + idstring = f"{addon}.{data['xmlid']}" + if self.env.ref(idstring): + mpath = Path(get_module_path(addon)) / relative_path + for mfile in tuple(mpath.iterdir()): + create_attach(mfile, addon, idstring, relative_path) + action = self.env.ref( + "sheet_dataframe_process.try_file_action" + )._get_action_dict() + return action + + def try_import(self): + self.ensure_one() + transient = self.env["sheet.dataframe.transient"].create( + { + "filename": self.name, + "file": self._get_file(), + "config_id": self.config_id.id, + } + ) + action = self.env.ref( + "sheet_dataframe_process.sheet_dataframe_transient_action" + )._get_action_dict() + action["res_id"] = transient.id + return action + + def _get_file(self): + # TODO Clean + if self.template: + return self.template + module = self.name[: self.name.find("/")] + relative = self._get_test_file_paths().get(module) + relative = relative and relative.get("relative_path") + if relative: + path = Path(get_module_path(module)) + path = path / relative / self.name[self.name.rfind("/") + 1 :] + # myfile = path / self.name + with open(path, "rb") as f: + return base64.b64encode(f.read()) + + def _get_test_file_paths(self): + """ + You may override if you want populate files in your module + returns: + {"module_name": { + "relative_path": "tests/files", + "xmlid": "file_config_xml_id"} + } + } + """ + return { + "sheet_dataframe_process": { + "relative_path": "tests/files", + "xmlid": "file_config_contact", + } + } diff --git a/sheet_dataframe_process/pyproject.toml b/sheet_dataframe_process/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/sheet_dataframe_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/sheet_dataframe_process/readme/DESCRIPTION.md b/sheet_dataframe_process/readme/DESCRIPTION.md new file mode 100644 index 0000000000..b2eedffec0 --- /dev/null +++ b/sheet_dataframe_process/readme/DESCRIPTION.md @@ -0,0 +1,27 @@ +From an imported spreadsheet (xlsx), this module allows to transform file data in +Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference (column names, number of columns, dirty paging) but contains data related to same concepts. +Then you want apply them a common process to automate things. +For that you need to transform/arrange data to the same way + + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database but in memory: you don't need to iterate on each line to perform operations +- the operations are powerful: filter, add column resulting from calculation, select a subset of data + + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project diff --git a/sheet_dataframe_process/security/ir.model.access.xml b/sheet_dataframe_process/security/ir.model.access.xml new file mode 100644 index 0000000000..eabce61120 --- /dev/null +++ b/sheet_dataframe_process/security/ir.model.access.xml @@ -0,0 +1,51 @@ + + + File Config + + + + + + + + + + File Field + + + + + + + + + + File Partner Field + + + + + + + + + + sheet.dataframe.transient + + + + + + + + + + try.file + + + + + + + + diff --git a/sheet_dataframe_process/static/description/icon.png b/sheet_dataframe_process/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..d1d2b572970f1b1180a2fef75dc439bffb034de6 GIT binary patch literal 2605 zcmV+|3exq7P)^6 zrc5)2DbtK$$}~ea1ozl_)bf6v8Nl{M&h_PP``crr>kS+4>^^UL?T7`y`ZD+SN@ug1 z`o=|DGWf~4+z>oia!40tJI|P%esa1!ST)Y)2@=vp4@dYzgghq1!^`z{0eEtc{gzV) z;Oq-c0A4<1^>bJ?#j|>fr^82HYiA6!Fk@t}(?`B^(CP^iON)KgW$rgmWWQUR3E=Nf zIcALtr0I=hq=mAZowQ^-c_#kcxTeH^=#G%<@{>0@=p*$k0D{2kDW2-T=ALY10o1H< zcsb&W+Zg~{eyY06eWi_-?>lQ$pzQtpCbw>)l{@}?OKRUsLdqyR(ZJR=>PZ4?XSfA1 ze39b;V6rW|$|eqCx%F^ix7+0>XllS=E@EEm?`sIJJ z1K2pb>y^1(iBFO|Wt=1itUG|Cn>t?;pz}NQsBQq~MSflYNubomseF1? zVHZXwj*Y0`C$qZaj^PEoeKPBvQ&|AsSm@d?v-?Ur4WP|SwRxjI$<=lmKu?go(?tyt zWgBL8ZDxs? z8)eJ?Xs>P31L$wqFCMh+xnu;ee^sm1&?ihIi0DQ_PZo(Y`p5c>Zk;zo0J!K#bhDH0 zJju$PZqeHeyv>lj63|9jOT@)=`l16Txy#`pqbL)_K8@Ks^TQ)IH>lieQK9ekd9FZ+ zcxYS-_E6~&K3k&}KPHHwcK=GjBxg8c!a$2JJrLVr$v-0q5kRSplT`ZnoH){A+wtr( z&5X?`eDG+y%CIK4Zu6lWKZmELdRCTtqI%Op*+eV1xuOezeGp3cf`eK1i9Ns*KP+=^ zt8^;6`FobG+`|K6WIs6m3nM7>iN;0XYi;>myu4(8CU42BZd4nk}v4sDbv9eEZj%qj3ovOH>- z%TLb!FyHAX=a2F4UD=wF2UWo@zsjt+X#kLC;;T~O_)K)zjiF#f{_uY**PhQ9|-R)=@Z<}Dd-hdN{#u*c;Eh2tpKE0f9G7oy`5fi_UJgpi_dRmJiFfpV0)!=eI#=m6YI;}>&xW{MLd`k9KG{!=WQn~ zd5vZ_wQyG+fblupzkiULGS8mN#tRMx!1=pOl!P)dN!m+t7(~Ljuazkwi(ywkUG*)% zAbeS0E8FyW&f)qD06UkrMyV_1iC9=(ONX3Xy1XfwB zn7Aw-VrcVaPw%tc?4$u4e)ewEj@g^TQnX1@8X%7Z`X;j1&ghwx%LyV;o*w|PcSWmo zSZb6xR7WuKgaAk)=KRT550Zwqg&B-^q97l=WA^5-6m3C}2FN1;pwJR3wDczal?ox> z%*$sLdVeKNrT9}Z|B#;WkPw5i4F{}ud#Ef{-1S)7cuP23m@*=UrvLv$49fOhHA;<6 zAGxT|KTw8#R=!$_aquX#g!Ggc;0gI0(S3qA3$w08zx~TBAbgZi*_Fi#$|QtRV=8j< zK(gn{rrCoH8}Gz^=!C1~=|CLH&a^Pnp&h$PSUtt_v-z&;9WXAT5;ToB3mFTjYEh7co9g=o1Eq!?Z@dAQEMH z0an#9ZC&8}R|9LH#d%}=CmL9-)$$ZV8)Z9cGmq7?*{ty1k`DDlD3fzJAcnIS#Xjv_ z=S*AvnFCE%nsro?b<9#*u&T)C2@*?+eTY=m0O&|j`CwFfHiI~nEuG+% zlF^k#egJP)cAjq5XJ-i0MGcilxfB=WtuTenP&VK(M1O$P%M+& P00000NkvXXu0mjf2J`yU literal 0 HcmV?d00001 diff --git a/sheet_dataframe_process/static/description/index.html b/sheet_dataframe_process/static/description/index.html new file mode 100644 index 0000000000..55197dcb66 --- /dev/null +++ b/sheet_dataframe_process/static/description/index.html @@ -0,0 +1,453 @@ + + + + + +Sheet Dataframe Process + + + +
+

Sheet Dataframe Process

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

From an imported spreadsheet (xlsx), this module allows to transform +file data in Polars dataframe and process them according to rules in +order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update or import

+

Typical use case:

+

You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way

+

Why dataframe ?

+
    +
  • a dataframe is a kind of in-memory dataset on which you can operate
    • +
  • you can operates on your entire dataset a bit like with a database +but in memory: you don’t need to iterate on each line to perform +operations
    • +
  • the operations are powerful: filter, add column resulting from +calculation, select a subset of data
    • +
+

Why Polars ?

+
    +
  • performance: code in rust
    • +
  • environment consideration
    • +
  • dynamic project
    • +
+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/sheet_dataframe_process/tests/__init__.py b/sheet_dataframe_process/tests/__init__.py new file mode 100644 index 0000000000..d9b96c4fa5 --- /dev/null +++ b/sheet_dataframe_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/sheet_dataframe_process/tests/files/4_fields.xlsx b/sheet_dataframe_process/tests/files/4_fields.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..48351c2a9f88525249f1431861403031786af0ee GIT binary patch literal 5391 zcmaJ_1z42r(q3ZeTuNyql-`d9=}VI+%uXg7?|V$92^`#S3Sf4a82-0zfE0X4sN_W zSNDpf4)rcD!Q*|uD2`LXMQNPeqGnaaW+nmpb`~d^y2$))#NmlN8HgaZrYp|nQQ!r% za7opPYYEJzViKuq5Q*NxW)V=_J$T{b!z(nL_o$at$1@<&+b1$&uT{jwTI5FD?PFNK)LNk#vvJK8n2W zAT955i4<1IJ|{<YDbs$Nu0uapcc|06U0kz~Cv>i7XWl|A{jYbo^ngzc zWrldnW`{p69u^Esj{EM92EZ&HFbUx=H~0Vr8{`&)@)S$dh7(n5j~p$6sOr(%4bQYl zRL`H{_lSjtePbXnPP-KzizF7jvm;HQ7}ve@Ag|h?jw>v*_$5=k@UXQB*>31y=C?Av zIn&rD2mXOJkbyXZ{z9PLtFdCnH`++qV05%P)Ng2nE6r z382n2E}wn;(%qC0a&bIPKBdfISH6H;1{czLX0E?8@JDLBIIXk_%t7l0cFZ@ zcc}x7-%TzB$*T*5%aGJpIrqj>E}oXMmTdocl3mb8;Ev}Cg)VhOq@R0B#TNBDP!r~P zh){>F?pl5}7sw_Un-?e?WqoFueZgWbIsueWy1$&4+qRB2up5@?5{2MlI?miAMw=2->>Kkj!gUJJyE31A;CZ$2llo27w5K5WdoeYV)vR#$S9+?HWVQL)7Q_2A~`~4DcZ%I?m z5QVU&erC}&B*D)`VrbhbL)H{Su-CwuSRvrZHASADhx!OmT?(axJIP^J19|&4CNr;B zA*AXsRk;-Rtqd(NR+<*3{L+JM2E)ZNdz5*V5!B?agn4{WXl2609C9##u)t;kj$$n# z$3H5(nV<|=MUH&xNR4z^O0|uVAi2+4on|K*4iG$2(3P!Vb!dH3a&(cDWeSOBDpcK~1&4Eb$mIA`hjp{e0pJZ-8 zMyMjE4Go^ooUKec(vO^wn&Wkl*Ne1tR0$m*bFXoY(dfSH3xR;p8k={v}r750|s5BWN?G_lNk?cX-hTcS9c*+*dsn82=rR!@vd|}Rq5qUmc$dn z)lkju=ud#Y@hcCy*}!1#ZoEIg{<^|;i=Ht&Uj&71 z(_ruP;wM@{`movr1(x~YT?UP^cV?|ql z-m~IIlUt7kIjbgOp50<w=Z%aT0yQcdIu8V?D>qOWOSB6-AG`OyVLTCBd1PdAX=+%O@&_Xm|Dc0h>6pM z-ZL4~r8o}J{n11UL&_Nsb3F+Hquy*?#nmP|#U^}Kc-#a+yQYd8b5ugeA_cJDv*ohU@-K84`8=rribqL`Z0%*sM{oLvo~ifNse@B1^=*mf5%l6SmEc$o z$j5gvBd6Z+?CbWj!#8xm=c3k_=nV1{XWIQLT09c~m60lvq(WAH{=86mQY25>q@eZ} zz6R{i&tE)eJ*IKjVTM3y(zW2Fz_S*La9Gc3Sk$=-TE0_&8x$>$8()5mvrWu{9PWMgG@JqFrqNS%YU)f}8*P=ru* z2Tk{5JvJc+GMZKy2X`1WU#NPNrn?vQ7mts=b9K;`q;r6EC4ESgt#E!|_Cb&3>)4nM zpI&2O3w`U%x_#rufXeZO7n6^KEf(ySKM8!Oo7=Zia z=9S{p(`WMRUsSJ;9`sjjfCQ2X)LV;-9=2*49&D zpi!WR#}Ex%S;R>Zw=kY_t)t)qW42y3ME!{; z$nl;A;4h1sYEQdQ57$>_T!)0$DLyGhj?r-x7jt{Sss~`ZB@QYw<&7`Ns^v~nRvZWy zJ70(dTOI!ebNUdY5{Kj6!L6`*@Lj-agRp(0Fy zA<8{EmhcXyNnO{j7MB}3uk@s%AVQnPQnX_dVKrfB8(SXH{nrqCGWDf=Ena2**5HME zEtX{Tt+jIm@tt8Q?k^w0O@YbJvsWgV%CSYX{H@ssE(CbyceY<$NMAM|XOTsr3p~7_ z%7Ju`Ei(w(;cna=<}o*TZi@&+yUJ3yWhs1Sm;eA3ije&?TciFvOa0AjuXDD)Ic}nw z{ZEd2PI2$FM+w^^Mx-T1rtl zkchPaKE*ay4=F^FH;!Ad`#2JuGRwa!OF=7iQzZ{bOj3W$AI#v5dB=tCRPD$tkNAb| zb1N?Byq%zh$h)fVqRwfETD@dJ>*eSMRr4Ht8?VVxsG7@^zgu9{7PC&Uizs&ThOGa97w*<_|H}K%hw6sa`^>+@NHHeGseN z$vPNDQH6x%lqI$RFlT>%*r|-p1I;MS|y0n_q3^ch9F=`Y1kU=LK zSer0kzb6%%Vob>y3?QkP8fTeeQr4q}8<)Pz)F&<#<7pCTR3FS4Os?*JPcxUWjd8@Q>YM4nSVuD=yunsv6nNZU^zbZl$|4@o2wWAQSoi4C`AplnF0t!){@WXd>z^>CG1k^ z+Ocib5Jd`+eBcEyX7O+0a?p?U>Kpm77kj)Hrcl?Q^D~d*)$wB0UL&VBXioX z^aA(_cF!iZE2g=M#`ddr{rXpj4`mRMuQ>*(yKZ#gkTf=%O1cc{mJ*_ASx*_}C1+&@!qMd2?_N(I4LMgEw@)0(}H zKAyrOg;bA%?~8NZu0-hEs>Epx&+40xo^(HZn8J?4xTNk7*4t+I32rySvu`M;JVm+g zpVK7Sb+=h~xVgg}e|ipCjn@e#4_jR<#+jYh!A=}|6)kdC=QLrd9?cEjrxqRH#~iX3 z7tSE5J}z-CS57iZYS+li$qwKqB)ZKFKQ9zf>gZoFqYus5z~XB5oPNj*jOa&;=3=xL zEhe`soI$P-J`uIxVOD@cvRxnoamdb$o%d+TVmgyrvteZ4ZuyITJNPJgu0mwFy#I9x z&gAEq|Hyj&=b>Gl0t7ulz4Za=^XGjT$-$l6VNUL*IzG-YH{+lDRA*SDi;=u-ztZOQ z)6UI~8{61)KcG+12w0aM-yyTw zD~h$MSVGlnQt$KWPA_OLY`?tBqx@Riuo^%t862=D$$%!+F!%a1zrM&~x7S{Tx*8+; zCxy!`uc+X&oe7d7dU3uYC%Yd4P1xT ge;b%Uz0Ln~<~3D-s4xKlc&IA~RYQDG-7Ns{e?Z#?9{>OV literal 0 HcmV?d00001 diff --git a/sheet_dataframe_process/tests/files/missing_required_column.xlsx b/sheet_dataframe_process/tests/files/missing_required_column.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d9b277bcecf7c5589b6d2e7a4da2fda67329893f GIT binary patch literal 5330 zcmaJ_bzGGD@?N?XRwSfLx*I`Cx)+eHWmmepV?k*|T7e@7A|N41h;+AfEwz-uAtV&( zyX(2X3)kO0XWq}oAMecj&dl?jd1kcLFfd60xVX3g(pr!a;D!(&|69029X)t?P~W95 zS~c7G2%qi+#Bdx7&&%NEzWuDO{FyfrVlXKqBZzy_8#&gnY^xBp={E8X zUq@aa7k3A+i;Dw~uajfEZm%<(kGS!e6LZ0d+uWWi4Z|SeIbfc+us8FCBAG)BY0rLI z-qivLw19n9{`Lzog2jeRKjVcghfSJR!(9D=Hmep%?N2-rvz^;;Yl#g0FN_9Q>Bh1H zJeD(qTk{9`gAd32_eO%C){hxP2o`Jou!L*n=R@<9i!=t4)T<7itV79b&|8d8b%@n3 zUJ`UlL`0s@5t^kjK8qi^EzGtpL#Uk4vG6$WgJU&UWJF;ZL!#)QjrpCOi2kgT5`$Tb z`1Jk25L-}Rf>Cb)mi@=kLf#plo+OE7kU;qoE<%#xm_0+sQI4nf(+H@Ga1db}AC=d0b$6x)=D(V`ll|2u7e6S5vUKBMJa? z%7PCL*uZMw2|^M~d$65Lqc?CL&5Seh!$yo4sb)5H*j6=}@M`mB&(8ZR|2`t2NZ4~= zTN;<&9zpR=N;s(`KBs_6)}Wkr%N%)9`*zqnM3x9WG2t5P>FcOR)0G0+!J}BR@9}o- zg_wPrSO|TjDfCR1xTeCjE0JvexR|AA^E^E#zlYEh-yIBIXpPFa@Rg2#+v|9bD9=mm zUIbzX^36&phj0`wR4~Hw3X*flY$ZO9C8eUYn3vnUiq^LinHI47sKz@buy1Q2A#>3I z{G9*=8*`y5pHnmdz!l@4U_}FvqSsiV3Njq}w<%?cf`FN2U9LbZaL6v*GQr)amQ^l_~z${|^`Lld1Hol&u z;78Cr(EUToCbI$Q94iP{5<5b5l(OtyEBXGjp6QseSK**o8QGVVJny-svBFgFIikct zEUDj^b&ZJ$a)&TF{%v{*6I&2FLfOgTjB z4B;~*zZ3r5=Iu{W6rknQxnt6hHLwKuPk^Slu7e)7P^hN|@2?+z6t)`;;LdO{VkEYS zkA@3Nz4jH{I7)D{ASU$jFo}8eGexx&g$SVji^`S2i(4x5j1OWhb;NGhhzm-3Jb9X@ z+4kj&c(r#1L0e4ZPP{3+O<@2CawPwe*bPr4uVG>406sDbBmaR$aOQEORA+Op0-;bN z0N{V95M=LH>n3q~>cOn|z5~k_)aAi!)mxjuCaxI0kfPh|GoP8v1e(T1ylBRO6|hI| z3&)mFlaGeFswZKdGBOp|4(BBMAXun(a>t))B`#O*fAn2g#pPw?w+SzBC#`CD>utI= zm)Tq_n7AmQyMHaJ9 zev4&lLvj4#OmOc8xh-ba`=%8-LK+9cEJs00-)Fira;hJVtYfUBlP{a}S=gD4=njb9 zaLa`2>gg1*$w+ijo!z3~-Il9PMrY>Kk70T1X=UM8bGKx=JXHGC^dy1xiG(8Q%hkcT zg;R4EI%E9@d&6P+_1sHdttGTiiLABy@?Fh@nav{YU82Wfx>Xh2n3FVRGXrN1RTIGG zBbo4iKT*EOxvTks^~ZFS8D8`Fw(VLkdI+QZ-n0WTUC)87v-K+xsVCRU42N znyE{6+}lM^QD@kxR-_wYHz*;k9lfkS;V zf}z`6LR~duI*lwIEf!nTehNn+|rpbVd^`jo(^Mu=R zYt*weG*pY_oXVczn8Eq*oFvwIKOSuYKwS`*osV**9I|vN8RfkMrYH0`k4hC%%7)R) z{Qj8g|AFOff)DQW%8A&C?ItAWd`EV_6OXh+VZV)sGM$=4o0ckOxv+%sB$H&uUXFm- zRV$?PwR{?v!Im)_p1x&jIzdsym!MRa3O~3W`r@s@b{*4n%v~dWWkJ{wu!RvBeb=(!h*~5ZzbS^F(zA6t9e-P#gMz+ zxt&n<{wCj-DGP(#v0OHp_O%eS;snFfW0+!Tu3Vu73m5$AbzTS>stRQpUhv9b0sxW7 z&iVIj?A~8h=r7)Slau|$VUsi*esS1~yNbu1Dmd10VvTXK zhmx!C8dn{P8S#MIoEEj7XT=*+48*`w{M!g`A6 zmc^TZKlplx@CMy!lLSXViQpwpUICmVr;|0QHEBkgX{Nq%*dHJfAG`^RC$xM11TJ{+ zQtjaFbLPW?+DwJg@`d?8bW~)tzMc4wBK>ChN1jUhZ^B_~Y@MdK{q|MK!z9b0}&3`NAM)F!GTu zf`nECgJM}5ygY_Wx5s;|W%-doDT`nX*8ntq_MmW9UCBr(!?@&ugRpvvj%We!xYs5r zu2fCjs8o)^i_zY%Cb7Mun(__|4~h_JBO7SjY#81*2q^6RY|^30`7~$FhpjiVWs&WZ z1iYb2x+$a~F7q+vA{%MK#p6wG1CnP``ZZrEr-WHZby=VS;%=jgU@-du4*b&diU<{w zyM{kGMvx9yB6--~gM|cn2gJo%+ug<0gV)-{9r}yBgUqyGNFH`<@VrTW^I?=o3s5+~ zJz6KOtJZke@UH5ZxAoaE8HM8(ZQ$pxIb$Bp_jrHkZ|=xY>GWeJ(LbV9HvcL|hUed# zb}4)i2Nke^ax=){@YyDrcbe*$JNS-QqdQ?ye4$j4|D-zHWvsw25{PkEFicljsm#HQ z|IgPv72xyOyP>RB6bO8-{!D0{@X^FZR6?3K9rwiJ^laH7{uJ%7kJ!nfOUaYrWdof! zG#+o?M@X^NvC;JCG>U>rMKtaRjcIP3vi@}X{%tOPi(sZK219klnkmJ>lFa=J<#5wn zOM~X#ZWW6vD+02fJj|-2=)576hd7t&YF?;)8f5zCN2pdyr#MAfWaaKtsS_rIYcJ%3 z7lGE)ti?mG7yJ`Bm5{EoxsfUK132uk?m3l2al)soIM3urhHNEI#UaIeg!O&*yz?#s z*cr?wNNSy4?R2xc!oLWilbcwnZH&qP{3`!;5tXH{qMf)l$PCT>BXEgtX35Ic6_upy z=Nc|;sA2T%u6{?g2K#&Z6l|xL-g2|IXl6_MfaC&eYAP{(%@zP6v6bYDS%1f7h?IRQ zO$&~l2JEduv;f~6zMVJGsr}+1OP?G>kq=5Fw1JCOx_Rg1XfiAjD3^wNU&xm3jfoZY zPg)@Y1^ZXyo264+Z%6kkb^>}4g9oy(=sJ%6d+ijhxWu16qmM+^NpvXXwjGQGS;1+| zmUTwMy<-$!V=j(PD!w%?3L8oe}P<1CuX7XeTkqpc-SW@8UBvTNcQ5KAwKd#)} z^79rQE+7vMQg^kaH==1~`l$ZYq z?mIWbX6@zS>EiS&azh{R+xSQ$5%Yz(GjKheq|uMDVh{9=pD)y)dARgw#0CX0h3_Uj zqkE`1CUqhIo_L1D{u3`JJAj+$7Av>QMS++~YwwaJZN%#}Y_8AVQ>IKEhNrObRZb95;|q!cbf0_A*+sgx-YCOyr#t#z66t{Vm}OwOf$Iy`Kv% z3;|AWf^FdBD8CSsfY5a0Tkj)3e}8CEewoO-ID0~!JuUS7T%jIjzbK~OpjJCQY4cvW zZQaYZ^;U{8XB_E-`VK!4+;Z-+K+Ng6%YfVoXlAvC(MohnKr^p7{VOL(_WM?eT>XUm zzHvZpb&uRW>P9ZwQ3s({36-6bp=xsF?&8G_qM{LEkHhkBIbd+;;XAQ>w{lPevzd$n zd)9md6*3;vrf3DN%8YH_!mlq5a&N}#E_Dulvs$wg3lh)k8V}c@>tlYb!a5|k;~(nK zd3&zFV#rlSM5+wfWp-X>spm&e4^b!k@|k20yPiRV!7``K%cPYdQ}<80B0{27;rk4} zOOj@)uU-b)ehwV-k3K7Jl-D4Sv}_SJ=>@b78NQy=2bfRR8GXo?qZ#oDHi1zYH!@tv zA7C#jSE*x*d{~iQenWG7NquQ|RF+5fv#J&mbo*gg(C$MzH0j#ex^IGpV)Gq7yHT1N z^yq&oTy6Nogq?1W6E8~O1Q&~#h?lR;9B>b6?dvGMwD3R0LnUeXLG5h_5_scSs3f7G zlLD@{M>iKHsO`~z%Z**q?}|4^X4F3AT3C^f`fa=NyYkKP1T|G(i#c)vM=D>>*uSgZ z|8C)C zK}9vUYgs1zAFb|p^_ztSRduf=>edbQf7jsOwQn9@)FHi=W@JvU`|N*8(eDOsLhHW` eq$A(vf1G)3H7sPB004aC6^iU3lE?uS0QeWTiq)3@ literal 0 HcmV?d00001 diff --git a/sheet_dataframe_process/tests/test_module.py b/sheet_dataframe_process/tests/test_module.py new file mode 100644 index 0000000000..64b541bdd3 --- /dev/null +++ b/sheet_dataframe_process/tests/test_module.py @@ -0,0 +1,23 @@ +from odoo.tests.common import TransactionCase + + +class TestModule(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env["try.file"]._populate() + cls.file_records = cls.env["try.file"].search([]) + + def test_missing(self): + wiz = self.get_wizard(self.file_records, "missing_required") + self.assertEqual(wiz.missing_cols, "['street']") + + def test_four_fields(self): + wiz = self.get_wizard(self.file_records, "4_fields") + self.assertFalse(wiz.missing_cols) + + def get_wizard(self, file_recs, file_str): + action = file_recs.filtered( + lambda s, file_str=file_str: file_str in s.name + ).try_import() + return self.env["sheet.dataframe.transient"].browse(action.get("res_id")) diff --git a/sheet_dataframe_process/views/file_config.xml b/sheet_dataframe_process/views/file_config.xml new file mode 100644 index 0000000000..d44080633c --- /dev/null +++ b/sheet_dataframe_process/views/file_config.xml @@ -0,0 +1,94 @@ + + + file.config + +
+
+
+ + + + + + + + + + + +