diff --git a/polars_process/README.rst b/polars_process/README.rst new file mode 100644 index 0000000000..45e7cd0438 --- /dev/null +++ b/polars_process/README.rst @@ -0,0 +1,115 @@ +============== +Polars Process +============== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github + :target: https://github.com/OCA/reporting-engine/tree/18.0/polars_process + :alt: OCA/reporting-engine +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-polars_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +From an imported spreadsheet (xlsx) or db query, this module allows to +transform data in Polars dataframe and process them according to rules +in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database + but in memory: you don't need to iterate on each line to perform + operations +- the operations are powerful: filter, add column resulting from + calculation, select a subset of data + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/polars_process/__init__.py b/polars_process/__init__.py new file mode 100644 index 0000000000..aee8895e7a --- /dev/null +++ b/polars_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/polars_process/__manifest__.py b/polars_process/__manifest__.py new file mode 100644 index 0000000000..bbdd05fe25 --- /dev/null +++ b/polars_process/__manifest__.py @@ -0,0 +1,34 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Polars Process", + "version": "18.0.1.0.0", + "summary": "Allow to create a Polars dataframe from file or db query and " + "process it according to rules", + "category": "Reporting", + "license": "AGPL-3", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/reporting-engine", + "maintainers": ["bealdav"], + "depends": [ + "contacts", + ], + "external_dependencies": { + "python": [ + "polars", + "fastexcel", + ] + }, + "data": [ + "data/action.xml", + "data/demo.xml", + "security/ir.model.access.xml", + "wizards/df_process.xml", + "views/dataframe.xml", + "views/df_field.xml", + "views/df_source.xml", + "views/menu.xml", + ], + "installable": True, +} diff --git a/polars_process/data/action.xml b/polars_process/data/action.xml new file mode 100644 index 0000000000..dede8792e1 --- /dev/null +++ b/polars_process/data/action.xml @@ -0,0 +1,10 @@ + + + + 🐻‍❄️ Populate file polars example + + + code + env["df.source"]._populate() + + diff --git a/polars_process/data/demo.xml b/polars_process/data/demo.xml new file mode 100644 index 0000000000..94411d3d8a --- /dev/null +++ b/polars_process/data/demo.xml @@ -0,0 +1,44 @@ + + + + stop + import_preprocess + + + + + + + Country + + + + + Name + + + + + + Street + + + + + + 2nd Street + street2 + + + + + Date + + + + + + Colour + Color + + diff --git a/polars_process/models/__init__.py b/polars_process/models/__init__.py new file mode 100644 index 0000000000..96b8121bf9 --- /dev/null +++ b/polars_process/models/__init__.py @@ -0,0 +1,4 @@ +from . import dataframe +from . import df_field +from . import df_source +from . import ir_model_fields diff --git a/polars_process/models/dataframe.py b/polars_process/models/dataframe.py new file mode 100644 index 0000000000..983291d882 --- /dev/null +++ b/polars_process/models/dataframe.py @@ -0,0 +1,39 @@ +from odoo import fields, models + + +class Dataframe(models.Model): + _name = "dataframe" + _inherit = "mail.thread" + _description = "File Configuration" + _rec_name = "code" + + model_id = fields.Many2one( + comodel_name="ir.model", + required=True, + copy=False, + ondelete="cascade", + tracking=True, + ) + code = fields.Char(help="Allow to browse between several identical models") + rename = fields.Boolean(help="Rename dataframe fields") + action = fields.Selection( + selection=[ + ("display", "Display"), + ("dataframe", "Dataframe"), + ], + default="display", + tracking=True, + help="Some other behaviors can be implemented", + ) + on_fail = fields.Selection( + selection=[("stop", "Stop"), ("skip", "Skip record (TODO)")], + default="stop", + tracking=True, + help="What should be the behavior in case of failure regarding constraint " + "fields (required, format, etc)\n\n" + " - Stop: stop the process by raising an exception\n" + " - Skip record: current line'll be ignored from the next process", + ) + field_ids = fields.One2many( + comodel_name="df.field", inverse_name="dataframe_id", copy=True + ) diff --git a/polars_process/models/df_field.py b/polars_process/models/df_field.py new file mode 100644 index 0000000000..45f41e3855 --- /dev/null +++ b/polars_process/models/df_field.py @@ -0,0 +1,32 @@ +from odoo import fields, models + + +class FileField(models.Model): + _name = "df.field" + _inherit = ["mail.thread"] + _description = "Configuration de l'import de champ" + _order = "field_id ASC" + + dataframe_id = fields.Many2one( + comodel_name="dataframe", required=True, ondelete="cascade" + ) + sequence = fields.Integer() + field_id = fields.Many2one( + comodel_name="ir.model.fields", + ondelete="cascade", + required=True, + domain="[('model_id', '=', model_id)]", + ) + model_id = fields.Many2one( + comodel_name="ir.model", + related="dataframe_id.model_id", + readonly=True, + ) + name = fields.Char(help="Name field in the source file (spreadsheet)") + renamed = fields.Char(help="If specified, renamed in dataframe") + required = fields.Boolean( + help="Prevent to import missing data if field is missing in some records", + ) + check_type = fields.Boolean( + help="Check data type is compatible", + ) diff --git a/polars_process/models/df_source.py b/polars_process/models/df_source.py new file mode 100644 index 0000000000..7c441b1874 --- /dev/null +++ b/polars_process/models/df_source.py @@ -0,0 +1,100 @@ +import base64 +from pathlib import Path + +from odoo import _, fields, models +from odoo.modules.module import get_module_path + + +class DfSource(models.Model): + _name = "df.source" + _description = "Dataframe data source" + + dataframe_id = fields.Many2one( + comodel_name="dataframe", required=True, ondelete="cascade" + ) + name = fields.Char() + sequence = fields.Integer() + rename = fields.Boolean(help="Display renamed Dataframe in wizard") + template = fields.Binary(string="File", attachment=False) + readonly = fields.Boolean(help="Imported records from module are readonly created") + + def _populate(self): + def create_attach(myfile, addon, idstring, relative_path): + with open(myfile, "rb") as f: + name = f.name[f.name.find(addon) :] + vals = { + "dataframe_id": self.env.ref(idstring).id, + "name": name, + "readonly": True, + "rename": True, + } + if ".sql" in name: + vals["query"] = self._get_file(name) + self.env[self._name].sudo().create(vals) + + self.env[self._name].search([("template", "=", False)]).unlink() + paths = self._get_test_file_paths() + for addon, data in paths.items(): + relative_path = data["relative_path"] + idstring = data["xmlid"] + if self.env.ref(idstring): + mpath = Path(get_module_path(addon)) / relative_path + for mfile in tuple(mpath.iterdir()): + create_attach(mfile, addon, idstring, relative_path) + action = self.env.ref("polars_process.df_source_action")._get_action_dict() + return action + + def start(self): + self.ensure_one() + vals = { + "filename": self.name, + "df_source_id": self.id, + "dataframe_id": self.dataframe_id.id, + "file": base64.b64encode(self._get_file()), + } + transient = self.env["df.process.wiz"].create(vals) + action = self.env.ref("polars_process.df_process_wiz_action")._get_action_dict() + action["res_id"] = transient.id + return action + + def _get_file(self, name=None): + # TODO Clean + if self.template: + return self.template + name = self.name or name + module = name[: name.find("/")] + relative = self._get_test_file_paths().get(module) + relative = relative and relative.get("relative_path") + if relative: + path = Path(get_module_path(module)) + path = path / relative / name[name.rfind("/") + 1 :] + with open(path, "rb") as f: + return f.read() + + def _get_test_file_paths(self): + """ + You may override if you want populate files in your module + returns: + {"module_name": { + "relative_path": "tests/files", + "xmlid": "dataframe_xml_id"} + } + } + """ + return { + "polars_process": { + "relative_path": "tests/files", + "xmlid": "polars_process.dataframe_contact", + } + } + + def ui_form(self): + self.ensure_one() + return { + "name": _("Dataframe source"), + "res_model": self._name, + "view_mode": "form", + "res_id": self.id, + "type": "ir.actions.act_window", + "target": "current", + } diff --git a/polars_process/models/ir_model_fields.py b/polars_process/models/ir_model_fields.py new file mode 100644 index 0000000000..14d2039c81 --- /dev/null +++ b/polars_process/models/ir_model_fields.py @@ -0,0 +1,14 @@ +from odoo import api, models + + +class IrModelFields(models.Model): + _inherit = "ir.model.fields" + + @api.depends("field_description", "model") + def _compute_display_name(self): + super()._compute_display_name() + if self.env.context.get("technical_name"): + for field in self: + if self.env.context.get("technical_name"): + field.display_name = field.name + return diff --git a/polars_process/pyproject.toml b/polars_process/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/polars_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/polars_process/readme/DESCRIPTION.md b/polars_process/readme/DESCRIPTION.md new file mode 100644 index 0000000000..9d5d69103f --- /dev/null +++ b/polars_process/readme/DESCRIPTION.md @@ -0,0 +1,26 @@ +From an imported spreadsheet (xlsx) or db query, this module allows to transform data in Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update or import + +Typical use case: + +You receive files from your vendors and these files have many difference (column names, number of columns, dirty paging) but contains data related to same concepts. +Then you want apply them a common process to automate things. +For that you need to transform/arrange data to the same way + + +Why dataframe ? + +- a dataframe is a kind of in-memory dataset on which you can operate +- you can operates on your entire dataset a bit like with a database but in memory: you don't need to iterate on each line to perform operations +- the operations are powerful: filter, add column resulting from calculation, select a subset of data + + +Why Polars ? + +- performance: code in rust +- environment consideration +- dynamic project diff --git a/polars_process/security/ir.model.access.xml b/polars_process/security/ir.model.access.xml new file mode 100644 index 0000000000..7bb63a8d88 --- /dev/null +++ b/polars_process/security/ir.model.access.xml @@ -0,0 +1,41 @@ + + + Dataframe + + + + + + + + + + Df Field + + + + + + + + + + df.process.wiz + + + + + + + + + + df.source + + + + + + + + diff --git a/polars_process/static/description/icon.png b/polars_process/static/description/icon.png new file mode 100644 index 0000000000..d1d2b57297 Binary files /dev/null and b/polars_process/static/description/icon.png differ diff --git a/polars_process/static/description/index.html b/polars_process/static/description/index.html new file mode 100644 index 0000000000..0113640e53 --- /dev/null +++ b/polars_process/static/description/index.html @@ -0,0 +1,453 @@ + + + + + +Polars Process + + + +
+

Polars Process

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

From an imported spreadsheet (xlsx) or db query, this module allows to +transform data in Polars dataframe and process them according to rules +in order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update or import

+

Typical use case:

+

You receive files from your vendors and these files have many difference +(column names, number of columns, dirty paging) but contains data +related to same concepts. Then you want apply them a common process to +automate things. For that you need to transform/arrange data to the same +way

+

Why dataframe ?

+
    +
  • a dataframe is a kind of in-memory dataset on which you can operate
    • +
  • you can operates on your entire dataset a bit like with a database +but in memory: you don’t need to iterate on each line to perform +operations
    • +
  • the operations are powerful: filter, add column resulting from +calculation, select a subset of data
    • +
+

Why Polars ?

+
    +
  • performance: code in rust
    • +
  • environment consideration
    • +
  • dynamic project
    • +
+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/polars_process/tests/__init__.py b/polars_process/tests/__init__.py new file mode 100644 index 0000000000..d9b96c4fa5 --- /dev/null +++ b/polars_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/polars_process/tests/files/4_fields.xlsx b/polars_process/tests/files/4_fields.xlsx new file mode 100644 index 0000000000..815bedfc2a Binary files /dev/null and b/polars_process/tests/files/4_fields.xlsx differ diff --git a/polars_process/tests/files/missing_required_column.xlsx b/polars_process/tests/files/missing_required_column.xlsx new file mode 100644 index 0000000000..c6f9f226f4 Binary files /dev/null and b/polars_process/tests/files/missing_required_column.xlsx differ diff --git a/polars_process/tests/files/wrong_date.xlsx b/polars_process/tests/files/wrong_date.xlsx new file mode 100644 index 0000000000..52e5c82136 Binary files /dev/null and b/polars_process/tests/files/wrong_date.xlsx differ diff --git a/polars_process/tests/test_module.py b/polars_process/tests/test_module.py new file mode 100644 index 0000000000..e7bafcad7e --- /dev/null +++ b/polars_process/tests/test_module.py @@ -0,0 +1,45 @@ +from lxml import etree + +from odoo.tests.common import TransactionCase + + +class TestModule(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env["df.source"]._populate() + cls.file_records = cls.env["df.source"].search([]) + + def test_missing_required_column(self): + wiz = self.get_wizard(self.file_records, "missing_required_column") + comment = sanitize(str(wiz.comment)) + root = etree.fromstring(comment) + self.assertEqual( + root.xpath('//div[@id="missing-columns-data"]')[0].text, + "['Street']", + ) + + def test_four_fields(self): + wiz = self.get_wizard(self.file_records, "4_fields") + comment = sanitize(str(wiz.comment)) + root = etree.fromstring(comment) + self.assertEqual( + len(root.xpath('//div[@id="missing-values-in-name"]')), + 1, + "Missing value in Name column", + ) + self.assertEqual( + len(root.xpath('//div[@id="missing-values-in-street"]')), + 1, + "Missing values in Street column", + ) + + def get_wizard(self, source_recs, file_str): + source = source_recs.filtered(lambda s, file_str=file_str: file_str in s.name) + action = source.start() + return self.env["df.process.wiz"].browse(action.get("res_id")) + + +def sanitize(string): + string = string.replace("
", "
") + return string diff --git a/polars_process/views/dataframe.xml b/polars_process/views/dataframe.xml new file mode 100644 index 0000000000..90f0b9e8b0 --- /dev/null +++ b/polars_process/views/dataframe.xml @@ -0,0 +1,69 @@ + + + dataframe + +
+ +
+ + + + + + + + + + + + + + + + + + + + + dataframe + + + + + + + + + + + dataframe + + + + + + + + + + + + + + + Dataframe + dataframe + list,form + dataframe + + diff --git a/polars_process/views/df_field.xml b/polars_process/views/df_field.xml new file mode 100644 index 0000000000..d5161e2e98 --- /dev/null +++ b/polars_process/views/df_field.xml @@ -0,0 +1,27 @@ + + + df.field + + + + + + + + + + + + + + + Df Field + df.field + list + df-field + + diff --git a/polars_process/views/df_source.xml b/polars_process/views/df_source.xml new file mode 100644 index 0000000000..6eab51c5a3 --- /dev/null +++ b/polars_process/views/df_source.xml @@ -0,0 +1,75 @@ + + + df.source + +
+ +
+ + + + + + + + + + + + + + + + + df.source + + + + +