[18.0][ADD] sheet_dataframe_process module: convert file to polars dataframe and…

requirements.txt

@@ -0,0 +1,3 @@
+# generated from manifests external_dependencies
+fastexcel
+polars

sheet_dataframe_process/README.rst

@@ -0,0 +1,115 @@
+=======================
+Sheet Dataframe Process
+=======================
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github
+    :target: https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process
+    :alt: OCA/reporting-engine
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-sheet_dataframe_process
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+From an imported spreadsheet (xlsx), this module allows to transform
+file data in Polars dataframe and process them according to rules in
+order to:
+
+-  filter data and display
+-  obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to
+create/update or import
+
+Typical use case:
+
+You receive files from your vendors and these files have many difference
+(column names, number of columns, dirty paging) but contains data
+related to same concepts. Then you want apply them a common process to
+automate things. For that you need to transform/arrange data to the same
+way
+
+Why dataframe ?
+
+-  a dataframe is a kind of in-memory dataset on which you can operate
+-  you can operates on your entire dataset a bit like with a database
+   but in memory: you don't need to iterate on each line to perform
+   operations
+-  the operations are powerful: filter, add column resulting from
+   calculation, select a subset of data
+
+Why Polars ?
+
+-  performance: code in rust
+-  environment consideration
+-  dynamic project
+
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/reporting-engine/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/reporting-engine/issues/new?body=module:%20sheet_dataframe_process%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* Akretion
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px
+    :target: https://github.com/bealdav
+    :alt: bealdav
+
+Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-bealdav| 
+
+This module is part of the `OCA/reporting-engine <https://github.com/OCA/reporting-engine/tree/18.0/sheet_dataframe_process>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

sheet_dataframe_process/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import wizards

sheet_dataframe_process/__manifest__.py

@@ -0,0 +1,35 @@
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Sheet Dataframe Process",
+    "version": "18.0.1.0.0",
+    "summary": "Allow to create a Polars dataframe from a sheet file and "
+    "process it according to rules",
+    "category": "Reporting",
+    "license": "AGPL-3",
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "development_status": "Alpha",
+    "website": "https://github.com/OCA/reporting-engine",
+    "maintainers": ["bealdav"],
+    "depends": [
+        "contacts",
+    ],
+    "external_dependencies": {
+        "python": [
+            "polars",
+            "fastexcel",
+        ]
+    },
+    "data": [
+        "data/action.xml",
+        "data/demo.xml",
+        "security/ir.model.access.xml",
+        "wizards/sheet_dataframe.xml",
+        "views/file_config.xml",
+        "views/file_field.xml",
+        "views/file_partner_field.xml",
+        "views/try_file.xml",
+        "views/menu.xml",
+    ],
+    "installable": True,
+}

sheet_dataframe_process/data/action.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+    <record id="refresh_file_example" model="ir.actions.server">
+        <field name="name">🐻‍❄️ Populate file polars example</field>
+        <field name="model_id" ref="model_try_file" />
+        <field name="binding_model_id" ref="model_try_file" />
+        <field name="state">code</field>
+        <field name="code">env["try.file"]._populate()</field>
+    </record>
+</odoo>

sheet_dataframe_process/data/demo.xml

@@ -0,0 +1,38 @@
+<odoo>
+    <record id="file_config_contact" model="file.config">
+        <field name="model_id" ref="base.model_res_partner" />
+        <field name="on_fail">stop</field>
+        <field
+            name="partner_ids"
+            eval="[ref('base.res_partner_18'), ref('base.res_partner_5')]"
+        />
+    </record>
+
+    <record id="file_config_contact_country" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__country_code" />
+    </record>
+    <record id="file_config_contact_name" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__name" />
+        <field name="required" eval="1" />
+    </record>
+    <record id="file_config_contact_street" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__street" />
+        <field name="required" eval="1" />
+    </record>
+    <record id="file_config_contact_street2" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__street2" />
+    </record>
+    <record id="file_config_contact_date" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__write_date" />
+        <field name="check_type" eval="1" />
+    </record>
+    <record id="file_config_contact_color" model="file.field">
+        <field name="config_id" ref="file_config_contact" />
+        <field name="field_id" ref="base.field_res_partner__color" />
+    </record>
+</odoo>

sheet_dataframe_process/models/__init__.py

@@ -0,0 +1,6 @@
+from . import file_config
+from . import file_field
+from . import file_partner_field
+from . import try_file
+from . import ir_model_fields
+from . import partner

sheet_dataframe_process/models/file_config.py

@@ -0,0 +1,84 @@
+from odoo import Command, fields, models
+
+# Demo record
+UISTRING = "sheet_dataframe_process.file_config_contact"
+
+
+class FileConfig(models.Model):
+    _name = "file.config"
+    _inherit = "mail.thread"
+    _description = "File Configuration"
+    _rec_name = "model_id"
+
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        required=True,
+        copy=False,
+        ondelete="cascade",
+        tracking=True,
+    )
+    code = fields.Char(help="Allow to browse between several identical models")
+    action = fields.Selection(
+        selection=[
+            ("display", "Display"),
+            ("dataframe", "Dataframe"),
+        ],
+        default="display",
+        tracking=True,
+        help="Some other behaviors can be implemented",
+    )
+    on_fail = fields.Selection(
+        selection=[("stop", "Stop"), ("skip", "Skip record (TODO)")],
+        default="stop",
+        tracking=True,
+        help="What should be the behavior in case of failure regarding constraint "
+        "fields (required, format, etc)\n\n"
+        " - Stop: stop the process by raising an exception\n"
+        " - Skip record: current line'll be ignored from the next process",
+    )
+    partner_ids = fields.Many2many(
+        comodel_name="res.partner",
+        domain="[('active', 'in', (True, False))]",
+        tracking=True,
+    )
+    field_ids = fields.One2many(
+        comodel_name="file.field", inverse_name="config_id", copy=True
+    )
+    field_match_ids = fields.One2many(
+        comodel_name="file.partner.field", inverse_name="config_id", copy=True
+    )
+
+    def populate_match_lines(self):
+        # TODO use api depends instead of ui button ?
+        self.ensure_one()
+        for partner in self.partner_ids:
+            ffields = self.field_match_ids.filtered(
+                lambda s, partner=partner: s.partner_id == partner
+            ).mapped("field_id")
+            line_ids = self.field_ids.filtered(
+                lambda s, ffields=ffields: s.field_id not in ffields
+            ).mapped("id")
+            self.field_match_ids = [
+                Command.create({"partner_id": partner.id, "line_id": x})
+                for x in line_ids
+            ]
+        self.field_match_ids.filtered(
+            lambda s: s.partner_id not in s.config_id.partner_ids
+        ).unlink()
+        for rec in self:
+            if rec == self.env.ref(UISTRING):
+                rec._populate_demo_column_names()
+
+    def _populate_match(self, mfield, mstring, uidstring):
+        record = self.field_match_ids.filtered(
+            lambda s, field=mfield, uidstring=uidstring: s.field_id.name == field
+            and s.partner_id == s.config_id.partner_ids[0]
+        )
+        if record:
+            record.matching_column = mstring
+
+    def _populate_demo_column_names(self):
+        self.field_match_ids.matching_column = False
+        self._populate_match("street", "myStreet", UISTRING)
+        self._populate_match("street2", "Second street", UISTRING)
+        self._populate_match("country_code", "Country", UISTRING)

sheet_dataframe_process/models/file_field.py

@@ -0,0 +1,29 @@
+from odoo import fields, models
+
+
+class FileField(models.Model):
+    _name = "file.field"
+    _inherit = ["mail.thread"]
+    _description = "Configuration de l'import de champ"
+    _order = "field_id ASC"
+
+    config_id = fields.Many2one(
+        comodel_name="file.config", required=True, ondelete="cascade"
+    )
+    field_id = fields.Many2one(
+        comodel_name="ir.model.fields",
+        ondelete="cascade",
+        required=True,
+        domain="[('model_id', '=', model_id)]",
+    )
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        related="config_id.model_id",
+        readonly=True,
+    )
+    required = fields.Boolean(
+        help="Prevent to import missing data if field is missing in some records",
+    )
+    check_type = fields.Boolean(
+        help="Check data type is compatible",
+    )

sheet_dataframe_process/models/file_partner_field.py

@@ -0,0 +1,16 @@
+from odoo import fields, models
+
+
+class FilePartnerField(models.Model):
+    _name = "file.partner.field"
+    _inherits = {"file.field": "line_id"}
+    _description = "Configuration de l'import de champ"
+    _order = "partner_id ASC, field_id ASC"
+
+    line_id = fields.Many2one(
+        comodel_name="file.field", required=True, ondelete="cascade"
+    )
+    partner_id = fields.Many2one(
+        comodel_name="res.partner",
+    )
+    matching_column = fields.Char(help="Field name in spreadsheet")

sheet_dataframe_process/models/ir_model_fields.py

@@ -0,0 +1,14 @@
+from odoo import api, models
+
+
+class IrModelFields(models.Model):
+    _inherit = "ir.model.fields"
+
+    @api.depends("field_description", "model")
+    def _compute_display_name(self):
+        super()._compute_display_name()
+        if self.env.context.get("technical_name"):
+            for field in self:
+                if self.env.context.get("technical_name"):
+                    field.display_name = field.name
+        return

sheet_dataframe_process/models/partner.py

@@ -0,0 +1,19 @@
+from odoo import models
+
+
+class ResPartner(models.Model):
+    _inherit = "res.partner"
+
+    def sheet_dataframe_import(self):
+        self.ensure_one()
+        transient = self.env["sheet.dataframe.transient"].create(
+            {
+                "config_id": self.env.context.get("config_id")["id"],
+                "partner_id": self.id,
+            }
+        )
+        action = self.env.ref(
+            "sheet_dataframe_process.sheet_dataframe_transient_action"
+        )._get_action_dict()
+        action["res_id"] = transient.id
+        return action