[ADD] polars_process: process polars dataframe from file

polars_process/README.rst

@@ -0,0 +1,115 @@
+==============
+Polars Process
+==============
+
+.. 
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! This file is generated by oca-gen-addon-readme !!
+   !! changes will be overwritten.                   !!
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+   !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d
+   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png
+    :target: https://odoo-community.org/page/development-status
+    :alt: Alpha
+.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
+    :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Freporting--engine-lightgray.png?logo=github
+    :target: https://github.com/OCA/reporting-engine/tree/18.0/polars_process
+    :alt: OCA/reporting-engine
+.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
+    :target: https://translation.odoo-community.org/projects/reporting-engine-18-0/reporting-engine-18-0-polars_process
+    :alt: Translate me on Weblate
+.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
+    :target: https://runboat.odoo-community.org/builds?repo=OCA/reporting-engine&target_branch=18.0
+    :alt: Try me on Runboat
+
+|badge1| |badge2| |badge3| |badge4| |badge5|
+
+From an imported spreadsheet (xlsx) or db query, this module allows to
+transform data in Polars dataframe and process them according to rules
+in order to:
+
+-  filter data and display
+-  obtain another dataframe with only the expected data to use in Odoo
+
+A such dataframe can help to prepare data in order to be used to
+create/update or import
+
+Typical use case:
+
+You receive files from your vendors and these files have many difference
+(column names, number of columns, dirty paging) but contains data
+related to same concepts. Then you want apply them a common process to
+automate things. For that you need to transform/arrange data to the same
+way
+
+Why dataframe ?
+
+-  a dataframe is a kind of in-memory dataset on which you can operate
+-  you can operates on your entire dataset a bit like with a database
+   but in memory: you don't need to iterate on each line to perform
+   operations
+-  the operations are powerful: filter, add column resulting from
+   calculation, select a subset of data
+
+Why Polars ?
+
+-  performance: code in rust
+-  environment consideration
+-  dynamic project
+
+.. IMPORTANT::
+   This is an alpha version, the data model and design can change at any time without warning.
+   Only for development or testing purpose, do not use in production.
+   `More details on development status <https://odoo-community.org/page/development-status>`_
+
+**Table of contents**
+
+.. contents::
+   :local:
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues <https://github.com/OCA/reporting-engine/issues>`_.
+In case of trouble, please check there if your issue has already been reported.
+If you spotted it first, help us to smash it by providing a detailed and welcomed
+`feedback <https://github.com/OCA/reporting-engine/issues/new?body=module:%20polars_process%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
+
+Do not contact contributors directly about support or help with technical issues.
+
+Credits
+=======
+
+Authors
+-------
+
+* Akretion
+
+Maintainers
+-----------
+
+This module is maintained by the OCA.
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px
+    :target: https://github.com/bealdav
+    :alt: bealdav
+
+Current `maintainer <https://odoo-community.org/page/maintainer-role>`__:
+
+|maintainer-bealdav| 
+
+This module is part of the `OCA/reporting-engine <https://github.com/OCA/reporting-engine/tree/18.0/polars_process>`_ project on GitHub.
+
+You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

polars_process/__init__.py

@@ -0,0 +1,2 @@
+from . import models
+from . import wizards

polars_process/__manifest__.py

@@ -0,0 +1,34 @@
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+    "name": "Polars Process",
+    "version": "18.0.1.0.0",
+    "summary": "Allow to create a Polars dataframe from file or db query and "
+    "process it according to rules",
+    "category": "Reporting",
+    "license": "AGPL-3",
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "development_status": "Alpha",
+    "website": "https://github.com/OCA/reporting-engine",
+    "maintainers": ["bealdav"],
+    "depends": [
+        "contacts",
+    ],
+    "external_dependencies": {
+        "python": [
+            "polars",
+            "fastexcel",
+        ]
+    },
+    "data": [
+        "data/action.xml",
+        "data/demo.xml",
+        "security/ir.model.access.xml",
+        "wizards/df_process.xml",
+        "views/dataframe.xml",
+        "views/df_field.xml",
+        "views/df_source.xml",
+        "views/menu.xml",
+    ],
+    "installable": True,
+}

polars_process/data/action.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+    <record id="refresh_df_source_action" model="ir.actions.server">
+        <field name="name">🐻‍❄️ Populate file polars example</field>
+        <field name="model_id" ref="model_df_source" />
+        <field name="binding_model_id" ref="model_df_source" />
+        <field name="state">code</field>
+        <field name="code">env["df.source"]._populate()</field>
+    </record>
+</odoo>

polars_process/data/demo.xml

@@ -0,0 +1,44 @@
+<odoo>
+    <record id="dataframe_contact" model="dataframe">
+        <field name="model_id" ref="base.model_res_partner" />
+        <field name="on_fail">stop</field>
+        <field name="code">import_preprocess</field>
+        <field name="rename" eval="1" />
+    </record>
+
+    <record id="dataframe_contact_country" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__country_code" />
+        <field name="name">Country</field>
+    </record>
+    <record id="dataframe_contact_name" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__name" />
+        <field name="name">Name</field>
+        <field name="required" eval="1" />
+    </record>
+    <record id="dataframe_contact_street" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__street" />
+        <field name="name">Street</field>
+        <field name="required" eval="1" />
+    </record>
+    <record id="dataframe_contact_street2" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__street2" />
+        <field name="name">2nd Street</field>
+        <field name="renamed">street2</field>
+    </record>
+    <record id="dataframe_contact_date" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__write_date" />
+        <field name="name">Date</field>
+        <field name="check_type" eval="1" />
+    </record>
+    <record id="dataframe_contact_color" model="df.field">
+        <field name="dataframe_id" ref="dataframe_contact" />
+        <field name="field_id" ref="base.field_res_partner__color" />
+        <field name="name">Colour</field>
+        <field name="renamed">Color</field>
+    </record>
+</odoo>

polars_process/models/__init__.py

@@ -0,0 +1,4 @@
+from . import dataframe
+from . import df_field
+from . import df_source
+from . import ir_model_fields

polars_process/models/dataframe.py

@@ -0,0 +1,39 @@
+from odoo import fields, models
+
+
+class Dataframe(models.Model):
+    _name = "dataframe"
+    _inherit = "mail.thread"
+    _description = "File Configuration"
+    _rec_name = "code"
+
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        required=True,
+        copy=False,
+        ondelete="cascade",
+        tracking=True,
+    )
+    code = fields.Char(help="Allow to browse between several identical models")
+    rename = fields.Boolean(help="Rename dataframe fields")
+    action = fields.Selection(
+        selection=[
+            ("display", "Display"),
+            ("dataframe", "Dataframe"),
+        ],
+        default="display",
+        tracking=True,
+        help="Some other behaviors can be implemented",
+    )
+    on_fail = fields.Selection(
+        selection=[("stop", "Stop"), ("skip", "Skip record (TODO)")],
+        default="stop",
+        tracking=True,
+        help="What should be the behavior in case of failure regarding constraint "
+        "fields (required, format, etc)\n\n"
+        " - Stop: stop the process by raising an exception\n"
+        " - Skip record: current line'll be ignored from the next process",
+    )
+    field_ids = fields.One2many(
+        comodel_name="df.field", inverse_name="dataframe_id", copy=True
+    )

polars_process/models/df_field.py

@@ -0,0 +1,32 @@
+from odoo import fields, models
+
+
+class FileField(models.Model):
+    _name = "df.field"
+    _inherit = ["mail.thread"]
+    _description = "Configuration de l'import de champ"
+    _order = "field_id ASC"
+
+    dataframe_id = fields.Many2one(
+        comodel_name="dataframe", required=True, ondelete="cascade"
+    )
+    sequence = fields.Integer()
+    field_id = fields.Many2one(
+        comodel_name="ir.model.fields",
+        ondelete="cascade",
+        required=True,
+        domain="[('model_id', '=', model_id)]",
+    )
+    model_id = fields.Many2one(
+        comodel_name="ir.model",
+        related="dataframe_id.model_id",
+        readonly=True,
+    )
+    name = fields.Char(help="Name field in the source file (spreadsheet)")
+    renamed = fields.Char(help="If specified, renamed in dataframe")
+    required = fields.Boolean(
+        help="Prevent to import missing data if field is missing in some records",
+    )
+    check_type = fields.Boolean(
+        help="Check data type is compatible",
+    )

polars_process/models/df_source.py

@@ -0,0 +1,100 @@
+import base64
+from pathlib import Path
+
+from odoo import _, fields, models
+from odoo.modules.module import get_module_path
+
+
+class DfSource(models.Model):
+    _name = "df.source"
+    _description = "Dataframe data source"
+
+    dataframe_id = fields.Many2one(
+        comodel_name="dataframe", required=True, ondelete="cascade"
+    )
+    name = fields.Char()
+    sequence = fields.Integer()
+    rename = fields.Boolean(help="Display renamed Dataframe in wizard")
+    template = fields.Binary(string="File", attachment=False)
+    readonly = fields.Boolean(help="Imported records from module are readonly created")
+
+    def _populate(self):
+        def create_attach(myfile, addon, idstring, relative_path):
+            with open(myfile, "rb") as f:
+                name = f.name[f.name.find(addon) :]
+                vals = {
+                    "dataframe_id": self.env.ref(idstring).id,
+                    "name": name,
+                    "readonly": True,
+                    "rename": True,
+                }
+                if ".sql" in name:
+                    vals["query"] = self._get_file(name)
+                self.env[self._name].sudo().create(vals)
+
+        self.env[self._name].search([("template", "=", False)]).unlink()
+        paths = self._get_test_file_paths()
+        for addon, data in paths.items():
+            relative_path = data["relative_path"]
+            idstring = data["xmlid"]
+            if self.env.ref(idstring):
+                mpath = Path(get_module_path(addon)) / relative_path
+                for mfile in tuple(mpath.iterdir()):
+                    create_attach(mfile, addon, idstring, relative_path)
+        action = self.env.ref("polars_process.df_source_action")._get_action_dict()
+        return action
+
+    def start(self):
+        self.ensure_one()
+        vals = {
+            "filename": self.name,
+            "df_source_id": self.id,
+            "dataframe_id": self.dataframe_id.id,
+            "file": base64.b64encode(self._get_file()),
+        }
+        transient = self.env["df.process.wiz"].create(vals)
+        action = self.env.ref("polars_process.df_process_wiz_action")._get_action_dict()
+        action["res_id"] = transient.id
+        return action
+
+    def _get_file(self, name=None):
+        # TODO Clean
+        if self.template:
+            return self.template
+        name = self.name or name
+        module = name[: name.find("/")]
+        relative = self._get_test_file_paths().get(module)
+        relative = relative and relative.get("relative_path")
+        if relative:
+            path = Path(get_module_path(module))
+            path = path / relative / name[name.rfind("/") + 1 :]
+            with open(path, "rb") as f:
+                return f.read()
+
+    def _get_test_file_paths(self):
+        """
+        You may override if you want populate files in your module
+        returns:
+        {"module_name": {
+            "relative_path": "tests/files",
+            "xmlid": "dataframe_xml_id"}
+            }
+        }
+        """
+        return {
+            "polars_process": {
+                "relative_path": "tests/files",
+                "xmlid": "polars_process.dataframe_contact",
+            }
+        }
+
+    def ui_form(self):
+        self.ensure_one()
+        return {
+            "name": _("Dataframe source"),
+            "res_model": self._name,
+            "view_mode": "form",
+            "res_id": self.id,
+            "type": "ir.actions.act_window",
+            "target": "current",
+        }