intermediate

openslides_backend/action/actions/user/account_import.py

@@ -1,9 +1,9 @@
-from typing import Any, Dict, List
+from typing import Any, Dict, List, cast
 
 from ....models.models import ActionWorker
 from ....permissions.management_levels import OrganizationManagementLevel
 from ....shared.schema import required_id_schema
-from ...mixins.import_mixins import ImportMixin, ImportState
+from ...mixins.import_mixins import ImportMixin, ImportRow, ImportState, Lookup, ResultType
 from ...util.default_schema import DefaultSchema
 from ...util.register import register_action
 from .create import UserCreate
@@ -29,6 +29,108 @@ class AccountImport(DuplicateCheckMixin, ImportMixin):
     import_name = "account"
 
     def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
+        if not instance["import"]:
+            return {}
+
+        instance = super().update_instance(instance)
+        self.error = False
+        self.setup_lookups()
+
+        self.rows = [self.validate_entry(row) for row in self.result["rows"]]
+
+        if self.error:
+            self.error_store_ids.append(instance["id"])
+        else:
+            create_action_payload: List[Dict[str, Any]] = []
+            update_action_payload: List[Dict[str, Any]] = []
+
+            for row in self.rows:
+                if row["state"] == ImportState.DONE:
+                    create_action_payload.append(row["data"])
+                else:
+                    update_action_payload.append(row["data"])
+                if create_action_payload:
+                    self.execute_other_action(UserCreate, create_action_payload)
+                if update_action_payload:
+                    self.execute_other_action(UserUpdate, update_action_payload)
+
+        return {}
+
+
+    def validate_entry(self, row: Dict[str, Dict[str, Any]]) -> ImportRow:
+        entry = row["data"]
+        username = self.get_value_from_union_str_object(entry.get("username"))
+        check_result = self.username_lookup.check_duplicate(username)
+        id_ = cast(int, self.username_lookup.get_field_by_name(username, "id"))
+
+        if check_result == ResultType.FOUND_ID and id_ != 0:
+            if row["state"] != ImportState.DONE:
+                entry["messages"].append(f"error: row state expected to be 'DONE', but it is '{row['state']}'.")
+                row["state"] = ImportState.ERROR
+                entry["username"]["info"] = ImportState.ERROR
+            elif entry["id"] != id_:
+                row["state"] = ImportState.ERROR
+                entry["username"]["info"] = ImportState.ERROR
+                entry["messages"].append(f"error: username '{username}' found in different id ({id_} instead of {entry['id']})")
+        elif check_result == ResultType.FOUND_MORE_IDS:
+            row["state"] = ImportState.ERROR
+            entry["username"]["info"] = ImportState.ERROR
+            entry["messages"].append(f"error: username '{username}' is duplicated in import.")
+        elif check_result == ResultType.NOT_FOUND:
+            if row["state"] != ImportState.NEW:
+                entry["messages"].append(f"error: row state expected to be 'NEW', but it is '{row['state']}'.")
+                row["state"] = ImportState.ERROR
+
+        saml_id = self.get_value_from_union_str_object(entry.get("saml_id"))
+        if saml_id:
+            check_result = self.saml_id_lookup.check_duplicate(saml_id)
+            id_from_saml_id = cast(int, self.saml_id_lookup.get_field_by_name(saml_id, "id"))
+            if check_result == ResultType.FOUND_ID and id_ != 0:
+                if id_ != id_from_saml_id:
+                    row["state"] = ImportState.ERROR
+                    entry["saml_id"]["info"] = ImportState.ERROR
+                    entry["messages"].append(f"error: saml_id '{saml_id}' found in different id ({id_from_saml_id} instead of {id_})")
+            elif check_result == ResultType.FOUND_MORE_IDS:
+                row["state"] = ImportState.ERROR
+                entry["saml_id"]["info"] = ImportState.ERROR
+                entry["messages"].append(f"error: saml_id '{saml_id}' is duplicated in import.")
+
+            if (default_password := entry.get("default_password")) and type(default_password) == dict and default_password["info"] == ImportState.WARNING:
+                for field in ("password", "can_change_own_password"):
+                    value = self.username_lookup.get_field_by_name(username, field)
+                    if value:
+                        if field == "can_change_own_password":
+                            entry[field] = False
+                        else:
+                            entry[field] = ""
+        if not self.error and row["state"] == ImportState.ERROR:
+            self.error = True
+        return { "state": row["state"], "data": row["data"], "messages": row.get("messages", [])}
+
+    def setup_lookups(self) -> None:
+        rows = self.result["rows"]
+        self.username_lookup = Lookup(
+            self.datastore,
+            "user",
+            [
+                ((entry := row["data"])["username"]["value"], entry)
+                for row in rows
+            ],
+            field="username",
+            mapped_fields=["saml_id", "default_password", "password", "can_change_own_password"],
+        )
+        self.saml_id_lookup = Lookup(
+            self.datastore,
+            "user",
+            [
+                (entry["saml_id"]["value"], entry)
+                for row in rows
+                if "saml_id" in (entry :=row["data"])
+            ],
+            field="saml_id",
+        )
+
+    def xupdate_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         instance = super().update_instance(instance)
 
         # handle abort in on_success

openslides_backend/action/actions/user/account_json_upload.py

@@ -85,10 +85,7 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         self.setup_lookups(data)
         self.create_usernames(data)
 
-        self.rows = [
-            self.validate_entry(entry, payload_index)
-            for payload_index, entry in enumerate(data)
-        ]
+        self.rows = [self.validate_entry(entry) for entry in data]
 
         # generate statistics
         itemCount = len(self.rows)
@@ -115,8 +112,7 @@ def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         return {}
 
     def validate_entry(
-        self, entry: Dict[str, Any], payload_index: int
-    ) -> Dict[str, Any]:
+        self, entry: Dict[str, Any]) -> Dict[str, Any]:
         messages: List[str] = []
         id_: Optional[int] = None
         old_saml_id: Optional[str] = None

openslides_backend/action/mixins/import_mixins.py

@@ -30,6 +30,12 @@ class ImportState(str, Enum):
     ERROR = "error"
 
 
+class ImportRow(TypedDict):
+    state: ImportState
+    data: Dict[str, Any]
+    messages: List[str]
+
+
 class ResultType(Enum):
     """Used by Lookup to differ the possible results in check_duplicate."""
 
@@ -104,7 +110,7 @@ def check_duplicate(self, name: SearchFieldType) -> ResultType:
 
     def get_field_by_name(
         self, name: SearchFieldType, fieldname: str
-    ) -> Optional[Union[int, str]]:
+    ) -> Optional[Union[int, str, bool]]:
         """Gets 'fieldname' from value of name_to_ids-dict"""
         if len(self.name_to_ids.get(name, [])) == 1:
             return self.name_to_ids[name][0].get(fieldname)
@@ -143,13 +149,23 @@ def count_warnings_in_payload(
                 count += BaseImportJsonUpload.count_warnings_in_payload(col)
         return count
 
+    @staticmethod
+    def get_value_from_union_str_object(field: Optional[Union[str, Dict[str, Any]]]) -> Optional[str]:
+        if type(field) == dict:
+            return field.get("value", "")
+        elif type(field) == str:
+            return field
+        else:
+            return None
+
 
 class ImportMixin(BaseImportJsonUpload):
     """
     Mixin for import actions. It works together with the json_upload.
     """
 
     import_name: str
+    rows: List[ImportRow] = []
 
     def prepare_action_data(self, action_data: ActionData) -> ActionData:
         self.error_store_ids: List[int] = []
@@ -188,6 +204,18 @@ def create_action_result_element(
             "rows": self.result.get("rows", []),
         }
 
+    def flatten_object_fields(self, fields: Optional[List[str]]) -> None:
+        """ replace objects from self.rows["data"] with their values. Uses the fields, if given, otherwise all"""
+        for row in self.rows:
+            entry = row["data"]
+            used_list= fields if fields else entry.keys()
+            for field in used_list:
+                if field in entry["data"]:
+                    if field == "username" and "id" in entry["data"][field]:
+                        entry["data"]["id"] = entry["data"][field]["id"]
+                    if type(dvalue := entry["data"][field]) == dict:
+                        entry["data"][field] = dvalue["value"]
+
     def get_on_success(self, action_data: ActionData) -> Callable[[], None]:
         def on_success() -> None:
             for instance in action_data:

tests/system/action/user/test_account_import.py

@@ -539,3 +539,8 @@ def test_json_upload_update_saml_id_in_existing_account(self) -> None:
             },
         )
         self.assert_model_not_exists("action_worker/1")
+
+    def test_json_upload_update_multiple_users_okay(self) -> None:
+        self.json_upload_multiple_users()
+        response_import = self.request("account.import", {"id": 1, "import": True})
+        self.assert_status_code(response_import, 200)

tests/system/action/user/test_account_json_upload.py

@@ -723,3 +723,109 @@ def json_upload_update_saml_id_in_existing_account(self) -> None:
             "saml_id": {"info": "done", "value": "new_one"},
             "username": {"info": "done", "value": "test", "id": 2},
         }
+
+    def json_upload_multiple_users(self) -> None:
+        self.set_models(
+            {
+                "user/2": {
+                    "username": "user2",
+                    "password": "secret",
+                    "default_password": "secret",
+                    "can_change_own_password": True,
+                    "default_vote_weight": "2.300000",
+                },
+                "user/3": {
+                    "username": "user3",
+                    "saml_id": "saml3",
+                    "password": "secret",
+                    "default_password": "secret",
+                    "can_change_own_password": True,
+                    "default_vote_weight": "3.300000",
+                },
+                "user/4": {
+                    "username": "user4",
+                    "first_name": "Martin",
+                    "last_name": "Luther King",
+                    "email": "[email protected]",
+                    "password": "secret",
+                    "default_password": "secret",
+                    "can_change_own_password": True,
+                    "default_vote_weight": "4.300000",
+                },
+            }
+        )
+        response = self.request(
+            "account.json_upload",
+            {
+                "data": [
+                    {
+                        "username": "user2",
+                        "saml_id": "test_saml_id2",
+                        "default_vote_weight": "2.345678"
+                    },
+                    {
+                        "saml_id": "saml3",
+                        "default_vote_weight": "3.345678"
+                    },
+                    {
+                        "first_name": "Martin",
+                        "last_name": "Luther King",
+                        "email": "[email protected]",
+                        "default_vote_weight": "4.345678",
+                    },
+                    {
+                        "username": "new_user5",
+                        "default_vote_weight": "5.345678",
+                        "saml_id": "saml5",
+                    },
+                    {
+                        "saml_id": "new_saml6",
+                        "default_vote_weight": "6.345678",
+                    },
+                    {
+                        "first_name": "Joan",
+                        "last_name": "Baez7",
+                        "default_vote_weight": "7.345678",
+                    },
+
+                ],
+            },
+        )
+        self.assert_status_code(response, 200)
+        worker = self.assert_model_exists("action_worker/1")
+        assert worker["state"] == ImportState.WARNING
+        assert worker["result"]["import"] == "account"
+        assert worker["result"]["rows"][0]["state"] == ImportState.DONE
+        assert worker["result"]["rows"][0]["messages"] == [
+            "Will remove password and default_password and forbid changing your OpenSlides password."
+        ]
+        assert worker["result"]["rows"][0]["data"] == {'id': 2, 'saml_id': {'info': 'new', 'value': 'test_saml_id2'}, 'username': {'id': 2, 'info': 'done', 'value': 'user2'}, 'default_password': {'info': 'warning', 'value': ''}, 'default_vote_weight': '2.345678'}
+
+        assert worker["result"]["rows"][1]["state"] == ImportState.DONE
+        assert worker["result"]["rows"][1]["messages"] == [
+            "Will remove password and default_password and forbid changing your OpenSlides password."
+        ]
+        assert worker["result"]["rows"][1]["data"] == {'id': 3, 'saml_id': {'info': 'new', 'value': 'saml3'}, 'username': {'id': 3, 'info': 'done', 'value': 'user3'}, 'default_password': {'info': 'warning', 'value': ''}, 'default_vote_weight': '3.345678'}
+
+        assert worker["result"]["rows"][2]["state"] == ImportState.DONE
+        assert worker["result"]["rows"][2]["messages"] == []
+        assert worker["result"]["rows"][2]["data"] == {'id': 4, 'email': '[email protected]', 'username': {'id': 4, 'info': 'done', 'value': 'user4'}, 'last_name': 'Luther King', 'first_name': 'Martin', 'default_vote_weight': '4.345678'}
+
+        assert worker["result"]["rows"][3]["state"] == ImportState.NEW
+        assert worker["result"]["rows"][3]["messages"] == [
+            "Will remove password and default_password and forbid changing your OpenSlides password."
+        ]
+        assert worker["result"]["rows"][3]["data"] == {'saml_id': {'info': 'new', 'value': 'saml5'}, 'username': {'info': 'done', 'value': 'new_user5'}, 'default_password': {'info': 'warning', 'value': ''}, 'default_vote_weight': '5.345678'}
+
+        assert worker["result"]["rows"][4]["state"] == ImportState.NEW
+        assert worker["result"]["rows"][4]["messages"] == [
+            "Will remove password and default_password and forbid changing your OpenSlides password."
+        ]
+        assert worker["result"]["rows"][4]["data"] == {'saml_id': {'info': 'new', 'value': 'new_saml6'}, 'username': {'info': 'generated', 'value': 'new_saml6'}, 'default_password': {'info': 'warning', 'value': ''}, 'default_vote_weight': '6.345678'}
+
+        assert worker["result"]["rows"][5]["state"] == ImportState.NEW
+        assert worker["result"]["rows"][5]["messages"] == []
+        default_password = worker["result"]["rows"][5]["data"].pop("default_password")
+        assert default_password["info"] == ImportState.GENERATED
+        assert default_password["value"]
+        assert worker["result"]["rows"][5]["data"] == {'username': {'info': 'generated', 'value': 'JoanBaez7'}, 'last_name': 'Baez7', 'first_name': 'Joan', 'default_vote_weight': '7.345678'}