From bf48bbb3f7026d70bc031a8396928320b48e299c Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 17 Apr 2024 11:24:00 +0100 Subject: [PATCH 01/65] add typing --- portality/dao.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index 1c2b32da5f..e8e342af88 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -9,7 +9,7 @@ from collections import UserDict from copy import deepcopy from datetime import timedelta -from typing import List +from typing import List, Self from portality.core import app, es_connection as ES from portality.lib import dates @@ -380,7 +380,7 @@ def refresh(cls): return ES.indices.refresh(index=cls.index_name()) @classmethod - def pull(cls, id_): + def pull(cls, id_) -> Self: """Retrieve object by id.""" if id_ is None or id_ == '': return None @@ -400,7 +400,7 @@ def pull(cls, id_): return cls(**out) @classmethod - def pull_by_key(cls, key, value): + def pull_by_key(cls, key, value) -> Self: res = cls.query(q={"query": {"term": {key + app.config['FACET_FIELD']: value}}}) if res.get('hits', {}).get('total', {}).get('value', 0) == 1: return cls.pull(res['hits']['hits'][0]['_source']['id']) @@ -408,7 +408,7 @@ def pull_by_key(cls, key, value): return None @classmethod - def object_query(cls, q=None, **kwargs): + def object_query(cls, q=None, **kwargs) -> List[Self]: result = cls.query(q, **kwargs) return [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] @@ -833,7 +833,7 @@ def autocomplete(cls, field, prefix, size=5): return result @classmethod - def q2obj(cls, **kwargs): + def q2obj(cls, **kwargs) -> List[Self]: extra_trace_info = '' if 'q' in kwargs: extra_trace_info = "\nQuery sent to ES (before manipulation in DomainObject.query):\n{}\n".format( From 7369c2ca6068daea5b9098f5aa34a65de1941689 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 17 Apr 2024 11:37:16 +0100 Subject: [PATCH 02/65] add typing --- portality/dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index e8e342af88..fc7bd5909e 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -9,7 +9,7 @@ from collections import UserDict from copy import deepcopy from datetime import timedelta -from typing import List, Self +from typing import List, Self, Iterable, Union, Dict from portality.core import app, es_connection as ES from portality.lib import dates @@ -565,7 +565,7 @@ def handle_es_raw_response(cls, res, wrap, extra_trace_info=''): @classmethod def iterate(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, - keepalive: str = '1m'): + keepalive: str = '1m') -> Iterable[Union[Self, Dict]]: """ Provide an iterable of all items in a model, use :param q: The query to scroll results on :param page_size: limited by ElasticSearch, check settings to override From 185be056120189bae4a9dd29a391ec12a465429e Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 18 Apr 2024 11:23:09 +0100 Subject: [PATCH 03/65] update typing --- portality/dao.py | 15 +++++++++------ .../scripts/accounts_with_marketing_consent.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index fc7bd5909e..67bf1e8afe 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -9,7 +9,10 @@ from collections import UserDict from copy import deepcopy from datetime import timedelta -from typing import List, Self, Iterable, Union, Dict +from typing import List, Iterable, Union, Dict + +if sys.version_info >= (3, 11): + from typing import Self from portality.core import app, es_connection as ES from portality.lib import dates @@ -380,7 +383,7 @@ def refresh(cls): return ES.indices.refresh(index=cls.index_name()) @classmethod - def pull(cls, id_) -> Self: + def pull(cls, id_) -> 'Self': """Retrieve object by id.""" if id_ is None or id_ == '': return None @@ -400,7 +403,7 @@ def pull(cls, id_) -> Self: return cls(**out) @classmethod - def pull_by_key(cls, key, value) -> Self: + def pull_by_key(cls, key, value) -> 'Self': res = cls.query(q={"query": {"term": {key + app.config['FACET_FIELD']: value}}}) if res.get('hits', {}).get('total', {}).get('value', 0) == 1: return cls.pull(res['hits']['hits'][0]['_source']['id']) @@ -408,7 +411,7 @@ def pull_by_key(cls, key, value) -> Self: return None @classmethod - def object_query(cls, q=None, **kwargs) -> List[Self]: + def object_query(cls, q=None, **kwargs) -> List['Self']: result = cls.query(q, **kwargs) return [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] @@ -565,7 +568,7 @@ def handle_es_raw_response(cls, res, wrap, extra_trace_info=''): @classmethod def iterate(cls, q: dict = None, page_size: int = 1000, limit: int = None, wrap: bool = True, - keepalive: str = '1m') -> Iterable[Union[Self, Dict]]: + keepalive: str = '1m') -> Iterable[Union['Self', Dict]]: """ Provide an iterable of all items in a model, use :param q: The query to scroll results on :param page_size: limited by ElasticSearch, check settings to override @@ -833,7 +836,7 @@ def autocomplete(cls, field, prefix, size=5): return result @classmethod - def q2obj(cls, **kwargs) -> List[Self]: + def q2obj(cls, **kwargs) -> List['Self']: extra_trace_info = '' if 'q' in kwargs: extra_trace_info = "\nQuery sent to ES (before manipulation in DomainObject.query):\n{}\n".format( diff --git a/portality/scripts/accounts_with_marketing_consent.py b/portality/scripts/accounts_with_marketing_consent.py index e45976e099..9734b7847e 100644 --- a/portality/scripts/accounts_with_marketing_consent.py +++ b/portality/scripts/accounts_with_marketing_consent.py @@ -20,7 +20,7 @@ HEADERS = ["ID", "Name", "Email", "Created", "Last Updated", "Updated Since Create?"] -def output_map(acc): +def output_map(acc: Account): updated_since_create = acc.created_timestamp < acc.last_updated_timestamp return { From 1536b7987d21292e6d7e43ce51dc8b30e2a292d8 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 18 Apr 2024 11:40:06 +0100 Subject: [PATCH 04/65] update typing --- portality/dao.py | 14 +++++++------- portality/models/editors.py | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index 67bf1e8afe..c6ad5d706a 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -1,23 +1,23 @@ -import time +import json import re import sys -import uuid -import json -import elasticsearch +import time import urllib.parse - +import uuid from collections import UserDict from copy import deepcopy from datetime import timedelta from typing import List, Iterable, Union, Dict -if sys.version_info >= (3, 11): - from typing import Self +import elasticsearch from portality.core import app, es_connection as ES from portality.lib import dates from portality.lib.dates import FMT_DATETIME_STD +if sys.version_info >= (3, 11): + from typing import Self + # All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on. # You can overwrite and add to the DomainObject functions as required. See models.py for some examples. diff --git a/portality/models/editors.py b/portality/models/editors.py index 8b935553cb..9664f06c1c 100644 --- a/portality/models/editors.py +++ b/portality/models/editors.py @@ -1,6 +1,12 @@ +import sys +from typing import List, Optional + from portality.dao import DomainObject, ScrollInitialiseException from portality.models import Account +if sys.version_info >= (3, 11): + from typing import Self + class EditorGroup(DomainObject): """ @@ -9,7 +15,7 @@ class EditorGroup(DomainObject): __type__ = "editor_group" @classmethod - def group_exists_by_name(cls, name): + def group_exists_by_name(cls, name) -> Optional[str]: q = EditorGroupQuery(name) res = cls.query(q=q.query()) ids = [hit.get("_source", {}).get("id") for hit in res.get("hits", {}).get("hits", []) if "_source" in hit] @@ -19,7 +25,7 @@ def group_exists_by_name(cls, name): return ids[0] @classmethod - def _groups_by_x(cls, **kwargs): + def _groups_by_x(cls, **kwargs) -> List['Self']: """ Generalised editor groups by maned / editor / associate """ # ~~-> EditorGroupMember:Query~~ q = EditorGroupMemberQuery(**kwargs) @@ -30,15 +36,15 @@ def _groups_by_x(cls, **kwargs): return [] @classmethod - def groups_by_maned(cls, maned): + def groups_by_maned(cls, maned) -> List['Self']: return cls._groups_by_x(maned=maned) @classmethod - def groups_by_editor(cls, editor): + def groups_by_editor(cls, editor) -> List['Self']: return cls._groups_by_x(editor=editor) @classmethod - def groups_by_associate(cls, associate): + def groups_by_associate(cls, associate) -> List['Self']: return cls._groups_by_x(associate=associate) @property @@ -101,7 +107,7 @@ def __init__(self, name): def query(self): q = { - "track_total_hits" : True, + "track_total_hits": True, "query": {"term": {"name.exact": self.name}} } return q @@ -111,6 +117,7 @@ class EditorGroupMemberQuery(object): """ ~~EditorGroupMember:Query->Elasticsearch:Technology~~ """ + def __init__(self, editor=None, associate=None, maned=None): self.editor = editor self.associate = associate @@ -120,7 +127,7 @@ def query(self): q = { "track_total_hits": True, "query": {"bool": {"should": []}}, - "sort": {"name.exact": {"order" : "asc"}} + "sort": {"name.exact": {"order": "asc"}} } if self.editor is not None: et = {"term": {"editor.exact": self.editor}} @@ -129,6 +136,6 @@ def query(self): at = {"term": {"associates.exact": self.associate}} q["query"]["bool"]["should"].append(at) if self.maned is not None: - mt = {"term" : {"maned.exact" : self.maned}} + mt = {"term": {"maned.exact": self.maned}} q["query"]["bool"]["should"].append(mt) return q From 77d1ec87eb59498c4b44c042b4a209dc17abf003 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 19 Apr 2024 09:47:46 +0100 Subject: [PATCH 05/65] rm editor_group --- portality/bll/services/todo.py | 131 ++++++++++++++++----------------- 1 file changed, 63 insertions(+), 68 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index fc57f66da7..67b7050b0e 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -1,9 +1,12 @@ -from portality.lib.argvalidate import argvalidate +from typing import Iterable + +from portality import constants from portality import models from portality.bll import exceptions -from portality import constants from portality.lib import dates -from datetime import datetime +from portality.lib.argvalidate import argvalidate +from portality.models import EditorGroup + class TodoService(object): """ @@ -14,16 +17,16 @@ class TodoService(object): def group_stats(self, group_id): # ~~-> EditorGroup:Model~~ eg = models.EditorGroup.pull(group_id) - stats = {"editor_group" : eg.data} + stats = {"editor_group": eg.data} - #~~-> Account:Model ~~ + # ~~-> Account:Model ~~ stats["editors"] = {} editors = [eg.editor] + eg.associates for editor in editors: acc = models.Account.pull(editor) stats["editors"][editor] = { - "email" : None if acc is None else acc.email - } + "email": None if acc is None else acc.email + } q = GroupStatsQuery(eg.name) resp = models.Application.query(q=q.query()) @@ -42,7 +45,8 @@ def group_stats(self, group_id): stats["by_editor"][bucket["key"]]["update_requests"] = b["doc_count"] stats["total"]["update_requests"] += b["doc_count"] - unassigned_buckets = resp.get("aggregations", {}).get("unassigned", {}).get("application_type", {}).get("buckets", []) + unassigned_buckets = resp.get("aggregations", {}).get("unassigned", {}).get("application_type", {}).get( + "buckets", []) stats["unassigned"] = {"applications": 0, "update_requests": 0} for ub in unassigned_buckets: if ub["key"] == constants.APPLICATION_TYPE_NEW_APPLICATION: @@ -63,7 +67,6 @@ def group_stats(self, group_id): return stats - def top_todo(self, account, size=25, new_applications=True, update_requests=True): """ Returns the top number of todo items for a given user @@ -74,7 +77,7 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True """ # first validate the incoming arguments to ensure that we've got the right thing argvalidate("top_todo", [ - {"arg" : account, "instance" : models.Account, "allow_none" : False, "arg_name" : "account"} + {"arg": account, "instance": models.Account, "allow_none": False, "arg_name": "account"} ], exceptions.ArgumentException) queries = [] @@ -90,7 +93,7 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True queries.append(TodoRules.maned_last_month_update_requests(size, maned_of)) queries.append(TodoRules.maned_new_update_requests(size, maned_of)) - if new_applications: # editor and associate editor roles only deal with new applications + if new_applications: # editor and associate editor roles only deal with new applications if account.has_role("editor"): groups = [g for g in models.EditorGroup.groups_by_editor(account.id)] regular_groups = [g for g in groups if g.maned != account.id] @@ -126,10 +129,10 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True todos.append({ "date": ap.last_manual_update_timestamp if sort == "last_manual_update" else ap.date_applied_timestamp, "date_type": sort, - "action_id" : [aid], - "title" : ap.bibjson().title, - "object_id" : ap.id, - "object" : ap, + "action_id": [aid], + "title": ap.bibjson().title, + "object_id": ap.id, + "object": ap, "boost": boost }) @@ -170,7 +173,7 @@ def maned_stalled(cls, size, maned_of): stalled = TodoQuery( musts=[ TodoQuery.lmu_older_than(8), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_new_application() ], must_nots=[ @@ -187,7 +190,7 @@ def maned_follow_up_old(cls, size, maned_of): follow_up_old = TodoQuery( musts=[ TodoQuery.cd_older_than(10), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_new_application() ], must_nots=[ @@ -204,7 +207,7 @@ def maned_ready(cls, size, maned_of): ready = TodoQuery( musts=[ TodoQuery.status([constants.APPLICATION_STATUS_READY]), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_new_application() ], sort=sort_date, @@ -219,7 +222,7 @@ def maned_completed(cls, size, maned_of): musts=[ TodoQuery.status([constants.APPLICATION_STATUS_COMPLETED]), TodoQuery.lmu_older_than(2), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_new_application() ], sort=sort_date, @@ -235,7 +238,7 @@ def maned_assign_pending(cls, size, maned_of): TodoQuery.exists("admin.editor_group"), TodoQuery.lmu_older_than(2), TodoQuery.status([constants.APPLICATION_STATUS_PENDING]), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_new_application() ], must_nots=[ @@ -258,7 +261,7 @@ def maned_last_month_update_requests(cls, size, maned_of): TodoQuery.exists("admin.editor_group"), TodoQuery.cd_older_than(since_som, unit="s"), # TodoQuery.status([constants.APPLICATION_STATUS_UPDATE_REQUEST]), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_update_request() ], must_nots=[ @@ -278,7 +281,7 @@ def maned_new_update_requests(cls, size, maned_of): TodoQuery.exists("admin.editor_group"), # TodoQuery.cd_older_than(4), # TodoQuery.status([constants.APPLICATION_STATUS_UPDATE_REQUEST]), - TodoQuery.editor_group(maned_of), + TodoQuery.editor_groups(maned_of), TodoQuery.is_update_request() ], must_nots=[ @@ -364,7 +367,7 @@ def editor_assign_pending(cls, groups, size): return constants.TODO_EDITOR_ASSIGN_PENDING, assign_pending, sort_date, 1 @classmethod - def associate_stalled(cls, acc_id, size): + def associate_stalled(cls, acc_id, size): sort_field = "created_date" stalled = TodoQuery( musts=[ @@ -386,7 +389,7 @@ def associate_stalled(cls, acc_id, size): return constants.TODO_ASSOCIATE_PROGRESS_STALLED, stalled, sort_field, 0 @classmethod - def associate_follow_up_old(cls, acc_id, size): + def associate_follow_up_old(cls, acc_id, size): sort_field = "created_date" follow_up_old = TodoQuery( musts=[ @@ -448,7 +451,7 @@ class TodoQuery(object): ~~->$Todo:Query~~ ~~^->Elasticsearch:Technology~~ """ - lmu_sort = {"last_manual_update" : {"order" : "asc"}} + lmu_sort = {"last_manual_update": {"order": "asc"}} # cd_sort = {"created_date" : {"order" : "asc"}} # NOTE that admin.date_applied and created_date should be the same for applications, but for some reason this is not always the case # therefore, we take a created_date sort to mean a date_applied sort @@ -463,16 +466,16 @@ def __init__(self, musts=None, must_nots=None, sort="last_manual_update", size=1 def query(self): sort = self.lmu_sort if self._sort == "last_manual_update" else self.cd_sort q = { - "query" : { - "bool" : { + "query": { + "bool": { "must": self._musts, "must_not": self._must_nots } }, - "sort" : [ + "sort": [ sort ], - "size" : self._size + "size": self._size } return q @@ -492,14 +495,6 @@ def is_update_request(cls): } } - @classmethod - def editor_group(cls, groups): - return { - "terms" : { - "admin.editor_group.exact" : [g.name for g in groups] - } - } - @classmethod def lmu_older_than(cls, weeks): return { @@ -523,25 +518,24 @@ def cd_older_than(cls, count, unit="w"): @classmethod def status(cls, statuses): return { - "terms" : { - "admin.application_status.exact" : statuses + "terms": { + "admin.application_status.exact": statuses } } @classmethod def exists(cls, field): return { - "exists" : { - "field" : field + "exists": { + "field": field } } @classmethod - def editor_groups(cls, groups): - gids = [g.name for g in groups] + def editor_groups(cls, groups: Iterable[EditorGroup]): return { "terms": { - "admin.editor_group.exact": gids + "admin.editor_group.exact": [g.name for g in groups] } } @@ -554,18 +548,19 @@ def editor(cls, acc_id): } -class GroupStatsQuery(): +class GroupStatsQuery: """ ~~->$GroupStats:Query~~ ~~^->Elasticsearch:Technology~~ """ + def __init__(self, group_name, editor_count=10): self.group_name = group_name self.editor_count = editor_count def query(self): return { - "track_total_hits" : True, + "track_total_hits": True, "query": { "bool": { "must": [ @@ -575,10 +570,10 @@ def query(self): } } ], - "must_not" : [ + "must_not": [ { - "terms" : { - "admin.application_status.exact" : [ + "terms": { + "admin.application_status.exact": [ constants.APPLICATION_STATUS_ACCEPTED, constants.APPLICATION_STATUS_REJECTED ] @@ -587,26 +582,26 @@ def query(self): ] } }, - "size" : 0, - "aggs" : { - "editor" : { - "terms" : { - "field" : "admin.editor.exact", - "size" : self.editor_count + "size": 0, + "aggs": { + "editor": { + "terms": { + "field": "admin.editor.exact", + "size": self.editor_count }, - "aggs" : { - "application_type" : { - "terms" : { + "aggs": { + "application_type": { + "terms": { "field": "admin.application_type.exact", "size": 2 } } } }, - "status" : { - "terms" : { - "field" : "admin.application_status.exact", - "size" : len(constants.APPLICATION_STATUSES_ALL) + "status": { + "terms": { + "field": "admin.application_status.exact", + "size": len(constants.APPLICATION_STATUSES_ALL) }, "aggs": { "application_type": { @@ -617,13 +612,13 @@ def query(self): } } }, - "unassigned" : { - "missing" : { + "unassigned": { + "missing": { "field": "admin.editor.exact" }, - "aggs" : { - "application_type" : { - "terms" : { + "aggs": { + "application_type": { + "terms": { "field": "admin.application_type.exact", "size": 2 } @@ -631,4 +626,4 @@ def query(self): } } } - } \ No newline at end of file + } From 5f2bc8b8bbb78cc90b53b29e3f6028977685f712 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 19 Apr 2024 11:29:13 +0100 Subject: [PATCH 06/65] add migrate --- .../3850_link_editor_groups_by_name/README.md | 5 +++ .../migrate.json | 21 ++++++++++ .../operations.py | 41 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 portality/migrate/3850_link_editor_groups_by_name/README.md create mode 100644 portality/migrate/3850_link_editor_groups_by_name/migrate.json create mode 100644 portality/migrate/3850_link_editor_groups_by_name/operations.py diff --git a/portality/migrate/3850_link_editor_groups_by_name/README.md b/portality/migrate/3850_link_editor_groups_by_name/README.md new file mode 100644 index 0000000000..2b33338c05 --- /dev/null +++ b/portality/migrate/3850_link_editor_groups_by_name/README.md @@ -0,0 +1,5 @@ +https://github.com/DOAJ/doajPM/issues/3245 + +The script looks for applications which are rejected and if they have a `related_journal` then it tags them as update requests. + + python portality/upgrade.py -u portality/migrate/3850_link_editor_groups_by_name/migrate.json \ No newline at end of file diff --git a/portality/migrate/3850_link_editor_groups_by_name/migrate.json b/portality/migrate/3850_link_editor_groups_by_name/migrate.json new file mode 100644 index 0000000000..6c0d23cc2a --- /dev/null +++ b/portality/migrate/3850_link_editor_groups_by_name/migrate.json @@ -0,0 +1,21 @@ +{ + "batch" : 10000, + "types": [ + { + "type" : "application", + "init_with_model" : true, + "keepalive" : "5m", + "functions" : [ + "portality.migrate.3850_link_editor_groups_by_name.operations.name_to_id" + ] + }, + { + "type" : "journal", + "init_with_model" : true, + "keepalive" : "5m", + "functions" : [ + "portality.migrate.3850_link_editor_groups_by_name.operations.name_to_id" + ] + } + ] +} \ No newline at end of file diff --git a/portality/migrate/3850_link_editor_groups_by_name/operations.py b/portality/migrate/3850_link_editor_groups_by_name/operations.py new file mode 100644 index 0000000000..2943cad131 --- /dev/null +++ b/portality/migrate/3850_link_editor_groups_by_name/operations.py @@ -0,0 +1,41 @@ +import warnings +from typing import Union + +from portality import constants, models +from portality.models import Application, Journal + + +def make_update_request(application): + if application.related_journal and not application.application_type == constants.APPLICATION_TYPE_UPDATE_REQUEST: + application.application_type = constants.APPLICATION_TYPE_UPDATE_REQUEST + return application + + +def name_to_id(model: Union[Application, Journal]): + eg_id = model.editor_group + if eg_id is None: + return model + new_val = models.EditorGroup.group_exists_by_name(eg_id) + if not new_val: + # print(f'editor group not found by name [{eg_id}]') + return model + + + # print(f'editor group [{eg_id}] -> [{new_val}]') + model.set_editor_group(new_val) + return model + + +def id_to_name(model: Union[Application, Journal]): + warnings.warn("TOBEREMOVE: rollback for debug only") + eg_id = model.editor_group + if eg_id is None: + return model + eg = models.EditorGroup.pull(eg_id) + if not eg: + # print(f'editor group not found by id [{eg_id}]') + return model + + # print(f'editor group [{eg_id}] -> [{eg.name}]') + model.set_editor_group(eg.name) + return model From cb604f6d7d242ad95edfc260a1fb1c76519720c0 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 19 Apr 2024 11:29:38 +0100 Subject: [PATCH 07/65] add TODO --- portality/api/current/data_objects/application.py | 1 + portality/models/v2/journal.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/portality/api/current/data_objects/application.py b/portality/api/current/data_objects/application.py index a88e9ff32c..88ddfe0d58 100644 --- a/portality/api/current/data_objects/application.py +++ b/portality/api/current/data_objects/application.py @@ -302,6 +302,7 @@ def editor_group(self): return self.__seamless__.get_single("admin.editor_group") def set_editor_group(self, eg): + # KTODO self.__seamless__.set_with_struct("admin.editor_group", eg) def remove_editor_group(self): diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 735a88a42d..4f4bbdc860 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -220,11 +220,17 @@ def owner_account(self): return Account.pull(self.owner) return None + # KTODO add property self.editor_group_ids + @property def editor_group(self): + # KTODO replace with self._cache_editor_group_name with query pull return self.__seamless__.get_single("admin.editor_group") + # KTODO add setter of editor_group for warning and DEBUGGING and redirect the value to self.editor_group_ids + def set_editor_group(self, eg): + # KTODO add warning and DEBUGGING and redirect the value to self.editor_group_ids self.__seamless__.set_with_struct("admin.editor_group", eg) def remove_editor_group(self): From bc49722460a7e0e1471c0c9760009d7917f3c595 Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 7 May 2024 13:24:45 +0100 Subject: [PATCH 08/65] extract find_editor_role_by_id --- portality/models/editors.py | 6 ++++++ portality/view/editor.py | 8 ++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/portality/models/editors.py b/portality/models/editors.py index 9664f06c1c..377dfec45b 100644 --- a/portality/models/editors.py +++ b/portality/models/editors.py @@ -100,6 +100,12 @@ def is_member(self, account_name): all_eds = self.associates + [self.editor] return account_name in all_eds + @classmethod + def find_editor_role_by_id(cls, editor_group_id, user_id) -> str: + # KTODO change to .pull() + eg = cls.pull_by_key("name", editor_group_id) + return "editor" if eg is not None and eg.editor == user_id else "associate_editor" + class EditorGroupQuery(object): def __init__(self, name): diff --git a/portality/view/editor.py b/portality/view/editor.py index 0ee5ed5118..5873f97b04 100644 --- a/portality/view/editor.py +++ b/portality/view/editor.py @@ -76,10 +76,7 @@ def journal_page(journal_id): abort(401) # # now check whether the user is the editor of the editor group - role = "associate_editor" - eg = models.EditorGroup.pull_by_key("name", journal.editor_group) - if eg is not None and eg.editor == current_user.id: - role = "editor" + role = models.EditorGroup.find_editor_role_by_id(journal.editor_group, current_user.id) # attempt to get a lock on the object try: @@ -133,8 +130,7 @@ def application(application_id): form_diff, current_journal = ApplicationFormXWalk.update_request_diff(ap) # Edit role is either associate_editor or editor, depending whether the user is group leader - eg = models.EditorGroup.pull_by_key("name", ap.editor_group) - role = 'editor' if eg is not None and eg.editor == current_user.id else 'associate_editor' + role = models.EditorGroup.find_editor_role_by_id(ap.editor_group, current_user.id) fc = ApplicationFormFactory.context(role, extra_param=exparam_editing_user()) if request.method == "GET": From 2e4371de54bcf02483159d670412656fde47fe8a Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 7 May 2024 13:51:46 +0100 Subject: [PATCH 09/65] extract find_group_stats --- .../tasks/async_workflow_notifications.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/portality/tasks/async_workflow_notifications.py b/portality/tasks/async_workflow_notifications.py index b0236af7ec..d48e8f6f11 100644 --- a/portality/tasks/async_workflow_notifications.py +++ b/portality/tasks/async_workflow_notifications.py @@ -233,16 +233,10 @@ def editor_notifications(emails_dict, limit=None): status_filters = [Facetview2.make_term_filter(term, status) for status in relevant_statuses] # First note - how many applications in editor's group have no associate editor assigned. - ed_app_query = EdAppQuery(status_filters) - ed_url = app.config.get("BASE_URL") + "/editor/group_applications" # Query for editor groups which have items in the required statuses, count their numbers - es = models.Suggestion.query(q=ed_app_query.query()) - group_stats = [(bucket.get("key"), bucket.get("doc_count")) for bucket in es.get("aggregations", {}).get("ed_group_counts", {}).get("buckets", [])] - - if limit is not None and isinstance(limit, int): - group_stats = group_stats[:limit] + group_stats = find_group_stats(EdAppQuery(status_filters).query(), limit) # Get the email addresses for the editor in charge of each group, Add the template to their email for (group_name, group_count) in group_stats: @@ -263,17 +257,11 @@ def editor_notifications(emails_dict, limit=None): newest_date = dates.now() - timedelta(weeks=X_WEEKS) newest_date_stamp = newest_date.strftime(FMT_DATETIME_STD) - ed_age_query = EdAgeQuery(newest_date_stamp, status_filters) - ed_fv_prefix = app.config.get('BASE_URL') + "/editor/group_applications?source=" fv_age = Facetview2.make_query(sort_parameter="last_manual_update") ed_age_url = ed_fv_prefix + Facetview2.url_encode_query(fv_age) - es = models.Suggestion.query(q=ed_age_query.query()) - group_stats = [(bucket.get("key"), bucket.get("doc_count")) for bucket in es.get("aggregations", {}).get("ed_group_counts", {}).get("buckets", [])] - - if limit is not None and isinstance(limit, int): - group_stats = group_stats[:limit] + group_stats = find_group_stats(EdAgeQuery(newest_date_stamp, status_filters).query(), limit) # Get the email addresses for the editor in charge of each group, Add the template to their email for (group_name, group_count) in group_stats: @@ -290,6 +278,15 @@ def editor_notifications(emails_dict, limit=None): _add_email_paragraph(emails_dict, ed_email, eg.editor, text) +def find_group_stats(ed_query, limit): + es = models.Suggestion.query(q=ed_query) + group_stats = [(bucket.get("key"), bucket.get("doc_count")) + for bucket in es.get("aggregations", {}).get("ed_group_counts", {}).get("buckets", [])] + if limit is not None and isinstance(limit, int): + group_stats = group_stats[:limit] + return group_stats + + def associate_editor_notifications(emails_dict, limit=None): """ Notify associates about two things: From 6e30c1b6f620ffed57cb935cbe030e5429cb397a Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 7 May 2024 14:10:09 +0100 Subject: [PATCH 10/65] editor group no longer need since no email need to be sent --- portality/forms/application_processors.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/portality/forms/application_processors.py b/portality/forms/application_processors.py index cd7fdaf5f4..9e52277eff 100644 --- a/portality/forms/application_processors.py +++ b/portality/forms/application_processors.py @@ -598,13 +598,6 @@ def finalise(self): # email managing editors if the application was newly set to 'ready' if self.source.application_status != constants.APPLICATION_STATUS_READY and self.target.application_status == constants.APPLICATION_STATUS_READY: - # Tell the ManEds who has made the status change - the editor in charge of the group - # ~~-> EditorGroup:Model~~ - editor_group_name = self.target.editor_group - editor_group_id = models.EditorGroup.group_exists_by_name(name=editor_group_name) - editor_group = models.EditorGroup.pull(editor_group_id) - editor_acc = editor_group.get_editor_account() - # record the event in the provenance tracker # ~~-> Provenance:Model~~ models.Provenance.make(current_user, "status:ready", self.target) From a4a2eb2c1d0e1d09ee7af3598d52fa767342bda5 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 10:37:08 +0100 Subject: [PATCH 11/65] remove no longer used suggestions_and_journals.js --- .../static/js/suggestions_and_journals.js | 415 ------------------ 1 file changed, 415 deletions(-) delete mode 100644 portality/static/js/suggestions_and_journals.js diff --git a/portality/static/js/suggestions_and_journals.js b/portality/static/js/suggestions_and_journals.js deleted file mode 100644 index 723f695091..0000000000 --- a/portality/static/js/suggestions_and_journals.js +++ /dev/null @@ -1,415 +0,0 @@ -jQuery(document).ready(function($) { - - ////// functions for handling edit locks /////////////////////// - - function setLockTimeout() { - var ts = $("#lock_expires").attr("data-timestamp"); - var d = new Date(ts); - var hours = d.getHours(); - var minutes = d.getMinutes(); - if (String(minutes).length == 1) { minutes = "0" + minutes } - var formatted = hours + ":" + minutes; - $("#lock_expires").html(formatted) - } - setLockTimeout(); - - function unlock(params) { - var type = params.type; - var id = params.id; - - function success_callback(data) { - var newWindow = window.open('', '_self', ''); //open the current window - window.close(); - } - - function error_callback(jqXHR, textStatus, errorThrown) { - alert("error releasing lock: " + textStatus + " " + errorThrown) - } - - $.ajax({ - type: "POST", - url: "/service/unlock/" + type + "/" + id, - contentType: "application/json", - dataType: "json", - success : success_callback, - error: error_callback - }) - } - - $("#unlock").click(function(event) { - event.preventDefault(); - var id = $(this).attr("data-id"); - var type = $(this).attr("data-type"); - unlock({type : type, id : id}) - }); - - // NOTE: this does not play well with page reloads, so not using it - //$(window).unload(function() { - // var id = $("#unlock").attr("data-id") - // var type = $("#unlock").attr("data-type") - // unlock({type : type, id : id}) - //}); - - //////////////////////////////////////////////////// - - //////////////////////////////////////////////////// - // functions for diff table - - $(".show_hide_diff_table").click(function(event) { - event.preventDefault(); - - var state = $(this).attr("data-state"); - if (state == "hidden") { - $(".form_diff").slideDown(); - $(this).html("hide").attr("data-state", "shown"); - } else if (state === "shown") { - $(".form_diff").slideUp(); - $(this).html("show").attr("data-state", "hidden"); - } - }); - - /////////////////////////////////////////////////// - - /////////////////////////////////////////////////// - // quick reject - - function showCustomRejectNote() { - $("#custom_reject_reason").show(); - } - - function hideCustomRejectNote() { - $("#custom_reject_reason").hide(); - } - - $("#submit_quick_reject").on("click", function(event) { - if ($("#reject_reason").val() == "" && $("#additional_reject_information").val() == "") { - alert("When selecting 'Other' as a reason for rejection, you must provide additional information"); - event.preventDefault(); - } - }); - - /////////////////////////////////////////////////// - - - // define a new highlight function, letting us highlight any element - // on a page - // adapted from http://stackoverflow.com/a/11589350 - jQuery.fn.highlight = function(color, fade_time) { - // some defaults - var color = color || "#F68B1F"; // the DOAJ color - var fade_time = fade_time || 1500; // milliseconds - - $(this).each(function() { - var el = $(this); - el.before("
"); - el.prev() - .width(el.width()) - .height(el.height()) - .css({ - "position": "absolute", - "background-color": color, - "opacity": ".9" - }) - .fadeOut(fade_time); - }); - }; - - // animated scrolling to an anchor - jQuery.fn.anchorAnimate = function(settings) { - - settings = jQuery.extend({ - speed : 700 - }, settings); - - return this.each(function(){ - var caller = this; - $(caller).click(function (event) { - event.preventDefault(); - var locationHref = window.location.href; - var elementClick = $(caller).attr("href"); - - var destination = $(elementClick).offset().top; - - $("html:not(:animated),body:not(:animated)").animate({ scrollTop: destination}, settings.speed, function() { - window.location.hash = elementClick; // ... but it also needs to be getting set here for the animation itself to work - }); - - setTimeout(function(){ - highlight_target(); - }, settings.speed + 50); - - return false; - }) - }) - }; - - toggle_optional_field('waiver_policy', ['#waiver_policy_url']); - toggle_optional_field('download_statistics', ['#download_statistics_url']); - toggle_optional_field('plagiarism_screening', ['#plagiarism_screening_url']); - toggle_optional_field('publishing_rights', ['#publishing_rights_url'], ["True"]); - toggle_optional_field('copyright', ['#copyright_url'], ["True"]); - toggle_optional_field('license_embedded', ['#license_embedded_url']); - toggle_optional_field('processing_charges', ['#processing_charges_amount', '#processing_charges_currency']); - toggle_optional_field('submission_charges', ['#submission_charges_amount', '#submission_charges_currency']); - toggle_optional_field('license', ['#license_checkbox'], ["Other"]); - - $('#country').select2({allowClear: true}); - $('#processing_charges_currency').select2({allowClear: true}); - $('#submission_charges_currency').select2({allowClear: true}); - - $("#keywords").select2({ - minimumInputLength: 1, - tags: [], - tokenSeparators: [","], - maximumSelectionSize: 6 - }); - - $("#languages").select2({allowClear: true}); - - autocomplete('#publisher', 'bibjson.publisher.name'); - autocomplete('#society_institution', 'bibjson.institution.name'); - autocomplete('#owner', 'id', 'account'); - autocomplete('#editor_group', 'name', 'editor_group', 1, false); - - exclusive_checkbox('digital_archiving_policy', 'No policy in place'); - exclusive_checkbox('article_identifiers', 'None'); - exclusive_checkbox('deposit_policy', 'None'); - - if ("onhashchange" in window) { - window.onhashchange = highlight_target(); - $('a.animated').anchorAnimate(); - } - - setup_subject_tree(); - if (typeof(notes_editable) !== 'undefined' && notes_editable === false) { // set by template - $('#add_note_btn').hide() - } else { - setup_remove_buttons(); - setup_add_buttons(); - } - - $("#editor_group").change(function(event) { - event.preventDefault(); - $("#editor").html("") - }); - - $(".application_journal_form").on("submit", function(event) { - $(".save-record").attr("disabled", "disabled"); - }); -}); - -function setup_subject_tree() { - $(function () { - $('#subject_tree').jstree({ - 'plugins':["checkbox","sort","search"], - 'core' : { - 'data' : lcc_jstree - }, - "checkbox" : { - "three_state" : false - }, - "search" : { - "fuzzy" : false, - "show_only_matches" : true - } - }); - }); - - $('#subject_tree') - .on('ready.jstree', function (e, data) { - var subjects = $('#subject').val() || []; - for (var i = 0; i < subjects.length; i++) { - $('#subject_tree').jstree('select_node', subjects[i]); - } - }); - - $('#subject_tree') - .on('changed.jstree', function (e, data) { - var subjects = $('#subject').val(data.selected); - }); - - - $('#subject_tree_container').prepend('
\ - \ -
\ - \ -

Selecting a subject will not automatically select its sub-categories.

\ -
\ -
'); - - var to = false; - $('#subject_tree_search').keyup(function () { - if(to) { clearTimeout(to); } - to = setTimeout(function () { - var v = $('#subject_tree_search').val(); - $('#subject_tree').jstree(true).search(v); - }, 750); - }); - - // Previously the superseded container was completely hidden, but we want to show its errors, so hide its children. - //$('#subject-container').hide(); - $('#subject-container').find("label").hide(); - $('#subject-container').find("#subject").hide(); - $('#subject-container').css('margin-bottom', 0); -} - -function exclusive_checkbox(field_name, exclusive_val) { - var doit = function() { - if (this.checked) { - $('#' + field_name + ' :checkbox:not([value="' + exclusive_val + '"])').prop('disabled', true); - $('#' + field_name + ' .extra_input_field').prop('disabled', true); - } else { - $('#' + field_name + ' :checkbox:not([value="' + exclusive_val + '"])').prop('disabled', false); - $('#' + field_name + ' .extra_input_field').prop('disabled', false); - } - }; - - $('#' + field_name + ' :checkbox[value="' + exclusive_val + '"]').each(doit); // on page load too - $('#' + field_name + ' :checkbox[value="' + exclusive_val + '"]').change(doit); // when exclusive checkbox ticked -} - -function highlight_target() { - $(window.location.hash).highlight() -} - -function setup_add_buttons() { - var customisations = { - // by container element id - container of the add more button and the fields being added - 'notes-outer-container': {'value': 'Add new note', 'id': 'add_note_btn'} - }; - - $('.addable-field-container').each(function() { - var e = $(this); - var id = e.attr('id'); - var value = customisations[id]['value'] || 'Add'; - - var thebtn = '\ -
\ - '; - - cur_number_of_notes += 1; // this doesn't get decremented in the remove button because there's no point, WTForms will understand it - // even if the ID-s go 0, 2, 7, 13 etc. - $(this).after(thefield); - if (notes_deletable) { - setup_remove_buttons(); - } - }; - - var button_handlers = { - // by button element id - 'add_note_btn': add_note_btn - }; - for (var button_id in button_handlers) { - $('#' + button_id).unbind('click'); - $('#' + button_id).click(button_handlers[button_id]); - } -} - -function setup_remove_buttons() { - $('.deletable').each(function() { - var e = $(this); - var id = e.attr('id'); - if(e.find('button[id="remove_'+id+'"]').length == 0) { - e.append('
'); - } - setup_remove_button_handler(); - }); -} - -function setup_remove_button_handler() { - $(".remove_button").unbind("click"); - $(".remove_button").click( function(event) { - event.preventDefault(); - var toremove = $(this).attr('target'); - $('#' + toremove).remove(); - }); -} - -/* -Permits Managing Editors to assign editors on the Application and Journal edit forms. - */ -function load_eds_in_group(ed_query_url) { - var ed_group_name = $("#s2id_editor_group").find('span').text(); - $.ajax({ - type : "GET", - data : {egn : ed_group_name}, - dataType: "json", - url: ed_query_url, - success: function(resp) - { - // Get the options for the drop-down from our ajax request - var assoc_options = []; - if (resp != null) - { - assoc_options = [["", "Choose an editor"]]; - - for (var i=0; i").attr("value", assoc_options[j][0]).text(assoc_options[j][1]) - ); - } - } - }) -} From 52a06b7f87e87cec16c7156f0d806b9d7de49674 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 13:16:15 +0100 Subject: [PATCH 12/65] add autocomplete_pair, support different value in id and text --- portality/dao.py | 38 +++++++++++++++++++++++++--- portality/forms/application_forms.py | 12 +++++---- portality/models/__init__.py | 6 +++-- portality/static/js/formulaic.js | 11 +++++--- portality/view/admin.py | 2 +- portality/view/doaj.py | 23 ++++++++++++++--- 6 files changed, 74 insertions(+), 18 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index c6ad5d706a..07e01863ac 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -7,7 +7,7 @@ from collections import UserDict from copy import deepcopy from datetime import timedelta -from typing import List, Iterable, Union, Dict +from typing import List, Iterable, Union, Dict, TypedDict import elasticsearch @@ -25,6 +25,10 @@ ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_\.]+?\] in order to sort on.*', re.DOTALL) CONTENT_TYPE_JSON = {'Content-Type': 'application/json'} +class IdText(TypedDict): + id: str + text: str + class ElasticSearchWriteException(Exception): pass @@ -797,7 +801,7 @@ def wildcard_autocomplete_query(cls, field, substring, before=True, after=True, return cls.send_query(q.query()) @classmethod - def advanced_autocomplete(cls, filter_field, facet_field, substring, size=5, prefix_only=True): + def advanced_autocomplete(cls, filter_field, facet_field, substring, size=5, prefix_only=True) -> List[IdText]: analyzed = True if " " in substring: analyzed = False @@ -818,7 +822,7 @@ def advanced_autocomplete(cls, filter_field, facet_field, substring, size=5, pre return result @classmethod - def autocomplete(cls, field, prefix, size=5): + def autocomplete(cls, field, prefix, size=5) -> List[IdText]: res = None # if there is a space in the prefix, the prefix query won't work, so we fall back to a wildcard # we only do this if we have to, because the wildcard query is a little expensive @@ -835,6 +839,15 @@ def autocomplete(cls, field, prefix, size=5): result.append({"id": term['key'], "text": term['key']}) return result + @classmethod + def autocomplete_pair(cls, query_field, query_prefix, id_field, text_field, size=5) -> List[IdText]: + query = AutocompletePairQuery(query_field, query_prefix, + source_fields=[id_field, text_field], + size=size).query() + objs = cls.q2obj(q=query) + return [{"id": getattr(obj, id_field), "text": getattr(obj, text_field)} for obj in objs] + + @classmethod def q2obj(cls, **kwargs) -> List['Self']: extra_trace_info = '' @@ -1060,6 +1073,25 @@ def query(self): } +class AutocompletePairQuery(object): + def __init__(self, query_field, query_value, source_fields=None, size=5): + self.query_field = query_field + self.query_value = query_value + self.source_fields = source_fields + self.size = size + + def query(self): + query = { + "query": { + "prefix": {self.query_field: self.query_value} + }, + "size": self.size + } + if self.source_fields is not None: + query["_source"] = self.source_fields + return query + + ######################################################################### # A query handler that knows how to speak facetview2 ######################################################################### diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py index 3ac5f845b9..091558443d 100644 --- a/portality/forms/application_forms.py +++ b/portality/forms/application_forms.py @@ -1720,7 +1720,8 @@ class FieldDefinitions: "label": "Group", "input": "text", "widgets": [ - {"autocomplete": {"type" : "editor_group", "field": "name", "include" : False}} # ~~^-> Autocomplete:FormWidget~~ + {"autocomplete": {"type" : "editor_group", "field": "name", "include" : False, + "id_field": "id" }} # ~~^-> Autocomplete:FormWidget~~ ], "contexts" : { "editor" : { @@ -1728,7 +1729,8 @@ class FieldDefinitions: }, "admin" : { "widgets" : [ - {"autocomplete": {"type": "editor_group", "field": "name", "include" : False}}, # ~~^-> Autocomplete:FormWidget~~ + {"autocomplete": {"type": "editor_group", "field": "name", "include" : False, + "id_field": "id"}}, # ~~^-> Autocomplete:FormWidget~~ {"load_editors" : {"field" : "editor"}} ] } @@ -2561,11 +2563,11 @@ def editor_choices(field, formulaic_context): if wtf is None: return [{"display" : "", "value" : ""}] - editor_group_name = wtf.data - if editor_group_name is None: + editor_group_id = wtf.data + if editor_group_id is None: return [{"display" : "", "value" : ""}] else: - eg = EditorGroup.pull_by_key("name", editor_group_name) + eg = EditorGroup.pull(editor_group_id) if eg is not None: editors = [eg.editor] editors += eg.associates diff --git a/portality/models/__init__.py b/portality/models/__init__.py index eea1859a70..9609c3ce50 100644 --- a/portality/models/__init__.py +++ b/portality/models/__init__.py @@ -1,4 +1,6 @@ -from typing import Any +from typing import Any, Optional + +from portality.dao import DomainObject # import the versioned objects, so that the current version is the default one from portality.models.v2 import shared_structs from portality.models.v2.bibjson import JournalLikeBibJSON @@ -32,7 +34,7 @@ import sys -def lookup_model(name='', capitalize=True, split_on="_"): +def lookup_model(name='', capitalize=True, split_on="_") -> Optional[DomainObject]: parts = name.split(split_on) if capitalize: parts = [p.capitalize() for p in parts] diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index 0c5932c3ce..16efe1d126 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -2151,14 +2151,17 @@ var formulaic = { let mininput = this.params.min_input === undefined ? 3 : this.params.min_input; let include_input = this.params.include === undefined ? true : this.params.include; let allow_clear = this.params.allow_clear_input === undefined ? true : this.params.allow_clear_input; + const id_field = this.params.id_field; + let url = `${current_scheme}//${current_domain}/autocomplete/${doc_type}/${doc_field}`; + if (id_field) { + url += `/${id_field}`; + } let ajax = { - url: current_scheme + "//" + current_domain + "/autocomplete/" + doc_type + "/" + doc_field, + url: url, dataType: 'json', data: function (term, page) { - return { - q: term - }; + return { q: term }; }, results: function (data, page) { return { results: data["suggestions"] }; diff --git a/portality/view/admin.py b/portality/view/admin.py index 484441a02a..84ce91b1c1 100644 --- a/portality/view/admin.py +++ b/portality/view/admin.py @@ -640,7 +640,7 @@ def user_autocomplete(): @ssl_required def eg_associates_dropdown(): egn = request.values.get("egn") - eg = models.EditorGroup.pull_by_key("name", egn) + eg = models.EditorGroup.pull(egn) if eg is not None: editors = [eg.editor] diff --git a/portality/view/doaj.py b/portality/view/doaj.py index a10217534c..19fed92987 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -229,22 +229,23 @@ def get_from_local_store(container, filename): file_handle = localStore.get(container, filename) return send_file(file_handle, mimetype="application/octet-stream", as_attachment=True, attachment_filename=filename) +def _no_suggestions_response(): + return jsonify({'suggestions': [{"id": "", "text": "No results found"}]}) @blueprint.route('/autocomplete//', methods=["GET", "POST"]) def autocomplete(doc_type, field_name): prefix = request.args.get('q', '') if not prefix: - return jsonify({'suggestions': [{"id": "", "text": "No results found"}]}) # select2 does not understand 400, which is the correct code here... + return _no_suggestions_response() # select2 does not understand 400, which is the correct code here... m = models.lookup_model(doc_type) if not m: - return jsonify({'suggestions': [{"id": "", "text": "No results found"}]}) # select2 does not understand 404, which is the correct code here... + return _no_suggestions_response() # select2 does not understand 404, which is the correct code here... size = request.args.get('size', 5) filter_field = app.config.get("AUTOCOMPLETE_ADVANCED_FIELD_MAPS", {}).get(field_name) - suggs = [] if filter_field is None: suggs = m.autocomplete(field_name, prefix, size=size) else: @@ -255,6 +256,22 @@ def autocomplete(doc_type, field_name): # http://flask.pocoo.org/docs/security/#json-security +@blueprint.route('/autocomplete///', methods=["GET", "POST"]) +def autocomplete_pair(doc_type, field_name, id_field): + prefix = request.args.get('q', '') + if not prefix: + return _no_suggestions_response() # select2 does not understand 400, which is the correct code here... + + m = models.lookup_model(doc_type) + if not m: + return _no_suggestions_response() # select2 does not understand 404, which is the correct code here... + + size = request.args.get('size', 5) + suggs = m.autocomplete_pair(field_name, prefix.lower(), id_field, field_name, size=size) + return jsonify({'suggestions': suggs}) + + + def find_toc_journal_by_identifier(identifier): if identifier is None: abort(404) From 332dbc2f5db3545c4037c9d0cb5f7f7bf47511bb Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 13:27:38 +0100 Subject: [PATCH 13/65] extract select2Param --- portality/static/js/autocomplete.js | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/portality/static/js/autocomplete.js b/portality/static/js/autocomplete.js index 48bd9af870..d7a531786c 100644 --- a/portality/static/js/autocomplete.js +++ b/portality/static/js/autocomplete.js @@ -17,30 +17,21 @@ function autocomplete(selector, doc_field, doc_type, mininput, include_input, al return { results: data["suggestions"] }; } }; - var csc = function(term) {return {"id":term, "text": term};}; var initSel = function (element, callback) { var data = {id: element.val(), text: element.val()}; callback(data); }; + let select2Param = { + minimumInputLength: mininput, + ajax: ajax, + initSelection : initSel, + placeholder: "Start typing…", + allowClear: allow_clear + }; if (include_input) { // apply the create search choice - $(selector).select2({ - minimumInputLength: mininput, - ajax: ajax, - createSearchChoice: csc, - initSelection : initSel, - placeholder: "Start typing…", - allowClear: allow_clear - }); - } else { - // go without the create search choice option - $(selector).select2({ - minimumInputLength: mininput, - ajax: ajax, - initSelection : initSel, - placeholder: "Start typing…", - allowClear: allow_clear - }); + select2Param.createSearchChoice = function(term) {return {"id":term, "text": term};}; } + $(selector).select2(select2Param); } From 3c30ebfb8b2bd8ca7fe3e7696420267a8522a350 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 13:34:39 +0100 Subject: [PATCH 14/65] Revert "extract select2Param" This reverts commit 332dbc2f5db3545c4037c9d0cb5f7f7bf47511bb. --- portality/static/js/autocomplete.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/portality/static/js/autocomplete.js b/portality/static/js/autocomplete.js index d7a531786c..48bd9af870 100644 --- a/portality/static/js/autocomplete.js +++ b/portality/static/js/autocomplete.js @@ -17,21 +17,30 @@ function autocomplete(selector, doc_field, doc_type, mininput, include_input, al return { results: data["suggestions"] }; } }; + var csc = function(term) {return {"id":term, "text": term};}; var initSel = function (element, callback) { var data = {id: element.val(), text: element.val()}; callback(data); }; - let select2Param = { - minimumInputLength: mininput, - ajax: ajax, - initSelection : initSel, - placeholder: "Start typing…", - allowClear: allow_clear - }; if (include_input) { // apply the create search choice - select2Param.createSearchChoice = function(term) {return {"id":term, "text": term};}; + $(selector).select2({ + minimumInputLength: mininput, + ajax: ajax, + createSearchChoice: csc, + initSelection : initSel, + placeholder: "Start typing…", + allowClear: allow_clear + }); + } else { + // go without the create search choice option + $(selector).select2({ + minimumInputLength: mininput, + ajax: ajax, + initSelection : initSel, + placeholder: "Start typing…", + allowClear: allow_clear + }); } - $(selector).select2(select2Param); } From 6745be5e35f3fc815590c5af6067ed099c305276 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 13:39:07 +0100 Subject: [PATCH 15/65] extract select2Param --- portality/static/js/formulaic.js | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index 16efe1d126..56ba722628 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -2168,7 +2168,6 @@ var formulaic = { } }; - var csc = function(term) {return {"id":term, "text": term};}; var initSel = function (element, callback) { var data = {id: element.val(), text: element.val()}; @@ -2179,26 +2178,18 @@ var formulaic = { $(selector).on("focus", formulaic.widgets._select2_shift_focus); + let select2Param = { + minimumInputLength: mininput, + ajax: ajax, + initSelection : initSel, + allowClear: allow_clear, + width: 'resolve' + }; if (include_input) { // apply the create search choice - $(selector).select2({ - minimumInputLength: mininput, - ajax: ajax, - createSearchChoice: csc, - initSelection : initSel, - allowClear: allow_clear, - width: 'resolve' - }); - } else { - // go without the create search choice option - $(selector).select2({ - minimumInputLength: mininput, - ajax: ajax, - initSelection : initSel, - allowClear: allow_clear, - width: 'resolve' - }); + select2Param.createSearchChoice = (term) => ({"id": term, "text": term}); } + $(selector).select2(select2Param); $(selector).on("focus", formulaic.widgets._select2_shift_focus); }; From e23f944cfc8cc102d17df0dbb4bf7c86ddfe6640 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 14:27:56 +0100 Subject: [PATCH 16/65] remove if else no longer meaningful --- portality/lib/formulaic.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/portality/lib/formulaic.py b/portality/lib/formulaic.py index 423daea97c..a3b6aaaee6 100644 --- a/portality/lib/formulaic.py +++ b/portality/lib/formulaic.py @@ -776,11 +776,7 @@ def render_form_control(self, custom_args=None, wtfinst=None): for k, v in custom_args.items(): kwargs[k] = v - wtf = None - if wtfinst is not None: - wtf = wtfinst - else: - wtf = self.wtfield + wtf = self.wtfield if add_data_as_choice and hasattr(wtf, "choices") and wtf.data not in [c[0] for c in wtf.choices]: wtf.choices += [(wtf.data, wtf.data)] From cab97438de35630e0119cbabadf80978e08b6269 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 8 May 2024 14:29:40 +0100 Subject: [PATCH 17/65] Revert "remove if else no longer meaningful" This reverts commit e23f944cfc8cc102d17df0dbb4bf7c86ddfe6640. --- portality/lib/formulaic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portality/lib/formulaic.py b/portality/lib/formulaic.py index a3b6aaaee6..423daea97c 100644 --- a/portality/lib/formulaic.py +++ b/portality/lib/formulaic.py @@ -776,7 +776,11 @@ def render_form_control(self, custom_args=None, wtfinst=None): for k, v in custom_args.items(): kwargs[k] = v - wtf = self.wtfield + wtf = None + if wtfinst is not None: + wtf = wtfinst + else: + wtf = self.wtfield if add_data_as_choice and hasattr(wtf, "choices") and wtf.data not in [c[0] for c in wtf.choices]: wtf.choices += [(wtf.data, wtf.data)] From 2c0735349a907d0c5eb5713bcf8a9544ea34f3ac Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 12:05:35 +0100 Subject: [PATCH 18/65] add autocomplete_text_mapping --- portality/dao.py | 33 ++++++++++++++++++++++++++- portality/static/js/formulaic.js | 21 ++++++++++++++++-- portality/view/doaj.py | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index 07e01863ac..a4ca5d6202 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -7,7 +7,7 @@ from collections import UserDict from copy import deepcopy from datetime import timedelta -from typing import List, Iterable, Union, Dict, TypedDict +from typing import List, Iterable, Union, Dict, TypedDict, Optional import elasticsearch @@ -25,6 +25,7 @@ ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_\.]+?\] in order to sort on.*', re.DOTALL) CONTENT_TYPE_JSON = {'Content-Type': 'application/json'} + class IdText(TypedDict): id: str text: str @@ -847,6 +848,20 @@ def autocomplete_pair(cls, query_field, query_prefix, id_field, text_field, size objs = cls.q2obj(q=query) return [{"id": getattr(obj, id_field), "text": getattr(obj, text_field)} for obj in objs] + @classmethod + def get_target_value(cls, query, field_name) -> Optional[str]: + query['_source'] = [field_name] + query['size'] = 2 + res = cls.query(q=query) + size = res.get('hits', {}).get('total', {}).get('value', 0) + + if size > 1: + app.logger.debug("More than one record found for query {q}".format(q=json.dumps(query, indent=2))) + + if size > 0: + return res['hits']['hits'][0]['_source'].get(field_name) + + return None @classmethod def q2obj(cls, **kwargs) -> List['Self']: @@ -1139,6 +1154,22 @@ def url_encode_query(query): return urllib.parse.quote(json.dumps(query).replace(' ', '')) +######################################################################### +# id to text Queries +######################################################################### + +class IdTextTermQuery: + def __init__(self, id_field, id_value, field_name): + self.id_field = id_field + self.id_value = id_value + self.field_name = field_name + + def query(self): + return { + "query": {"term": {self.id_field: self.id_value}} + } + + def patch_model_for_bulk(obj: DomainObject): obj.data['es_type'] = obj.__type__ obj.data['id'] = obj.makeid() diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index 56ba722628..2df4cfb642 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -2170,8 +2170,25 @@ var formulaic = { var initSel = function (element, callback) { - var data = {id: element.val(), text: element.val()}; - callback(data); + function setIdText(id, text) { + callback({id: id, text: text || id}); + } + const eleVal = element.val(); + + if (!id_field) { + setIdText(eleVal, eleVal); + return; + } + + $.ajax({ + type : "GET", + data : {id : eleVal}, + dataType: "json", + url: `${current_scheme}//${current_domain}/autocomplete-text/${doc_type}/${doc_field}/${id_field}`, + success: function(resp) { + setIdText(eleVal, resp?.text); + } + }) }; let selector = "[name='" + this.fieldDef.name + "']"; diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 19fed92987..12232687ed 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -3,6 +3,7 @@ import urllib.error import urllib.parse import urllib.request +from typing import Optional from flask import Blueprint, request, make_response from flask import render_template, abort, redirect, url_for, send_file, jsonify @@ -13,6 +14,7 @@ from portality import models from portality import store from portality.core import app +from portality.dao import IdTextTermQuery from portality.decorators import ssl_required, api_key_required from portality.forms.application_forms import JournalFormFactory from portality.lcc import lcc_jstree @@ -229,9 +231,11 @@ def get_from_local_store(container, filename): file_handle = localStore.get(container, filename) return send_file(file_handle, mimetype="application/octet-stream", as_attachment=True, attachment_filename=filename) + def _no_suggestions_response(): return jsonify({'suggestions': [{"id": "", "text": "No results found"}]}) + @blueprint.route('/autocomplete//', methods=["GET", "POST"]) def autocomplete(doc_type, field_name): prefix = request.args.get('q', '') @@ -271,6 +275,40 @@ def autocomplete_pair(doc_type, field_name, id_field): return jsonify({'suggestions': suggs}) +@blueprint.route('/autocomplete-text///', methods=["GET", "POST"]) +def autocomplete_text_mapping(doc_type, field_name, id_field): + """ + This route is used by the autocomplete widget to get the text value by id + """ + + id_value = request.args.get('id') + text = id_text_mapping(doc_type, field_name, id_field, id_value) + return jsonify({'id': id_value, 'text': text}) + + +def id_text_mapping(doc_type, field_name, id_field, id_value) -> Optional[str]: + + query_factory_mapping = { + ('editor_group', 'id', 'name', ): lambda: IdTextTermQuery(id_field, id_value, field_name).query(), + } + query_factory = query_factory_mapping.get((doc_type, id_field, field_name)) + if not query_factory: + app.logger.warning(f"Unsupported id_text_mapping for " + f"doc_type[{doc_type}], field_name[{field_name}], id_field[{id_field}]") + return None + + if not id_value: + return None + + m = models.lookup_model(doc_type) + if not m: + app.logger.warning(f"model not found for doc_type[{doc_type}]") + return None + + query = query_factory() + return m.get_target_value(query, field_name) + + def find_toc_journal_by_identifier(identifier): if identifier is None: From 60399092772504403f20b7e4384e2ec22b60713c Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 12:11:31 +0100 Subject: [PATCH 19/65] avoid unnecessary text value requests --- portality/static/js/formulaic.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index 2df4cfb642..15af4501e2 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -2175,7 +2175,11 @@ var formulaic = { } const eleVal = element.val(); - if (!id_field) { + if (!id_field || !eleVal) { + /* + no need to query text value if id_field is not provided (not id text mapping) + or if the element value is empty + */ setIdText(eleVal, eleVal); return; } From 2ca27460a05954a26ab5a93f8b37da540c8aaf57 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 12:45:47 +0100 Subject: [PATCH 20/65] find editor_group by pull id --- portality/bll/services/authorisation.py | 4 ++-- .../application_editor_group_assigned_notify.py | 3 +-- .../consumers/application_maned_ready_notify.py | 2 +- .../consumers/journal_discontinuing_soon_notify.py | 2 +- .../journal_editor_group_assigned_notify.py | 3 +-- portality/forms/validate.py | 8 ++++---- portality/notifications/application_emails.py | 6 +----- portality/tasks/async_workflow_notifications.py | 14 ++++++++------ portality/tasks/journal_bulk_edit.py | 2 +- 9 files changed, 20 insertions(+), 24 deletions(-) diff --git a/portality/bll/services/authorisation.py b/portality/bll/services/authorisation.py index 5be4e65c77..dd6bf52083 100644 --- a/portality/bll/services/authorisation.py +++ b/portality/bll/services/authorisation.py @@ -74,7 +74,7 @@ def can_edit_application(self, account, application): if not application.editor_group: return False - eg = models.EditorGroup.pull_by_key("name", application.editor_group) + eg = models.EditorGroup.pull(application.editor_group) if eg is not None and eg.editor == account.id: return True @@ -129,7 +129,7 @@ def can_edit_journal(self, account: models.Account, journal: models.Journal): passed = True # now check whether the user is the editor of the editor group - eg = models.EditorGroup.pull_by_key("name", journal.editor_group) # ~~->EditorGroup:Model~~ + eg = models.EditorGroup.pull(journal.editor_group) # ~~->EditorGroup:Model~~ if eg is not None and eg.editor == account.id: passed = True diff --git a/portality/events/consumers/application_editor_group_assigned_notify.py b/portality/events/consumers/application_editor_group_assigned_notify.py index 22b277d283..c6771d1a2b 100644 --- a/portality/events/consumers/application_editor_group_assigned_notify.py +++ b/portality/events/consumers/application_editor_group_assigned_notify.py @@ -27,8 +27,7 @@ def consume(cls, event): if not application.editor_group: return - editor_group = models.EditorGroup.pull_by_key("name", application.editor_group) - + editor_group = models.EditorGroup.pull(application.editor_group) if not editor_group.editor: raise exceptions.NoSuchPropertyException("Editor Group {x} does not have property `editor`".format(x=editor_group.id)) diff --git a/portality/events/consumers/application_maned_ready_notify.py b/portality/events/consumers/application_maned_ready_notify.py index 58b22384b8..1825cfaba3 100644 --- a/portality/events/consumers/application_maned_ready_notify.py +++ b/portality/events/consumers/application_maned_ready_notify.py @@ -29,7 +29,7 @@ def consume(cls, event): return - eg = models.EditorGroup.pull_by_key("name", application.editor_group) + eg = models.EditorGroup.pull(application.editor_group) managing_editor = eg.maned if not managing_editor: return diff --git a/portality/events/consumers/journal_discontinuing_soon_notify.py b/portality/events/consumers/journal_discontinuing_soon_notify.py index afeeb8c671..42a0d6aaa4 100644 --- a/portality/events/consumers/journal_discontinuing_soon_notify.py +++ b/portality/events/consumers/journal_discontinuing_soon_notify.py @@ -32,7 +32,7 @@ def consume(cls, event): if not journal.editor_group: return - eg = models.EditorGroup.pull_by_key("name", journal.editor_group) + eg = models.EditorGroup.pull(journal.editor_group) managing_editor = eg.maned if not managing_editor: return diff --git a/portality/events/consumers/journal_editor_group_assigned_notify.py b/portality/events/consumers/journal_editor_group_assigned_notify.py index 217a404b80..f7c5c2e6d2 100644 --- a/portality/events/consumers/journal_editor_group_assigned_notify.py +++ b/portality/events/consumers/journal_editor_group_assigned_notify.py @@ -28,8 +28,7 @@ def consume(cls, event): if not journal.editor_group: return - editor_group = models.EditorGroup.pull_by_key("name", journal.editor_group) - + editor_group = models.EditorGroup.pull(journal.editor_group) if not editor_group.editor: raise exceptions.NoSuchPropertyException("Editor Group {x} does not have property `editor`".format(x=editor_group.id)) diff --git a/portality/forms/validate.py b/portality/forms/validate.py index c7dd02cda2..1cfeb6edc0 100644 --- a/portality/forms/validate.py +++ b/portality/forms/validate.py @@ -505,13 +505,13 @@ def __call__(self, form, field): # lifted from from formcontext editor = field.data if editor is not None and editor != "": - editor_group_name = group_field.data - if editor_group_name is not None and editor_group_name != "": - eg = EditorGroup.pull_by_key("name", editor_group_name) + editor_group_id = group_field.data + if editor_group_id is not None and editor_group_id != "": + eg = EditorGroup.pull(editor_group_id) if eg is not None: if eg.is_member(editor): return # success - an editor group was found and our editor was in it - raise validators.ValidationError("Editor '{0}' not found in editor group '{1}'".format(editor, editor_group_name)) + raise validators.ValidationError("Editor '{0}' not found in editor group '{1}'".format(editor, editor_group_id)) else: raise validators.ValidationError("An editor has been assigned without an editor group") diff --git a/portality/notifications/application_emails.py b/portality/notifications/application_emails.py index c0c65d92dd..80e093e412 100644 --- a/portality/notifications/application_emails.py +++ b/portality/notifications/application_emails.py @@ -5,8 +5,6 @@ from portality import models, app_email, constants from portality.core import app from portality.dao import Facetview2 -from portality.ui.messages import Messages -from portality.lib import dates def send_editor_completed_email(application): @@ -18,9 +16,7 @@ def send_editor_completed_email(application): url_for_application = url_root + url_for("editor.group_suggestions", source=string_id_query) # This is to the editor in charge of this application's assigned editor group - editor_group_name = application.editor_group - editor_group_id = models.EditorGroup.group_exists_by_name(name=editor_group_name) - editor_group = models.EditorGroup.pull(editor_group_id) + editor_group = models.EditorGroup.pull(application.editor_group) editor_acc = editor_group.get_editor_account() editor_id = editor_acc.id diff --git a/portality/tasks/async_workflow_notifications.py b/portality/tasks/async_workflow_notifications.py index d48e8f6f11..fd518a958a 100644 --- a/portality/tasks/async_workflow_notifications.py +++ b/portality/tasks/async_workflow_notifications.py @@ -239,9 +239,9 @@ def editor_notifications(emails_dict, limit=None): group_stats = find_group_stats(EdAppQuery(status_filters).query(), limit) # Get the email addresses for the editor in charge of each group, Add the template to their email - for (group_name, group_count) in group_stats: + for (group_id, group_count) in group_stats: # get editor group object by name - eg = models.EditorGroup.pull_by_key("name", group_name) + eg = models.EditorGroup.pull(group_id) if eg is None: continue @@ -249,7 +249,8 @@ def editor_notifications(emails_dict, limit=None): editor = eg.get_editor_account() ed_email = editor.email - text = render_template('email/workflow_reminder_fragments/editor_groupcount_frag', num=group_count, ed_group=group_name, url=ed_url) + text = render_template('email/workflow_reminder_fragments/editor_groupcount_frag', + num=group_count, ed_group=group_id, url=ed_url) _add_email_paragraph(emails_dict, ed_email, eg.editor, text) # Second note - records within editor group not touched for so long @@ -264,9 +265,9 @@ def editor_notifications(emails_dict, limit=None): group_stats = find_group_stats(EdAgeQuery(newest_date_stamp, status_filters).query(), limit) # Get the email addresses for the editor in charge of each group, Add the template to their email - for (group_name, group_count) in group_stats: + for (group_id, group_count) in group_stats: # get editor group object by name - eg = models.EditorGroup.pull_by_key("name", group_name) + eg = models.EditorGroup.pull(group_id) if eg is None: continue @@ -274,7 +275,8 @@ def editor_notifications(emails_dict, limit=None): editor = eg.get_editor_account() ed_email = editor.email - text = render_template('email/workflow_reminder_fragments/editor_age_frag', num=group_count, ed_group=group_name, url=ed_age_url, x_weeks=X_WEEKS) + text = render_template('email/workflow_reminder_fragments/editor_age_frag', + num=group_count, ed_group=group_id, url=ed_age_url, x_weeks=X_WEEKS) _add_email_paragraph(emails_dict, ed_email, eg.editor, text) diff --git a/portality/tasks/journal_bulk_edit.py b/portality/tasks/journal_bulk_edit.py index 3e39d4388e..3f31842e57 100644 --- a/portality/tasks/journal_bulk_edit.py +++ b/portality/tasks/journal_bulk_edit.py @@ -109,7 +109,7 @@ def run(self): # FIXME: this is a bit of a stop-gap, pending a more substantial referential-integrity-like solution # if the editor group is not being changed, validate that the editor is actually in the editor group, # and if not, unset them - eg = models.EditorGroup.pull_by_key("name", j.editor_group) + eg = models.EditorGroup.pull(j.editor_group) if eg is not None: all_eds = eg.associates + [eg.editor] if j.editor not in all_eds: From 590d128293325a996288e5f725616537cffed780 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 13:03:50 +0100 Subject: [PATCH 21/65] migrate group_exists_by_name usage --- .../events/consumers/application_editor_completed_notify.py | 4 +--- .../events/consumers/application_editor_inprogress_notify.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/portality/events/consumers/application_editor_completed_notify.py b/portality/events/consumers/application_editor_completed_notify.py index cd90c8bb51..86f14f9ed7 100644 --- a/portality/events/consumers/application_editor_completed_notify.py +++ b/portality/events/consumers/application_editor_completed_notify.py @@ -39,9 +39,7 @@ def consume(cls, event): associate_editor = event.who # Notification is to the editor in charge of this application's assigned editor group - editor_group_name = application.editor_group - editor_group_id = models.EditorGroup.group_exists_by_name(name=editor_group_name) - eg = models.EditorGroup.pull(editor_group_id) + eg = models.EditorGroup.pull(application.editor_group) group_editor = eg.editor if not group_editor: diff --git a/portality/events/consumers/application_editor_inprogress_notify.py b/portality/events/consumers/application_editor_inprogress_notify.py index 0c929e3631..cf3f9a456f 100644 --- a/portality/events/consumers/application_editor_inprogress_notify.py +++ b/portality/events/consumers/application_editor_inprogress_notify.py @@ -35,9 +35,7 @@ def consume(cls, event): return # Notification is to the editor in charge of this application's assigned editor group - editor_group_name = application.editor_group - editor_group_id = models.EditorGroup.group_exists_by_name(name=editor_group_name) - eg = models.EditorGroup.pull(editor_group_id) + eg = models.EditorGroup.pull(application.editor_group) group_editor = eg.editor if not group_editor: From d93fa5149b95e350c0d881f620cb4a7872ad6fc0 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 13:58:52 +0100 Subject: [PATCH 22/65] add comments --- portality/migrate/3850_link_editor_groups_by_name/operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/portality/migrate/3850_link_editor_groups_by_name/operations.py b/portality/migrate/3850_link_editor_groups_by_name/operations.py index 2943cad131..df1e30e520 100644 --- a/portality/migrate/3850_link_editor_groups_by_name/operations.py +++ b/portality/migrate/3850_link_editor_groups_by_name/operations.py @@ -27,6 +27,7 @@ def name_to_id(model: Union[Application, Journal]): def id_to_name(model: Union[Application, Journal]): + # for rollback for only warnings.warn("TOBEREMOVE: rollback for debug only") eg_id = model.editor_group if eg_id is None: From 36877e5d97d6702d7aebe816931017252e1a2bf5 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 9 May 2024 16:45:25 +0100 Subject: [PATCH 23/65] fix test cases for editor_group.id --- .../application_processors/test_editor_journal_review.py | 8 ++++---- .../application_processors/test_maned_journal_review.py | 8 ++++---- .../test_application_editor_completed_notify.py | 3 ++- .../test_application_editor_group_assigned_notify.py | 6 ++++-- .../test_application_editor_inprogress_notify.py | 3 ++- .../test_application_maned_ready_notify.py | 3 ++- .../test_journal_discontinuing_soon_notify.py | 8 ++++---- .../test_journal_editor_group_assigned_notify.py | 6 ++++-- doajtest/unit/test_fc_editor_app_review.py | 8 ++++---- doajtest/unit/test_fc_maned_app_review.py | 6 +++--- 10 files changed, 33 insertions(+), 26 deletions(-) diff --git a/doajtest/unit/application_processors/test_editor_journal_review.py b/doajtest/unit/application_processors/test_editor_journal_review.py index 1291cb02c9..b09126425b 100644 --- a/doajtest/unit/application_processors/test_editor_journal_review.py +++ b/doajtest/unit/application_processors/test_editor_journal_review.py @@ -13,7 +13,7 @@ ##################################################################### @classmethod -def editor_group_pull(cls, field, value): +def editor_group_pull(cls, value): eg = models.EditorGroup() eg.set_editor("eddie") eg.set_associates(["associate", "assan"]) @@ -39,15 +39,15 @@ class TestEditorJournalReview(DoajTestCase): def setUp(self): super(TestEditorJournalReview, self).setUp() - self.editor_group_pull = models.EditorGroup.pull_by_key - models.EditorGroup.pull_by_key = editor_group_pull + self.editor_group_pull = models.EditorGroup.pull + models.EditorGroup.pull = editor_group_pull self.old_lookup_code = lcc.lookup_code lcc.lookup_code = mock_lookup_code def tearDown(self): super(TestEditorJournalReview, self).tearDown() - models.EditorGroup.pull_by_key = self.editor_group_pull + models.EditorGroup.pull = self.editor_group_pull lcc.lookup_code = self.old_lookup_code diff --git a/doajtest/unit/application_processors/test_maned_journal_review.py b/doajtest/unit/application_processors/test_maned_journal_review.py index 07badcbabe..d03419e76d 100644 --- a/doajtest/unit/application_processors/test_maned_journal_review.py +++ b/doajtest/unit/application_processors/test_maned_journal_review.py @@ -17,7 +17,7 @@ ##################################################################### @classmethod -def editor_group_pull(cls, field, value): +def editor_group_pull(cls, value): eg = models.EditorGroup() eg.set_editor("eddie") eg.set_associates(["associate", "assan"]) @@ -35,15 +35,15 @@ class TestManEdJournalReview(DoajTestCase): def setUp(self): super(TestManEdJournalReview, self).setUp() - self.editor_group_pull = models.EditorGroup.pull_by_key - models.EditorGroup.pull_by_key = editor_group_pull + self.editor_group_pull = models.EditorGroup.pull + models.EditorGroup.pull = editor_group_pull self.old_lookup_code = lcc.lookup_code lcc.lookup_code = mock_lookup_code def tearDown(self): super(TestManEdJournalReview, self).tearDown() - models.EditorGroup.pull_by_key = self.editor_group_pull + models.EditorGroup.pull = self.editor_group_pull lcc.lookup_code = self.old_lookup_code def test_01_maned_review_success(self): diff --git a/doajtest/unit/event_consumers/test_application_editor_completed_notify.py b/doajtest/unit/event_consumers/test_application_editor_completed_notify.py index 436b60559d..568abde415 100644 --- a/doajtest/unit/event_consumers/test_application_editor_completed_notify.py +++ b/doajtest/unit/event_consumers/test_application_editor_completed_notify.py @@ -41,7 +41,8 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name("test group") + eg.set_id(app.editor_group) eg.set_editor(acc.id) eg.save(blocking=True) diff --git a/doajtest/unit/event_consumers/test_application_editor_group_assigned_notify.py b/doajtest/unit/event_consumers/test_application_editor_group_assigned_notify.py index 17e0c74b0e..30dbd031f3 100644 --- a/doajtest/unit/event_consumers/test_application_editor_group_assigned_notify.py +++ b/doajtest/unit/event_consumers/test_application_editor_group_assigned_notify.py @@ -37,7 +37,8 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name("test group") + eg.set_id(app.editor_group) eg.set_editor("editor") eg.save(blocking=True) @@ -67,7 +68,8 @@ def test_consume_fail(self): # app.save(blocking=True) eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name('test group') + eg.set_id(app.editor_group) eg.save(blocking=True) event = models.Event(constants.EVENT_APPLICATION_EDITOR_GROUP_ASSIGNED, context={"application": app.data}) diff --git a/doajtest/unit/event_consumers/test_application_editor_inprogress_notify.py b/doajtest/unit/event_consumers/test_application_editor_inprogress_notify.py index 215d36538c..b9fcab1953 100644 --- a/doajtest/unit/event_consumers/test_application_editor_inprogress_notify.py +++ b/doajtest/unit/event_consumers/test_application_editor_inprogress_notify.py @@ -45,7 +45,8 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name('test group') + eg.set_id(app.editor_group) eg.set_editor(acc.id) eg.save(blocking=True) diff --git a/doajtest/unit/event_consumers/test_application_maned_ready_notify.py b/doajtest/unit/event_consumers/test_application_maned_ready_notify.py index a11967aef0..ca929d3122 100644 --- a/doajtest/unit/event_consumers/test_application_maned_ready_notify.py +++ b/doajtest/unit/event_consumers/test_application_maned_ready_notify.py @@ -41,7 +41,8 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name("test group") + eg.set_id(app.editor_group) eg.set_maned(acc.id) eg.save(blocking=True) diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py index f4f70f2f78..3b00acc18c 100644 --- a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py +++ b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py @@ -14,7 +14,7 @@ def pull_application(cls, id): return app @classmethod -def pull_by_key(cls, key, value): +def pull_editor_group(cls, value): ed = models.EditorGroup() acc = models.Account() acc.set_id('testuser') @@ -30,13 +30,13 @@ def setUp(self): super(TestJournalDiscontinuingSoonNotify, self).setUp() self.pull_application = models.Application.pull models.Application.pull = pull_application - self.pull_by_key = models.EditorGroup.pull_by_key - models.EditorGroup.pull_by_key = pull_by_key + self.pull_editor_group = models.EditorGroup.pull + models.EditorGroup.pull = pull_editor_group def tearDown(self): super(TestJournalDiscontinuingSoonNotify, self).tearDown() models.Application.pull = self.pull_application - models.EditorGroup.pull_by_key = self.pull_by_key + models.EditorGroup.pull = self.pull_editor_group def test_consumes(self): diff --git a/doajtest/unit/event_consumers/test_journal_editor_group_assigned_notify.py b/doajtest/unit/event_consumers/test_journal_editor_group_assigned_notify.py index b3948af0ea..dd46170631 100644 --- a/doajtest/unit/event_consumers/test_journal_editor_group_assigned_notify.py +++ b/doajtest/unit/event_consumers/test_journal_editor_group_assigned_notify.py @@ -37,7 +37,8 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name("test group") + eg.set_id(app.editor_group) eg.set_editor("editor") eg.save(blocking=True) @@ -67,7 +68,8 @@ def test_consume_fail(self): # app.save(blocking=True) eg = models.EditorGroup() - eg.set_name(app.editor_group) + eg.set_name("test group") + eg.set_id(app.editor_group) eg.save(blocking=True) event = models.Event(constants.EVENT_JOURNAL_EDITOR_GROUP_ASSIGNED, context={"journal": app.data}) diff --git a/doajtest/unit/test_fc_editor_app_review.py b/doajtest/unit/test_fc_editor_app_review.py index 5f7c90e992..caa28876c5 100644 --- a/doajtest/unit/test_fc_editor_app_review.py +++ b/doajtest/unit/test_fc_editor_app_review.py @@ -18,7 +18,7 @@ ##################################################################### @classmethod -def editor_group_pull(cls, field, value): +def editor_group_pull(cls, value): eg = models.EditorGroup() eg.set_editor("eddie") eg.set_associates(["associate", "assan"]) @@ -61,8 +61,8 @@ class TestEditorAppReview(DoajTestCase): def setUp(self): super(TestEditorAppReview, self).setUp() - self.editor_group_pull = models.EditorGroup.pull_by_key - models.EditorGroup.pull_by_key = editor_group_pull + self.editor_group_pull = models.EditorGroup.pull + models.EditorGroup.pull = editor_group_pull self.old_lcc_choices = lcc.lcc_choices lcc.lcc_choices = mock_lcc_choices @@ -73,7 +73,7 @@ def setUp(self): def tearDown(self): super(TestEditorAppReview, self).tearDown() - models.EditorGroup.pull_by_key = self.editor_group_pull + models.EditorGroup.pull = self.editor_group_pull lcc.lcc_choices = self.old_lcc_choices lcc.lookup_code = self.old_lookup_code diff --git a/doajtest/unit/test_fc_maned_app_review.py b/doajtest/unit/test_fc_maned_app_review.py index dae4d8a1a6..7d9e65f98b 100644 --- a/doajtest/unit/test_fc_maned_app_review.py +++ b/doajtest/unit/test_fc_maned_app_review.py @@ -17,7 +17,7 @@ ##################################################################### @classmethod -def editor_group_pull(cls, field, value): +def editor_group_pull(cls, value): eg = models.EditorGroup() eg.set_editor("eddie") eg.set_associates(["associate", "assan"]) @@ -61,8 +61,8 @@ class TestManEdAppReview(DoajTestCase): def setUp(self): super(TestManEdAppReview, self).setUp() - self.editor_group_pull = models.EditorGroup.pull_by_key - models.EditorGroup.pull_by_key = editor_group_pull + self.editor_group_pull = models.EditorGroup.pull + models.EditorGroup.pull = editor_group_pull self.old_lcc_choices = lcc.lcc_choices lcc.lcc_choices = mock_lcc_choices From c3d8fdf0278c61b001a9c2dc91398d55517d23fe Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 11:11:35 +0100 Subject: [PATCH 24/65] add test cases for autocomplete_pair --- doajtest/unit/test_models.py | 19 ++++++++++++++ doajtest/unit/test_view_doaj.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 doajtest/unit/test_view_doaj.py diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index bf626153d7..62d6381ac2 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1177,6 +1177,25 @@ def test_22_autocomplete(self): res = models.Journal.advanced_autocomplete("index.publisher_ac", "bibjson.publisher.name", "BioMed C") assert len(res) == 1, "autocomplete for 'BioMed C': found {}, expected 2".format(len(res)) + def test_autocomplete_pair(self): + eg = models.EditorGroup() + eg.set_id("id-1") + eg.set_name("eg name 1") + eg.save() + + eg = models.EditorGroup() + eg.set_id("id-2") + eg.set_name("eg name 2") + eg.save(blocking=True) + + res = models.EditorGroup.autocomplete_pair('name', 'eg', 'id', 'name') + assert len(res) == 2 + assert {r['id']:r['text'] for r in res} == {'id-1': 'eg name 1', 'id-2': 'eg name 2'} + + def test_autocomplete_pair__not_found(self): + res = models.EditorGroup.autocomplete_pair('name', 'aksjdlaksjdlkasjdl', 'id', 'name') + assert len(res) == 0 + def test_23_provenance(self): """Read and write properties into the provenance model""" p = models.Provenance() diff --git a/doajtest/unit/test_view_doaj.py b/doajtest/unit/test_view_doaj.py new file mode 100644 index 0000000000..dadbf95cc9 --- /dev/null +++ b/doajtest/unit/test_view_doaj.py @@ -0,0 +1,46 @@ +from doajtest.helpers import DoajTestCase +from portality import models + + +class TestViewDoaj(DoajTestCase): + def test_autocomplete_pair(self): + eg = models.EditorGroup() + eg.set_name("English") + eg.set_id('egid') + eg.save(blocking=True) + + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + query = 'eng' + response = client.get(f'/autocomplete/{doc_type}/{field_name}/{id_field}?q={query}', ) + resp_json = response.json + assert response.status_code == 200 + assert len(resp_json['suggestions']) == 1 + assert resp_json['suggestions'][0]['id'] == eg.id + assert resp_json['suggestions'][0]['text'] == eg.name + + def test_autocomplete_pair__not_found(self): + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + query = 'alksjdlaksjdalksdjl' + response = client.get(f'/autocomplete/{doc_type}/{field_name}/{id_field}?q={query}', ) + assert_not_found(response) + + def test_autocomplete_pair__unknown_doc_type(self): + with self.app_test.test_client() as client: + doc_type = 'alskdjalskdjal' + field_name = 'name' + id_field = 'id' + query = 'eng' + response = client.get(f'/autocomplete/{doc_type}/{field_name}/{id_field}?q={query}', ) + assert_not_found(response) + + +def assert_not_found(response): + resp_json = response.json + assert response.status_code == 200 + assert len(resp_json['suggestions']) == 0 From 38d86167e109fd14ddcaf4daf2e5c299aa404441 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 11:13:47 +0100 Subject: [PATCH 25/65] reformat --- doajtest/unit/test_models.py | 133 +++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index 62d6381ac2..0b3452e9c9 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -399,10 +399,10 @@ def test_06_article_deletes(self): # now hit the key methods involved in article deletes query = { - "query" : { - "bool" : { - "must" : [ - {"term" : {"bibjson.title.exact" : "Test Article 0"}} + "query": { + "bool": { + "must": [ + {"term": {"bibjson.title.exact": "Test Article 0"}} ] } } @@ -457,10 +457,10 @@ def test_07_journal_deletes(self): # now hit the key methods involved in journal deletes query = { - "query" : { - "bool" : { - "must" : [ - {"term" : {"bibjson.title.exact" : "Test Journal 1"}} + "query": { + "bool": { + "must": [ + {"term": {"bibjson.title.exact": "Test Journal 1"}} ] } } @@ -479,14 +479,14 @@ def test_07_journal_deletes(self): assert len(self.list_today_article_history_files()) == 1 assert len(models.Journal.all()) == 4 - assert len(self.list_today_journal_history_files()) == 6 # Because all journals are snapshot at create time + assert len(self.list_today_journal_history_files()) == 6 # Because all journals are snapshot at create time @patch_history_dir('JOURNAL_HISTORY_DIR') def test_08_iterate(self): for jsrc in JournalFixtureFactory.make_many_journal_sources(count=99, in_doaj=True): j = models.Journal(**jsrc) j.save() - time.sleep(2) # index all the journals + time.sleep(2) # index all the journals journal_ids = [] theqgen = models.JournalQuery() for j in models.Journal.iterate(q=theqgen.all_in_doaj(), page_size=10): @@ -564,9 +564,11 @@ def test_12_archiving_policy(self): assert b.preservation_summary == ["LOCKSS", "CLOCKSS", "Somewhere else", ["A national library", "Trinity"]] b.add_archiving_policy("SAFE") - assert b.preservation_summary == ["LOCKSS", "CLOCKSS", "Somewhere else", "SAFE", ["A national library", "Trinity"]] + assert b.preservation_summary == ["LOCKSS", "CLOCKSS", "Somewhere else", "SAFE", + ["A national library", "Trinity"]] - assert b.flattened_archiving_policies == ['LOCKSS', 'CLOCKSS', "Somewhere else", 'SAFE', 'A national library: Trinity'], b.flattened_archiving_policies + assert b.flattened_archiving_policies == ['LOCKSS', 'CLOCKSS', "Somewhere else", 'SAFE', + 'A national library: Trinity'], b.flattened_archiving_policies def test_13_generic_bibjson(self): source = BibJSONFixtureFactory.generic_bibjson() @@ -589,7 +591,7 @@ def test_13_generic_bibjson(self): gbj.title = "Updated Title" gbj.add_identifier("doi", "10.1234/7") gbj.add_keyword("test") - gbj.add_keyword("ONE") # make sure keywords are stored in lowercase + gbj.add_keyword("ONE") # make sure keywords are stored in lowercase keyword = None # make sure None keyword doesn't cause error gbj.add_keyword(keyword) gbj.add_url("http://test", "test") @@ -600,11 +602,11 @@ def test_13_generic_bibjson(self): assert gbj.get_one_identifier("doi") == "10.1234/7" assert gbj.keywords == ["word", "key", "test", "one"] assert gbj.get_single_url("test") == "http://test" - assert gbj.subjects()[2] == {"scheme" : "TEST", "term" : "first", "code" : "one"} + assert gbj.subjects()[2] == {"scheme": "TEST", "term": "first", "code": "one"} gbj.remove_identifiers("doi") - gbj.set_keywords("TwO") # make sure keywords are stored in lowercase - gbj.set_subjects({"scheme" : "TEST", "term" : "first", "code" : "one"}) + gbj.set_keywords("TwO") # make sure keywords are stored in lowercase + gbj.set_subjects({"scheme": "TEST", "term": "first", "code": "one"}) assert gbj.keywords == ["two"] keywords = [] @@ -665,7 +667,8 @@ def test_14_journal_like_bibjson(self): assert bj.plagiarism_detection is True assert bj.plagiarism_url == "http://plagiarism.screening" assert bj.preservation is not None - assert bj.preservation_summary == ["LOCKSS", "CLOCKSS", "A safe place", ["A national library", "Trinity"], ["A national library", "Imperial"]] + assert bj.preservation_summary == ["LOCKSS", "CLOCKSS", "A safe place", ["A national library", "Trinity"], + ["A national library", "Imperial"]] assert bj.preservation_url == "http://digital.archiving.policy" assert bj.publisher_name == "The Publisher" assert bj.publisher_country == "US" @@ -787,7 +790,8 @@ def test_14_journal_like_bibjson(self): assert len(bj.apc) == 2 assert bj.deposit_policy == ["Never", "OK"] assert bj.pid_scheme == ["Handle", "PURL"] - assert bj.preservation_summary == ["LOCKSS", "MOUNTAIN", ["A national library", "UCL"], ["A national library", "LSE"]] + assert bj.preservation_summary == ["LOCKSS", "MOUNTAIN", ["A national library", "UCL"], + ["A national library", "LSE"]] # special methods assert bj.issns() == ["1111-111X", "0000-000X"], bj.issns() @@ -820,8 +824,8 @@ def test_14_journal_like_bibjson(self): assert bj.pid_scheme == ["ARK", "PURL"] assert bj.subject == bj.subjects() - bj.set_subjects({"scheme" : "whatever", "term" : "also whatever"}) - assert bj.subject == [{"scheme" : "whatever", "term" : "also whatever"}] + bj.set_subjects({"scheme": "whatever", "term": "also whatever"}) + assert bj.subject == [{"scheme": "whatever", "term": "also whatever"}] bj.remove_subjects() assert len(bj.subject) == 0 @@ -899,7 +903,6 @@ def test_14_journal_like_bibjson(self): assert bj.replaces == [] assert bj.subject == [] - def test_15_continuations(self): journal = models.Journal() bj = journal.bibjson() @@ -963,7 +966,8 @@ def test_16_article_bibjson(self): assert bj.publisher == "IEEE" assert bj.author[0].get("name") == "Test" assert bj.author[0].get("affiliation") == "University of Life" - assert bj.author[0].get("orcid_id") == "https://orcid.org/0000-0001-1234-1234", "received: {}".format(bj.author[0].get("orcid_id")) + assert bj.author[0].get("orcid_id") == "https://orcid.org/0000-0001-1234-1234", "received: {}".format( + bj.author[0].get("orcid_id")) bj.year = "2000" bj.month = "5" @@ -995,7 +999,8 @@ def test_16_article_bibjson(self): assert bj.publisher == "Elsevier" assert bj.author[1].get("name") == "Testing" assert bj.author[1].get("affiliation") == "School of Hard Knocks" - assert bj.author[1].get("orcid_id") == "0000-0001-4321-4321", "received: {}".format(bj.author[1].get("orcid_id")) + assert bj.author[1].get("orcid_id") == "0000-0001-4321-4321", "received: {}".format( + bj.author[1].get("orcid_id")) del bj.year del bj.month @@ -1190,7 +1195,7 @@ def test_autocomplete_pair(self): res = models.EditorGroup.autocomplete_pair('name', 'eg', 'id', 'name') assert len(res) == 2 - assert {r['id']:r['text'] for r in res} == {'id-1': 'eg name 1', 'id-2': 'eg name 2'} + assert {r['id']: r['text'] for r in res} == {'id-1': 'eg name 1', 'id-2': 'eg name 2'} def test_autocomplete_pair__not_found(self): res = models.EditorGroup.autocomplete_pair('name', 'aksjdlaksjdlkasjdl', 'id', 'name') @@ -1261,7 +1266,7 @@ def test_25_make_provenance(self): eg2 = models.EditorGroup() eg2.set_id("editor") - eg2.set_name("Editor") # note: REQUIRED so that the mapping includes .name, which is needed to find groups_by + eg2.set_name("Editor") # note: REQUIRED so that the mapping includes .name, which is needed to find groups_by eg2.set_editor(acc.id) eg2.save() @@ -1295,7 +1300,7 @@ def test_26_background_job(self): source = BackgroundFixtureFactory.example() source["params"]["ids"] = ["1", "2", "3"] source["params"]["type"] = "suggestion" - source["reference"]["query"] = json.dumps({"query" : {"match_all" : {}}}) + source["reference"]["query"] = json.dumps({"query": {"match_all": {}}}) bj = models.BackgroundJob(**source) bj.save() @@ -1308,8 +1313,8 @@ def test_26a_background_job_active(self): bj = models.BackgroundJob(**source) bj.save(blocking=True) - assert len(models.BackgroundJob.active(source["action"])) == 1, "expected 1 active, got {x}".format(x=len(models.BackgroundJob.active(source["action"]))) - + assert len(models.BackgroundJob.active(source["action"])) == 1, "expected 1 active, got {x}".format( + x=len(models.BackgroundJob.active(source["action"]))) def test_27_article_journal_sync(self): j = models.Journal(**JournalFixtureFactory.make_journal_source(in_doaj=True)) @@ -1386,25 +1391,29 @@ def test_30_article_stats(self): # make a bunch of articles variably in doaj/not in doaj, for/not for the issn we'll search for i in range(1, 3): article = models.Article( - **ArticleFixtureFactory.make_article_source(eissn="1111-1111", pissn="1111-1111", with_id=False, in_doaj=True) + **ArticleFixtureFactory.make_article_source(eissn="1111-1111", pissn="1111-1111", with_id=False, + in_doaj=True) ) article.set_created("2019-01-0" + str(i) + "T00:00:00Z") articles.append(article) for i in range(3, 5): article = models.Article( - **ArticleFixtureFactory.make_article_source(eissn="1111-1111", pissn="1111-1111", with_id=False, in_doaj=False) + **ArticleFixtureFactory.make_article_source(eissn="1111-1111", pissn="1111-1111", with_id=False, + in_doaj=False) ) article.set_created("2019-01-0" + str(i) + "T00:00:00Z") articles.append(article) for i in range(5, 7): article = models.Article( - **ArticleFixtureFactory.make_article_source(eissn="2222-2222", pissn="2222-2222", with_id=False, in_doaj=True) + **ArticleFixtureFactory.make_article_source(eissn="2222-2222", pissn="2222-2222", with_id=False, + in_doaj=True) ) article.set_created("2019-01-0" + str(i) + "T00:00:00Z") articles.append(article) for i in range(7, 9): article = models.Article( - **ArticleFixtureFactory.make_article_source(eissn="2222-2222", pissn="2222-2222", with_id=False, in_doaj=False) + **ArticleFixtureFactory.make_article_source(eissn="2222-2222", pissn="2222-2222", with_id=False, + in_doaj=False) ) article.set_created("2019-01-0" + str(i) + "T00:00:00Z") articles.append(article) @@ -1421,18 +1430,20 @@ def test_30_article_stats(self): def test_31_cache(self): models.Cache.cache_site_statistics({ - "journals" : 10, - "new_journals" : 20, - "countries" : 30, - "abstracts" : 40, - "no_apc" : 50 + "journals": 10, + "new_journals": 20, + "countries": 30, + "abstracts": 40, + "no_apc": 50 }) models.Cache.cache_csv("/csv/filename.csv") models.Cache.cache_sitemap("sitemap.xml") - models.Cache.cache_public_data_dump("ac", "af", "http://example.com/article", 100, "jc", "jf", "http://example.com/journal", 200) + models.Cache.cache_public_data_dump("ac", "af", "http://example.com/article", + 100, "jc", "jf", + "http://example.com/journal", 200) time.sleep(1) @@ -1526,26 +1537,26 @@ def test_32_journal_like_object_discovery(self): # find_by_journal_url(cls, url, in_doaj=None, max=10) # recent(cls, max=10): -# TODO: reinstate this test when author emails have been disallowed again -# ''' -# def test_33_article_with_author_email(self): -# """Check the system disallows articles with emails in the author field""" -# a_source = ArticleFixtureFactory.make_article_source() -# -# # Creating a model from a source with email is rejected by the DataObj -# a_source['bibjson']['author'][0]['email'] = 'author@example.com' -# with self.assertRaises(dataobj.DataStructureException): -# a = models.Article(**a_source) -# bj = a.bibjson() -# -# # Remove the email address again to create the model -# del a_source['bibjson']['author'][0]['email'] -# a = models.Article(**a_source) -# -# # We can't add an author with an email address any more. -# with self.assertRaises(TypeError): -# a.bibjson().add_author(name='Ms Test', affiliation='School of Rock', email='author@example.com') -# ''' + # TODO: reinstate this test when author emails have been disallowed again + # ''' + # def test_33_article_with_author_email(self): + # """Check the system disallows articles with emails in the author field""" + # a_source = ArticleFixtureFactory.make_article_source() + # + # # Creating a model from a source with email is rejected by the DataObj + # a_source['bibjson']['author'][0]['email'] = 'author@example.com' + # with self.assertRaises(dataobj.DataStructureException): + # a = models.Article(**a_source) + # bj = a.bibjson() + # + # # Remove the email address again to create the model + # del a_source['bibjson']['author'][0]['email'] + # a = models.Article(**a_source) + # + # # We can't add an author with an email address any more. + # with self.assertRaises(TypeError): + # a.bibjson().add_author(name='Ms Test', affiliation='School of Rock', email='author@example.com') + # ''' def test_34_preserve(self): model = models.PreservationState() @@ -1594,7 +1605,7 @@ def test_35_event(self): assert event2.context.get("key2") == "value2" assert event2.when == event.when - event3 = models.Event("ABCD", "another", {"key3" : "value3"}) + event3 = models.Event("ABCD", "another", {"key3": "value3"}) assert event3.id == "ABCD" assert event3.who == "another" assert event3.context.get("key3") == "value3" @@ -1714,9 +1725,9 @@ def test_40_autocheck_retrieves(self): ap2 = models.Autocheck.for_journal("9876") assert ap2.journal == "9876" + class TestAccount(DoajTestCase): def test_get_name_safe(self): - # have name acc = models.Account.make_account(email='user@example.com') acc_name = 'Account Name' @@ -1758,4 +1769,4 @@ def test_11_find_by_issn(self): assert journals[0].id == j1.id journals = models.Journal.find_by_issn_exact(["1111-1111", "3333-3333"], True) - assert len(journals) == 0 \ No newline at end of file + assert len(journals) == 0 From 62669555a47073c9decb13af7a1162b32b341eb7 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 11:46:27 +0100 Subject: [PATCH 26/65] fix test cases --- doajtest/unit/test_view_doaj.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doajtest/unit/test_view_doaj.py b/doajtest/unit/test_view_doaj.py index dadbf95cc9..cdbfeeb9e1 100644 --- a/doajtest/unit/test_view_doaj.py +++ b/doajtest/unit/test_view_doaj.py @@ -28,7 +28,9 @@ def test_autocomplete_pair__not_found(self): id_field = 'id' query = 'alksjdlaksjdalksdjl' response = client.get(f'/autocomplete/{doc_type}/{field_name}/{id_field}?q={query}', ) - assert_not_found(response) + resp_json = response.json + assert response.status_code == 200 + assert len(resp_json['suggestions']) == 0 def test_autocomplete_pair__unknown_doc_type(self): with self.app_test.test_client() as client: @@ -37,10 +39,9 @@ def test_autocomplete_pair__unknown_doc_type(self): id_field = 'id' query = 'eng' response = client.get(f'/autocomplete/{doc_type}/{field_name}/{id_field}?q={query}', ) - assert_not_found(response) + resp_json = response.json + assert response.status_code == 200 + assert len(resp_json['suggestions']) == 1 + assert resp_json['suggestions'][0]['id'] == '' -def assert_not_found(response): - resp_json = response.json - assert response.status_code == 200 - assert len(resp_json['suggestions']) == 0 From 999e4c6defba379a4c8abbccb67013d068581912 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 13:03:31 +0100 Subject: [PATCH 27/65] add doc --- portality/view/doaj.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 12232687ed..9680203a40 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -262,6 +262,24 @@ def autocomplete(doc_type, field_name): @blueprint.route('/autocomplete///', methods=["GET", "POST"]) def autocomplete_pair(doc_type, field_name, id_field): + """ + different from autocomplete, in response json, the values of `id` and `text` are used different field + `id` used value of `id_field` + `text` used value of `field_name` + parameter `q` will be used to search the prefix of `field_name` value + + Parameters + ---------- + doc_type + field_name + id_field + + Returns + ------- + Json string with follow format: + {suggestions: [{id: id_field, text: field_name}]} + """ + prefix = request.args.get('q', '') if not prefix: return _no_suggestions_response() # select2 does not understand 400, which is the correct code here... From 9febfcf9d7f23fd9780fce4ceed87a009dabe917 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 13:26:14 +0100 Subject: [PATCH 28/65] remove useless parameter `field_name` --- portality/dao.py | 3 +-- portality/view/doaj.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index a4ca5d6202..ac1083a0bb 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -1159,10 +1159,9 @@ def url_encode_query(query): ######################################################################### class IdTextTermQuery: - def __init__(self, id_field, id_value, field_name): + def __init__(self, id_field, id_value): self.id_field = id_field self.id_value = id_value - self.field_name = field_name def query(self): return { diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 9680203a40..83289c618d 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -307,7 +307,7 @@ def autocomplete_text_mapping(doc_type, field_name, id_field): def id_text_mapping(doc_type, field_name, id_field, id_value) -> Optional[str]: query_factory_mapping = { - ('editor_group', 'id', 'name', ): lambda: IdTextTermQuery(id_field, id_value, field_name).query(), + ('editor_group', 'id', 'name', ): lambda: IdTextTermQuery(id_field, id_value).query(), } query_factory = query_factory_mapping.get((doc_type, id_field, field_name)) if not query_factory: From 82336ddba8435e366f6613062a7fc26cc6da3580 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 13:26:47 +0100 Subject: [PATCH 29/65] template should use editor group name --- portality/tasks/async_workflow_notifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/tasks/async_workflow_notifications.py b/portality/tasks/async_workflow_notifications.py index fd518a958a..138acbd003 100644 --- a/portality/tasks/async_workflow_notifications.py +++ b/portality/tasks/async_workflow_notifications.py @@ -250,7 +250,7 @@ def editor_notifications(emails_dict, limit=None): ed_email = editor.email text = render_template('email/workflow_reminder_fragments/editor_groupcount_frag', - num=group_count, ed_group=group_id, url=ed_url) + num=group_count, ed_group=eg.name, url=ed_url) _add_email_paragraph(emails_dict, ed_email, eg.editor, text) # Second note - records within editor group not touched for so long @@ -276,7 +276,7 @@ def editor_notifications(emails_dict, limit=None): ed_email = editor.email text = render_template('email/workflow_reminder_fragments/editor_age_frag', - num=group_count, ed_group=group_id, url=ed_age_url, x_weeks=X_WEEKS) + num=group_count, ed_group=eg.name, url=ed_age_url, x_weeks=X_WEEKS) _add_email_paragraph(emails_dict, ed_email, eg.editor, text) From 71dc225f0d95839873cf9e4388385d0a83192950 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 13:41:30 +0100 Subject: [PATCH 30/65] avoid return None --- portality/view/doaj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 83289c618d..5b7023daea 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -301,7 +301,7 @@ def autocomplete_text_mapping(doc_type, field_name, id_field): id_value = request.args.get('id') text = id_text_mapping(doc_type, field_name, id_field, id_value) - return jsonify({'id': id_value, 'text': text}) + return jsonify({'id': id_value, 'text': text or ''}) def id_text_mapping(doc_type, field_name, id_field, id_value) -> Optional[str]: From 9b74a0439c9a316ba25d6a51601fbc71b12cbc7e Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 13 May 2024 13:41:49 +0100 Subject: [PATCH 31/65] add TYPE_CHECKING if --- portality/dao.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/portality/dao.py b/portality/dao.py index ac1083a0bb..629982a7a3 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -8,6 +8,7 @@ from copy import deepcopy from datetime import timedelta from typing import List, Iterable, Union, Dict, TypedDict, Optional +from typing import TYPE_CHECKING import elasticsearch @@ -15,8 +16,9 @@ from portality.lib import dates from portality.lib.dates import FMT_DATETIME_STD -if sys.version_info >= (3, 11): - from typing import Self +if TYPE_CHECKING: + if sys.version_info >= (3, 11): + from typing import Self # All models in models.py should inherit this DomainObject to know how to save themselves in the index and so on. # You can overwrite and add to the DomainObject functions as required. See models.py for some examples. From 9dfbd0c9dd50304736287d468955fe7397991872 Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 13:35:42 +0100 Subject: [PATCH 32/65] fix for editor group name in todo --- portality/bll/services/todo.py | 9 ++++++--- portality/models/editors.py | 6 +++++- portality/templates/dashboard/_todo.html | 8 +++++++- portality/templates/includes/_todo_item.html | 6 +++++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 67b7050b0e..f153a2f8cd 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -126,15 +126,18 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True for aid, q, sort, boost in queries: applications = models.Application.object_query(q=q.query()) for ap in applications: - todos.append({ + todo = { "date": ap.last_manual_update_timestamp if sort == "last_manual_update" else ap.date_applied_timestamp, "date_type": sort, "action_id": [aid], "title": ap.bibjson().title, "object_id": ap.id, "object": ap, - "boost": boost - }) + "boost": boost, + } + if ap.editor_group: + todo["editor_group_name"] = models.EditorGroup.find_name_by_id(ap.editor_group) or ap.editor_group + todos.append(todo) todos = self._rationalise_todos(todos, size) diff --git a/portality/models/editors.py b/portality/models/editors.py index 377dfec45b..58e450fd39 100644 --- a/portality/models/editors.py +++ b/portality/models/editors.py @@ -1,7 +1,7 @@ import sys from typing import List, Optional -from portality.dao import DomainObject, ScrollInitialiseException +from portality.dao import DomainObject, ScrollInitialiseException, IdTextTermQuery from portality.models import Account if sys.version_info >= (3, 11): @@ -47,6 +47,10 @@ def groups_by_editor(cls, editor) -> List['Self']: def groups_by_associate(cls, associate) -> List['Self']: return cls._groups_by_x(associate=associate) + @classmethod + def find_name_by_id(cls, editor_group_id) -> Optional[str]: + return cls.get_target_value(IdTextTermQuery('id', editor_group_id).query(), "name") + @property def name(self): return self.data.get("name") diff --git a/portality/templates/dashboard/_todo.html b/portality/templates/dashboard/_todo.html index 069ba89633..54e6fbe6c5 100644 --- a/portality/templates/dashboard/_todo.html +++ b/portality/templates/dashboard/_todo.html @@ -182,7 +182,13 @@

    {% set source = search_query_source(terms=[{"admin.editor_group.exact" : [todo.object.editor_group]}]) %} -
  • {{ todo.object.editor_group }}
    • +
  • + + + {{ todo.editor_group_name }} + + +
    • {% set editor = todo.object.editor %} {%- if editor %}
  • diff --git a/portality/templates/includes/_todo_item.html b/portality/templates/includes/_todo_item.html index 061e204af1..78a67f44b7 100644 --- a/portality/templates/includes/_todo_item.html +++ b/portality/templates/includes/_todo_item.html @@ -37,7 +37,11 @@

      {% set source = search_query_source(terms=[{"admin.editor_group.exact" : [todo.object.editor_group]}]) %} -
    • {{ todo.object.editor_group }}
      • +
    • + + {{ todo.editor_group_name }} + +
      • {% set editor = todo.object.editor %} {% if editor %}
    • {{ todo.object.editor }}
      • From 9d5c078f7a1da7a29867850376ddfcf4c20eedfe Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 13:36:26 +0100 Subject: [PATCH 33/65] use editor_group.id --- doajtest/testdrive/todo_editor.py | 11 +++---- .../testdrive/todo_maned_editor_associate.py | 33 +++++++++++-------- doajtest/unit/test_query_filters.py | 2 +- doajtest/unit/test_task_journal_bulkedit.py | 8 +++-- .../unit/test_task_suggestion_bulkedit.py | 8 +++-- portality/bll/services/todo.py | 10 +++--- portality/lib/query_filters.py | 2 +- portality/models/editors.py | 3 +- portality/models/v2/application.py | 5 +-- portality/static/js/dashboard.js | 16 ++++----- .../templates/layouts/dashboard_base.html | 9 +++-- portality/view/admin.py | 7 +++- 12 files changed, 65 insertions(+), 49 deletions(-) diff --git a/doajtest/testdrive/todo_editor.py b/doajtest/testdrive/todo_editor.py index 7380b483ba..3cbb87210b 100644 --- a/doajtest/testdrive/todo_editor.py +++ b/doajtest/testdrive/todo_editor.py @@ -68,13 +68,13 @@ def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_grou return ap -def build_applications(un, eg): +def build_applications(un: str, eg: models.Application): w = 7 * 24 * 60 * 60 apps = {} app = build_application(un + " Stalled Application", 6 * w, 7 * w, constants.APPLICATION_STATUS_IN_PROGRESS, - editor_group=eg.name) + editor_group=eg.id) app.save() apps["stalled"] = [{ "id": app.id, @@ -82,7 +82,7 @@ def build_applications(un, eg): }] app = build_application(un + " Old Application", 8 * w, 8 * w, constants.APPLICATION_STATUS_IN_PROGRESS, - editor_group=eg.name) + editor_group=eg.id) app.save() apps["old"] = [{ "id": app.id, @@ -90,7 +90,7 @@ def build_applications(un, eg): }] app = build_application(un + " Completed Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_COMPLETED, - editor_group=eg.name) + editor_group=eg.id) app.save() apps["completed"] = [{ "id": app.id, @@ -98,7 +98,7 @@ def build_applications(un, eg): }] app = build_application(un + " Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, - editor_group=eg.name) + editor_group=eg.id) app.remove_editor() app.save() apps["pending"] = [{ @@ -107,4 +107,3 @@ def build_applications(un, eg): }] return apps - diff --git a/doajtest/testdrive/todo_maned_editor_associate.py b/doajtest/testdrive/todo_maned_editor_associate.py index 1fa8ff936e..e5180ba97e 100644 --- a/doajtest/testdrive/todo_maned_editor_associate.py +++ b/doajtest/testdrive/todo_maned_editor_associate.py @@ -13,7 +13,8 @@ class TodoManedEditorAssociate(TestDrive): def setup(self) -> dict: un = self.create_random_str() pw = self.create_random_str() - admin = models.Account.make_account(un + "@example.com", un, "TodoManedEditorAssociate " + un, ["admin", "editor", constants.ROLE_ASSOCIATE_EDITOR]) + admin = models.Account.make_account(un + "@example.com", un, "TodoManedEditorAssociate " + un, + ["admin", "editor", constants.ROLE_ASSOCIATE_EDITOR]) admin.set_password(pw) admin.save() @@ -53,7 +54,6 @@ def setup(self) -> dict: eapps = build_editor_applications(un, eg2) mapps = build_maned_applications(un, eg1, owner.id, eg3) - return { "account": { "username": admin.id, @@ -96,13 +96,15 @@ def teardown(self, params) -> dict: return {"status": "success"} -def build_maned_applications(un, eg, owner, eponymous_group): +def build_maned_applications(un: str, + eg: models.EditorGroup, owner, + eponymous_group: models.EditorGroup): w = 7 * 24 * 60 * 60 apps = {} app = build_application(un + " Maned Stalled Application", 8 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, - editor_group=eg.name, owner=owner) + editor_group=eg.id, owner=owner) app.save() apps["stalled"] = [{ "id": app.id, @@ -110,7 +112,7 @@ def build_maned_applications(un, eg, owner, eponymous_group): }] app = build_application(un + " Maned Old Application", 10 * w, 10 * w, constants.APPLICATION_STATUS_IN_PROGRESS, - editor_group=eg.name, owner=owner) + editor_group=eg.id, owner=owner) app.save() apps["old"] = [{ "id": app.id, @@ -118,7 +120,7 @@ def build_maned_applications(un, eg, owner, eponymous_group): }] app = build_application(un + " Maned Ready Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_READY, - editor_group=eg.name, owner=owner) + editor_group=eg.id, owner=owner) app.save() apps["ready"] = [{ "id": app.id, @@ -126,7 +128,7 @@ def build_maned_applications(un, eg, owner, eponymous_group): }] app = build_application(un + " Maned Completed Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_COMPLETED, - editor_group=eg.name, owner=owner) + editor_group=eg.id, owner=owner) app.save() apps["completed"] = [{ "id": app.id, @@ -134,7 +136,7 @@ def build_maned_applications(un, eg, owner, eponymous_group): }] app = build_application(un + " Maned Pending Application", 2 * w, 2 * w, constants.APPLICATION_STATUS_PENDING, - editor_group=eg.name, owner=owner) + editor_group=eg.id, owner=owner) app.remove_editor() app.save() apps["pending"] = [{ @@ -144,7 +146,7 @@ def build_maned_applications(un, eg, owner, eponymous_group): app = build_application(un + " Maned Low Priority Pending Application", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, - editor_group=eponymous_group.name, owner=owner) + editor_group=eponymous_group.id, owner=owner) app.remove_editor() app.save() apps["low_priority_pending"] = [{ @@ -152,12 +154,14 @@ def build_maned_applications(un, eg, owner, eponymous_group): "title": un + " Maned Low Priority Pending Application" }] - lmur = build_application(un + " Last Month Maned Update Request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, - editor_group=eponymous_group.name, owner=owner, update_request=True) + lmur = build_application(un + " Last Month Maned Update Request", 5 * w, 5 * w, + constants.APPLICATION_STATUS_UPDATE_REQUEST, + editor_group=eponymous_group.id, owner=owner, update_request=True) lmur.save() - tmur = build_application(un + " This Month Maned Update Request", 0, 0, constants.APPLICATION_STATUS_UPDATE_REQUEST, - editor_group=eponymous_group.name, owner=owner, update_request=True) + tmur = build_application(un + " This Month Maned Update Request", 0, 0, + constants.APPLICATION_STATUS_UPDATE_REQUEST, + editor_group=eponymous_group.id, owner=owner, update_request=True) tmur.save() apps["update_request"] = [ @@ -174,7 +178,8 @@ def build_maned_applications(un, eg, owner, eponymous_group): return apps -def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None, owner=None, update_request=False): +def build_application(title, lmu_diff, cd_diff, status, editor=None, editor_group=None, owner=None, + update_request=False): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.bibjson().title = title diff --git a/doajtest/unit/test_query_filters.py b/doajtest/unit/test_query_filters.py index 59ce6dfafe..8987a8fdca 100644 --- a/doajtest/unit/test_query_filters.py +++ b/doajtest/unit/test_query_filters.py @@ -92,7 +92,7 @@ def test_05_editor(self): "track_total_hits": True, 'query': { 'bool': { - 'must': [{'terms': {'admin.editor_group.exact': [eg.name]}}] + 'must': [{'terms': {'admin.editor_group.exact': [eg.id]}}] } } }, newq.as_dict() diff --git a/doajtest/unit/test_task_journal_bulkedit.py b/doajtest/unit/test_task_journal_bulkedit.py index 12facfee39..f059f8e908 100644 --- a/doajtest/unit/test_task_journal_bulkedit.py +++ b/doajtest/unit/test_task_journal_bulkedit.py @@ -61,10 +61,12 @@ def test_01_editor_group_successful_assign(self): new_eg = EditorGroupFixtureFactory.setup_editor_group_with_editors(group_name='Test Editor Group') # test dry run - summary = journal_manage({"query": {"terms": {"_id": [j.id for j in self.journals]}}}, editor_group=new_eg.name, dry_run=True) + summary = journal_manage({"query": {"terms": {"_id": [j.id for j in self.journals]}}}, + editor_group=new_eg.id, dry_run=True) assert summary.as_dict().get("affected", {}).get("journals") == TEST_JOURNAL_COUNT, summary.as_dict() - summary = journal_manage({"query": {"terms": {"_id": [j.id for j in self.journals]}}}, editor_group=new_eg.name, dry_run=False) + summary = journal_manage({"query": {"terms": {"_id": [j.id for j in self.journals]}}}, + editor_group=new_eg.id, dry_run=False) assert summary.as_dict().get("affected", {}).get("journals") == TEST_JOURNAL_COUNT, summary.as_dict() sleep(1) @@ -74,7 +76,7 @@ def test_01_editor_group_successful_assign(self): modified_journals = [j.pull(j.id) for j in self.journals] for ix, j in enumerate(modified_journals): - assert j.editor_group == new_eg.name, \ + assert j.editor_group == new_eg.id, \ "modified_journals[{}].editor_group is {}" \ "\nHere is the BackgroundJob audit log:\n{}"\ .format(ix, j.editor_group, json.dumps(job.audit, indent=2)) diff --git a/doajtest/unit/test_task_suggestion_bulkedit.py b/doajtest/unit/test_task_suggestion_bulkedit.py index 6843db1375..5b1014faaa 100644 --- a/doajtest/unit/test_task_suggestion_bulkedit.py +++ b/doajtest/unit/test_task_suggestion_bulkedit.py @@ -70,10 +70,12 @@ def test_01_editor_group_successful_assign(self): new_eg = EditorGroupFixtureFactory.setup_editor_group_with_editors(group_name='Test Editor Group') # test dry run - summary = suggestion_manage({"query": {"terms": {"_id": [s.id for s in self.suggestions]}}}, editor_group=new_eg.name, dry_run=True) + summary = suggestion_manage({"query": {"terms": {"_id": [s.id for s in self.suggestions]}}}, + editor_group=new_eg.id, dry_run=True) assert summary.as_dict().get("affected", {}).get("applications") == TEST_SUGGESTION_COUNT, summary.as_dict() - summary = suggestion_manage({"query": {"terms": {"_id": [s.id for s in self.suggestions]}}}, editor_group=new_eg.name, dry_run=False) + summary = suggestion_manage({"query": {"terms": {"_id": [s.id for s in self.suggestions]}}}, + editor_group=new_eg.id, dry_run=False) assert summary.as_dict().get("affected", {}).get("applications") == TEST_SUGGESTION_COUNT, summary.as_dict() blocklist = [(s.id, s.last_updated) for s in self.suggestions] @@ -85,7 +87,7 @@ def test_01_editor_group_successful_assign(self): modified_suggestions = [s.pull(s.id) for s in self.suggestions] for ix, s in enumerate(modified_suggestions): - assert s.editor_group == new_eg.name, \ + assert s.editor_group == new_eg.id, \ "modified_suggestions[{}].editor_group is {}\n" \ "Here is the BackgroundJob audit log:\n{}".format( ix, s.editor_group, json.dumps(job.audit, indent=2) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index f153a2f8cd..0cfbbed124 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -28,7 +28,7 @@ def group_stats(self, group_id): "email": None if acc is None else acc.email } - q = GroupStatsQuery(eg.name) + q = GroupStatsQuery(eg.id) resp = models.Application.query(q=q.query()) stats["total"] = {"applications": 0, "update_requests": 0} @@ -538,7 +538,7 @@ def exists(cls, field): def editor_groups(cls, groups: Iterable[EditorGroup]): return { "terms": { - "admin.editor_group.exact": [g.name for g in groups] + "admin.editor_group.exact": [g.id for g in groups] } } @@ -557,8 +557,8 @@ class GroupStatsQuery: ~~^->Elasticsearch:Technology~~ """ - def __init__(self, group_name, editor_count=10): - self.group_name = group_name + def __init__(self, group_id, editor_count=10): + self.group_id = group_id self.editor_count = editor_count def query(self): @@ -569,7 +569,7 @@ def query(self): "must": [ { "term": { - "admin.editor_group.exact": self.group_name + "admin.editor_group.exact": self.group_id } } ], diff --git a/portality/lib/query_filters.py b/portality/lib/query_filters.py index 07b1bb7d6f..ae99f5b654 100644 --- a/portality/lib/query_filters.py +++ b/portality/lib/query_filters.py @@ -109,7 +109,7 @@ def editor(q): gnames = [] groups = models.EditorGroup.groups_by_editor(current_user.id) for g in groups: - gnames.append(g.name) + gnames.append(g.id) q.clear_match_all() q.add_must({"terms" : {"admin.editor_group.exact" : gnames}}) return q diff --git a/portality/models/editors.py b/portality/models/editors.py index 58e450fd39..8c2c8d3a32 100644 --- a/portality/models/editors.py +++ b/portality/models/editors.py @@ -106,8 +106,7 @@ def is_member(self, account_name): @classmethod def find_editor_role_by_id(cls, editor_group_id, user_id) -> str: - # KTODO change to .pull() - eg = cls.pull_by_key("name", editor_group_id) + eg = cls.pull(editor_group_id) return "editor" if eg is not None and eg.editor == user_id else "associate_editor" diff --git a/portality/models/v2/application.py b/portality/models/v2/application.py index 391156ed4a..54e9bc1e45 100644 --- a/portality/models/v2/application.py +++ b/portality/models/v2/application.py @@ -1,6 +1,7 @@ from copy import deepcopy +from typing import Iterable -from portality import constants +from portality import constants, models from portality.core import app from portality.lib import es_data_mapping, coerce, dates from portality.models.v2 import shared_structs @@ -92,7 +93,7 @@ def find_all_by_related_journal(cls, journal_id): @classmethod def assignment_to_editor_groups(cls, egs): - q = AssignedEditorGroupsQuery([eg.name for eg in egs]) + q = AssignedEditorGroupsQuery([eg.id for eg in egs]) res = cls.query(q.query()) buckets = res.get("aggregations", {}).get("editor_groups", {}).get("buckets", []) assignments = {} diff --git a/portality/static/js/dashboard.js b/portality/static/js/dashboard.js index 4acc2ae0bd..2b5f698963 100644 --- a/portality/static/js/dashboard.js +++ b/portality/static/js/dashboard.js @@ -72,7 +72,7 @@ doaj.dashboard.renderGroupInfo = function(data) { let appQuerySource = doaj.searchQuerySource({ "term" : [ {"admin.editor.exact" : ed}, - {"admin.editor_group.exact" : data.editor_group.name}, + {"admin.editor_group.exact" : data.editor_group.id}, {"index.application_type.exact" : "new application"} // this is required so we only see open applications, not finished ones ], "sort": [{"admin.date_applied": {"order": "asc"}}] @@ -80,7 +80,7 @@ doaj.dashboard.renderGroupInfo = function(data) { // // ~~-> UpdateRequestsSearch:Page ~~ // let urQuerySource = doaj.searchQuerySource({"term" : [ // {"admin.editor.exact" : ed}, - // {"admin.editor_group.exact" : data.editor_group.name}, + // {"admin.editor_group.exact" : data.editor_group.id}, // {"index.application_type.exact" : "update request"} // this is required so we only see open update requests, not finished ones // ]}) let appCount = 0; @@ -110,9 +110,9 @@ doaj.dashboard.renderGroupInfo = function(data) { } // ~~-> ApplicationSearch:Page~~ - let appUnassignedSource = doaj.searchQuerySource({ + let appUnassignedSource = doaj.searchQuerySource({ "term" : [ - {"admin.editor_group.exact" : data.editor_group.name}, + {"admin.editor_group.exact" : data.editor_group.id}, {"index.has_editor.exact": "No"}, {"index.application_type.exact" : "new application"} // this is required so we only see open applications, not finished ones ], @@ -120,7 +120,7 @@ doaj.dashboard.renderGroupInfo = function(data) { }); // ~~-> UpdateRequestsSearch:Page ~~ // let urUnassignedSource = doaj.searchQuerySource({"term" : [ - // {"admin.editor_group.exact" : data.editor_group.name}, + // {"admin.editor_group.exact" : data.editor_group.id}, // {"index.has_editor.exact": "No"}, // {"index.application_type.exact" : "update request"} // this is required so we only see open update requests, not finished ones // ]}) @@ -138,7 +138,7 @@ doaj.dashboard.renderGroupInfo = function(data) { // ~~-> ApplicationSearch:Page~~ let appStatusSource = doaj.searchQuerySource({ "term": [ - {"admin.editor_group.exact": data.editor_group.name}, + {"admin.editor_group.exact": data.editor_group.id}, {"admin.application_status.exact": status}, {"index.application_type.exact": "new application"} // this is required so we only see open applications, not finished ones ], @@ -155,14 +155,14 @@ doaj.dashboard.renderGroupInfo = function(data) { // ~~-> ApplicationSearch:Page~~ let appGroupSource = doaj.searchQuerySource({ "term" : [ - {"admin.editor_group.exact" : data.editor_group.name}, + {"admin.editor_group.exact" : data.editor_group.id}, {"index.application_type.exact" : "new application"} // this is required so we only see open applications, not finished ones ], "sort": [{"admin.date_applied": {"order": "asc"}}] }); // ~~-> UpdateRequestsSearch:Page ~~ // let urGroupSource = doaj.searchQuerySource({ "term" : [ - // {"admin.editor_group.exact" : data.editor_group.name}, + // {"admin.editor_group.exact" : data.editor_group.id}, // {"index.application_type.exact" : "update request"} // this is required so we only see open applications, not finished ones // ]}) let frag = `
      diff --git a/portality/templates/layouts/dashboard_base.html b/portality/templates/layouts/dashboard_base.html index f60482ed0e..53c3449257 100644 --- a/portality/templates/layouts/dashboard_base.html +++ b/portality/templates/layouts/dashboard_base.html @@ -98,7 +98,7 @@

      {% for eg in managed_groups|sort(attribute="name") %}
    • {% set app_source = search_query_source(term=[ - {"admin.editor_group.exact" : eg.name}, + {"admin.editor_group.exact" : eg.id}, {"index.application_type.exact" : "new application"} ], sort=[{"admin.date_applied": {"order": "asc"}}] ) %} @@ -118,11 +118,14 @@

      {% for eg in editor_of_groups|sort(attribute="name") %}
    • {% set app_source = search_query_source(term=[ - {"admin.editor_group.exact" : eg.name}, + {"admin.editor_group.exact" : eg.id}, {"index.application_type.exact" : "new application"} ], sort=[{"admin.date_applied": {"order": "asc"}}] ) %} - {{ eg.name }} + + {{ eg.name }} + {% set maned = eg.get_maned_account() %} {{ maned.id }} (Managing Editor) {{ " / " if not loop.last else "" }} diff --git a/portality/view/admin.py b/portality/view/admin.py index 84ce91b1c1..38acebc523 100644 --- a/portality/view/admin.py +++ b/portality/view/admin.py @@ -706,7 +706,7 @@ def bulk_assign_editor_group(doaj_type): summary = task( selection_query=get_query_from_request(payload, doaj_type), - editor_group=payload['editor_group'], + editor_group=models.EditorGroup.group_exists_by_name(payload['editor_group']) or payload['editor_group'], dry_run=payload.get('dry_run', True) ) @@ -744,6 +744,11 @@ def bulk_edit_journal_metadata(): if not "metadata" in payload: raise BulkAdminEndpointException("key 'metadata' not present in request json") + # replace editor_group name with id + if "editor_group" in payload["metadata"]: + _eg = payload["metadata"]["editor_group"] + payload["metadata"]["editor_group"] = models.EditorGroup.group_exists_by_name(_eg) or _eg + formdata = MultiDict(payload["metadata"]) formulaic_context = JournalFormFactory.context("bulk_edit", extra_param=exparam_editing_user()) fc = formulaic_context.processor(formdata=formdata) From 8785a8b76d971eb21b34ee18c15f7d15caa8506e Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 14:07:03 +0100 Subject: [PATCH 34/65] fix test data --- .../unit/test_bll_todo_top_todo_editor.py | 20 +++++-- doajtest/unit/test_bll_todo_top_todo_maned.py | 58 ++++++++++++------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/doajtest/unit/test_bll_todo_top_todo_editor.py b/doajtest/unit/test_bll_todo_top_todo_editor.py index f5f6d4a2e7..1b8f53b6e2 100644 --- a/doajtest/unit/test_bll_todo_top_todo_editor.py +++ b/doajtest/unit/test_bll_todo_top_todo_editor.py @@ -56,6 +56,7 @@ def test_top_todo(self, name, kwargs): w = 7 * 24 * 60 * 60 account = None + editor_group_id = None if account_arg == "admin": asource = AccountFixtureFactory.make_managing_editor_source() account = models.Account(**asource) @@ -65,6 +66,7 @@ def test_top_todo(self, name, kwargs): eg_source = EditorGroupFixtureFactory.make_editor_group_source(editor=account.id) eg = models.EditorGroup(**eg_source) eg.save(blocking=True) + editor_group_id = eg.id elif account_arg == "assed": asource = AccountFixtureFactory.make_assed1_source() account = models.Account(**asource) @@ -76,19 +78,23 @@ def test_top_todo(self, name, kwargs): ############################################################ # an application created more than 8 weeks ago - self.build_application("editor_follow_up_old", 2 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) + self.build_application("editor_follow_up_old", 2 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + apps, editor_group_id=editor_group_id) # an application that was last updated over 6 weeks ago - self.build_application("editor_stalled", 7 * w, 7 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) + self.build_application("editor_stalled", 7 * w, 7 * w, constants.APPLICATION_STATUS_IN_PROGRESS, + apps, editor_group_id=editor_group_id) # an application that was modifed recently into the completed status - self.build_application("editor_completed", 2 * w, 2 * w, constants.APPLICATION_STATUS_COMPLETED, apps) + self.build_application("editor_completed", 2 * w, 2 * w, constants.APPLICATION_STATUS_COMPLETED, + apps, editor_group_id=editor_group_id) # an application that is pending without an editor assigned def assign_pending(ap): ap.remove_editor() - self.build_application("editor_assign_pending", 2 * w, 2 * w, constants.APPLICATION_STATUS_PENDING, apps, additional_fn=assign_pending) + self.build_application("editor_assign_pending", 2 * w, 2 * w, constants.APPLICATION_STATUS_PENDING, + apps, additional_fn=assign_pending, editor_group_id=editor_group_id) wait_until_no_es_incomplete_tasks() models.Application.refresh() @@ -127,7 +133,8 @@ def assign_pending(ap): else: # the todo item is not positioned at all assert len(positions.get(k, [])) == 0 - def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additional_fn=None): + def build_application(self, id, lmu_diff, cd_diff, status, app_registry, + additional_fn=None, editor_group_id=None): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.set_id(id) @@ -139,5 +146,8 @@ def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additio if additional_fn is not None: additional_fn(ap) + if editor_group_id is not None: + ap.set_editor_group(editor_group_id) + ap.save() app_registry.append(ap) \ No newline at end of file diff --git a/doajtest/unit/test_bll_todo_top_todo_maned.py b/doajtest/unit/test_bll_todo_top_todo_maned.py index 5322a8c9ae..a5d8989b47 100644 --- a/doajtest/unit/test_bll_todo_top_todo_maned.py +++ b/doajtest/unit/test_bll_todo_top_todo_maned.py @@ -12,11 +12,11 @@ def load_cases(): return load_parameter_sets(rel2abs(__file__, "..", "matrices", "bll_todo_maned"), "top_todo_maned", "test_id", - {"test_id" : []}) + {"test_id": []}) EXCEPTIONS = { - "ArgumentException" : exceptions.ArgumentException + "ArgumentException": exceptions.ArgumentException } @@ -45,7 +45,7 @@ def test_top_todo(self, name, kwargs): ] category_args = { - cat : ( + cat: ( int(kwargs.get(cat)), int(kwargs.get(cat + "_order") if kwargs.get(cat + "_order") != "" else -1) ) for cat in categories @@ -58,12 +58,14 @@ def test_top_todo(self, name, kwargs): w = 7 * 24 * 60 * 60 account = None + editor_group_id = None if account_arg == "admin": asource = AccountFixtureFactory.make_managing_editor_source() account = models.Account(**asource) eg_source = EditorGroupFixtureFactory.make_editor_group_source(maned=account.id) eg = models.EditorGroup(**eg_source) eg.save(blocking=True) + editor_group_id = eg.id elif account_arg == "editor": asource = AccountFixtureFactory.make_editor_source() account = models.Account(**asource) @@ -78,27 +80,31 @@ def test_top_todo(self, name, kwargs): ############################################################ # an application stalled for more than 8 weeks (todo_maned_stalled) - self.build_application("maned_stalled", 9 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) + self.build_application("maned_stalled", 9 * w, 9 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, + editor_group_id=editor_group_id) # an application that was created over 10 weeks ago (but updated recently) (todo_maned_follow_up_old) - self.build_application("maned_follow_up_old", 2 * w, 11 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) + self.build_application("maned_follow_up_old", 2 * w, 11 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, + editor_group_id=editor_group_id) # an application that was modifed recently into the ready status (todo_maned_ready) - self.build_application("maned_ready", 2 * w, 2 * w, constants.APPLICATION_STATUS_READY, apps) + self.build_application("maned_ready", 2 * w, 2 * w, constants.APPLICATION_STATUS_READY, apps, + editor_group_id=editor_group_id) # an application that was modifed recently into the ready status (todo_maned_completed) - self.build_application("maned_completed", 3 * w, 3 * w, constants.APPLICATION_STATUS_COMPLETED, apps) + self.build_application("maned_completed", 3 * w, 3 * w, constants.APPLICATION_STATUS_COMPLETED, apps, + editor_group_id=editor_group_id) # an application that was modifed recently into the ready status (todo_maned_assign_pending) def assign_pending(ap): ap.remove_editor() self.build_application("maned_assign_pending", 4 * w, 4 * w, constants.APPLICATION_STATUS_PENDING, apps, - assign_pending) + assign_pending, editor_group_id=editor_group_id) # an update request self.build_application("maned_update_request", 5 * w, 5 * w, constants.APPLICATION_STATUS_UPDATE_REQUEST, apps, - update_request=True) + update_request=True, editor_group_id=editor_group_id) # Applications that should never be reported ############################################ @@ -109,7 +115,8 @@ def assign_pending(ap): # maned_ready # maned_completed # maned_assign_pending - self.build_application("not_stalled__not_old", 2 * w, 2 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps) + self.build_application("not_stalled__not_old", 2 * w, 2 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, + editor_group_id=editor_group_id) # an application that is old but rejected # counter to maned_stalled @@ -117,7 +124,8 @@ def assign_pending(ap): # maned_ready # maned_completed # maned_assign_pending - self.build_application("old_rejected", 11 * w, 11 * w, constants.APPLICATION_STATUS_REJECTED, apps) + self.build_application("old_rejected", 11 * w, 11 * w, constants.APPLICATION_STATUS_REJECTED, apps, + editor_group_id=editor_group_id) # an application that was created/modified over 10 weeks ago and is accepted # counter to maned_stalled @@ -125,7 +133,8 @@ def assign_pending(ap): # maned_ready # maned_completed # maned_assign_pending - self.build_application("old_accepted", 12 * w, 12 * w, constants.APPLICATION_STATUS_ACCEPTED, apps) + self.build_application("old_accepted", 12 * w, 12 * w, constants.APPLICATION_STATUS_ACCEPTED, apps, + editor_group_id=editor_group_id) # an application that was recently completed (<2wk) # counter to maned_stalled @@ -133,7 +142,8 @@ def assign_pending(ap): # maned_ready # maned_completed # maned_assign_pending - self.build_application("old_accepted", 1 * w, 1 * w, constants.APPLICATION_STATUS_COMPLETED, apps) + self.build_application("old_accepted", 1 * w, 1 * w, constants.APPLICATION_STATUS_COMPLETED, apps, + editor_group_id=editor_group_id) # pending application with no assed younger than 2 weeks # counter to maned_stalled @@ -142,11 +152,12 @@ def assign_pending(ap): # maned_completed # maned_assign_pending self.build_application("not_assign_pending", 1 * w, 1 * w, constants.APPLICATION_STATUS_PENDING, apps, - assign_pending) + assign_pending, editor_group_id=editor_group_id) # pending application with assed assigned # counter to maned_assign_pending - self.build_application("pending_assed_assigned", 3 * w, 3 * w, constants.APPLICATION_STATUS_PENDING, apps) + self.build_application("pending_assed_assigned", 3 * w, 3 * w, constants.APPLICATION_STATUS_PENDING, apps, + editor_group_id=editor_group_id) # pending application with no editor group assigned # counter to maned_assign_pending @@ -154,17 +165,18 @@ def noeditorgroup(ap): ap.remove_editor_group() self.build_application("pending_assed_assigned", 3 * w, 3 * w, constants.APPLICATION_STATUS_PENDING, apps, - noeditorgroup) + noeditorgroup, editor_group_id=editor_group_id) # application with no assed, but not pending # counter to maned_assign_pending - self.build_application("no_assed", 3 * w, 3 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, assign_pending) + self.build_application("no_assed", 3 * w, 3 * w, constants.APPLICATION_STATUS_IN_PROGRESS, apps, + assign_pending, editor_group_id=editor_group_id) wait_until_no_es_incomplete_tasks() models.Application.refresh() # size = int(size_arg) - size=25 + size = 25 raises = None if raises_arg: @@ -194,10 +206,11 @@ def noeditorgroup(ap): assert actions.get(k, 0) == v[0] if v[1] > -1: assert v[1] in positions.get(k, []) - else: # the todo item is not positioned at all + else: # the todo item is not positioned at all assert len(positions.get(k, [])) == 0 - def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additional_fn=None, update_request=False): + def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additional_fn=None, + update_request=False, editor_group_id=None): source = ApplicationFixtureFactory.make_application_source() ap = models.Application(**source) ap.set_id(id) @@ -215,5 +228,8 @@ def build_application(self, id, lmu_diff, cd_diff, status, app_registry, additio if additional_fn is not None: additional_fn(ap) + if editor_group_id is not None: + ap.set_editor_group(editor_group_id) + ap.save() - app_registry.append(ap) \ No newline at end of file + app_registry.append(ap) From f00abe0fc27dad30cb627878095de6c50802e6cd Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 14:16:52 +0100 Subject: [PATCH 35/65] fix doc --- portality/view/doaj.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 5b7023daea..8620cd8748 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -268,16 +268,8 @@ def autocomplete_pair(doc_type, field_name, id_field): `text` used value of `field_name` parameter `q` will be used to search the prefix of `field_name` value - Parameters - ---------- - doc_type - field_name - id_field - - Returns - ------- - Json string with follow format: - {suggestions: [{id: id_field, text: field_name}]} + Json string with follow format: + {suggestions: [{id: id_field, text: field_name}]} """ prefix = request.args.get('q', '') From 09adc04a5ee374ddf7c0c82c0e32aa4b640f7e80 Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 14:29:36 +0100 Subject: [PATCH 36/65] query for editor group name --- portality/events/consumers/application_assed_assigned_notify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/events/consumers/application_assed_assigned_notify.py b/portality/events/consumers/application_assed_assigned_notify.py index 954a359832..8fe9b2002c 100644 --- a/portality/events/consumers/application_assed_assigned_notify.py +++ b/portality/events/consumers/application_assed_assigned_notify.py @@ -36,7 +36,7 @@ def consume(cls, event): notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN notification.long = svc.long_notification(cls.ID).format( journal_title=application.bibjson().title, - group_name=application.editor_group + group_name=models.EditorGroup.find_name_by_id(application.editor_group) or application.editor_group ) notification.short = svc.short_notification(cls.ID).format( issns=", ".join(issn for issn in application.bibjson().issns()) From 244db40f446ff3992c721b728821a6b0da57161e Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 14:31:53 +0100 Subject: [PATCH 37/65] cleanup KTODO --- portality/api/current/data_objects/application.py | 1 - portality/models/v2/journal.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/portality/api/current/data_objects/application.py b/portality/api/current/data_objects/application.py index 88ddfe0d58..a88e9ff32c 100644 --- a/portality/api/current/data_objects/application.py +++ b/portality/api/current/data_objects/application.py @@ -302,7 +302,6 @@ def editor_group(self): return self.__seamless__.get_single("admin.editor_group") def set_editor_group(self, eg): - # KTODO self.__seamless__.set_with_struct("admin.editor_group", eg) def remove_editor_group(self): diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 4f4bbdc860..6813d97a9c 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -220,17 +220,13 @@ def owner_account(self): return Account.pull(self.owner) return None - # KTODO add property self.editor_group_ids @property def editor_group(self): - # KTODO replace with self._cache_editor_group_name with query pull return self.__seamless__.get_single("admin.editor_group") - # KTODO add setter of editor_group for warning and DEBUGGING and redirect the value to self.editor_group_ids def set_editor_group(self, eg): - # KTODO add warning and DEBUGGING and redirect the value to self.editor_group_ids self.__seamless__.set_with_struct("admin.editor_group", eg) def remove_editor_group(self): From 78237d78ba3b037e341a011871efbd1234cb3d92 Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 15:39:09 +0100 Subject: [PATCH 38/65] update test cases --- .../event_consumers/test_application_assed_inprogress_notify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doajtest/unit/event_consumers/test_application_assed_inprogress_notify.py b/doajtest/unit/event_consumers/test_application_assed_inprogress_notify.py index 6b62e36368..244fce3eba 100644 --- a/doajtest/unit/event_consumers/test_application_assed_inprogress_notify.py +++ b/doajtest/unit/event_consumers/test_application_assed_inprogress_notify.py @@ -42,6 +42,7 @@ def test_consume_success(self): acc.save() eg = models.EditorGroup() + eg.set_id(app.editor_group) eg.set_name(app.editor_group) eg.set_maned(acc.id) eg.save(blocking=True) From fd4bff1dba04f0ac30e9e4b7445a749b71e31ea4 Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 14 May 2024 17:36:54 +0100 Subject: [PATCH 39/65] avoid globally replace module, it will affect globally and broken unrelated test cases when running in parallel --- .../test_editor_journal_review.py | 18 +++---------- .../test_maned_journal_review.py | 18 +++---------- .../test_journal_discontinuing_soon_notify.py | 15 +++-------- doajtest/unit/test_fc_editor_app_review.py | 27 +++++-------------- doajtest/unit/test_fc_maned_app_review.py | 27 +++++-------------- 5 files changed, 26 insertions(+), 79 deletions(-) diff --git a/doajtest/unit/application_processors/test_editor_journal_review.py b/doajtest/unit/application_processors/test_editor_journal_review.py index b09126425b..2f8401acd3 100644 --- a/doajtest/unit/application_processors/test_editor_journal_review.py +++ b/doajtest/unit/application_processors/test_editor_journal_review.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from doajtest.helpers import DoajTestCase from portality import models @@ -36,25 +38,13 @@ def mock_lookup_code(code): class TestEditorJournalReview(DoajTestCase): - def setUp(self): - super(TestEditorJournalReview, self).setUp() - - self.editor_group_pull = models.EditorGroup.pull - models.EditorGroup.pull = editor_group_pull - - self.old_lookup_code = lcc.lookup_code - lcc.lookup_code = mock_lookup_code - - def tearDown(self): - super(TestEditorJournalReview, self).tearDown() - models.EditorGroup.pull = self.editor_group_pull - lcc.lookup_code = self.old_lookup_code - ########################################################### # Tests on the publisher's re-journal form ########################################################### + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_01_editor_review_success(self): """Give the editor's journal form a full workout""" diff --git a/doajtest/unit/application_processors/test_maned_journal_review.py b/doajtest/unit/application_processors/test_maned_journal_review.py index d03419e76d..421865b254 100644 --- a/doajtest/unit/application_processors/test_maned_journal_review.py +++ b/doajtest/unit/application_processors/test_maned_journal_review.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from doajtest.helpers import DoajTestCase from portality import models @@ -32,20 +34,8 @@ def mock_lookup_code(code): class TestManEdJournalReview(DoajTestCase): - def setUp(self): - super(TestManEdJournalReview, self).setUp() - - self.editor_group_pull = models.EditorGroup.pull - models.EditorGroup.pull = editor_group_pull - - self.old_lookup_code = lcc.lookup_code - lcc.lookup_code = mock_lookup_code - - def tearDown(self): - super(TestManEdJournalReview, self).tearDown() - models.EditorGroup.pull = self.editor_group_pull - lcc.lookup_code = self.old_lookup_code - + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_01_maned_review_success(self): """Give the Managing Editor's journal form a full workout""" diff --git a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py index 3b00acc18c..3c097e2a56 100644 --- a/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py +++ b/doajtest/unit/event_consumers/test_journal_discontinuing_soon_notify.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from portality import models from portality import constants from portality.bll import exceptions @@ -26,17 +28,6 @@ def pull_editor_group(cls, value): return ed class TestJournalDiscontinuingSoonNotify(DoajTestCase): - def setUp(self): - super(TestJournalDiscontinuingSoonNotify, self).setUp() - self.pull_application = models.Application.pull - models.Application.pull = pull_application - self.pull_editor_group = models.EditorGroup.pull - models.EditorGroup.pull = pull_editor_group - - def tearDown(self): - super(TestJournalDiscontinuingSoonNotify, self).tearDown() - models.Application.pull = self.pull_application - models.EditorGroup.pull = self.pull_editor_group def test_consumes(self): @@ -52,6 +43,8 @@ def test_consumes(self): event = models.Event(constants.EVENT_JOURNAL_DISCONTINUING_SOON, context = {"journal": {"1234"}, "discontinue_date": "2002-22-02"}) assert JournalDiscontinuingSoonNotify.consumes(event) + @patch('portality.models.EditorGroup.pull', pull_editor_group) + @patch('portality.models.Application.pull', pull_application) def test_consume_success(self): self._make_and_push_test_context("/") diff --git a/doajtest/unit/test_fc_editor_app_review.py b/doajtest/unit/test_fc_editor_app_review.py index caa28876c5..ec30e1a0c0 100644 --- a/doajtest/unit/test_fc_editor_app_review.py +++ b/doajtest/unit/test_fc_editor_app_review.py @@ -1,5 +1,6 @@ import time from copy import deepcopy +from unittest.mock import patch from werkzeug.datastructures import MultiDict @@ -58,30 +59,13 @@ def make_application_form(): class TestEditorAppReview(DoajTestCase): - def setUp(self): - super(TestEditorAppReview, self).setUp() - - self.editor_group_pull = models.EditorGroup.pull - models.EditorGroup.pull = editor_group_pull - - self.old_lcc_choices = lcc.lcc_choices - lcc.lcc_choices = mock_lcc_choices - - self.old_lookup_code = lcc.lookup_code - lcc.lookup_code = mock_lookup_code - - def tearDown(self): - super(TestEditorAppReview, self).tearDown() - - models.EditorGroup.pull = self.editor_group_pull - lcc.lcc_choices = self.old_lcc_choices - - lcc.lookup_code = self.old_lookup_code - ########################################################### # Tests on the editor's application form ########################################################### + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lcc_choices', mock_lcc_choices) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_01_editor_review_success(self): """Give the editor's application form a full workout""" acc = models.Account() @@ -167,6 +151,9 @@ def test_01_editor_review_success(self): ctx.pop() + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lcc_choices', mock_lcc_choices) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_02_classification_required(self): # Check we can mark an application 'ready' with a subject classification present in_progress_application = models.Suggestion(**ApplicationFixtureFactory.make_update_request_source()) diff --git a/doajtest/unit/test_fc_maned_app_review.py b/doajtest/unit/test_fc_maned_app_review.py index 7d9e65f98b..284bf0bfa8 100644 --- a/doajtest/unit/test_fc_maned_app_review.py +++ b/doajtest/unit/test_fc_maned_app_review.py @@ -1,5 +1,6 @@ import time from copy import deepcopy +from unittest.mock import patch from werkzeug.datastructures import MultiDict @@ -58,26 +59,9 @@ def make_application_form(): class TestManEdAppReview(DoajTestCase): - def setUp(self): - super(TestManEdAppReview, self).setUp() - - self.editor_group_pull = models.EditorGroup.pull - models.EditorGroup.pull = editor_group_pull - - self.old_lcc_choices = lcc.lcc_choices - lcc.lcc_choices = mock_lcc_choices - - self.old_lookup_code = lcc.lookup_code - lcc.lookup_code = mock_lookup_code - - def tearDown(self): - super(TestManEdAppReview, self).tearDown() - - models.EditorGroup.pull_by_key = self.editor_group_pull - lcc.lcc_choices = self.old_lcc_choices - - lcc.lookup_code = self.old_lookup_code - + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lcc_choices', mock_lcc_choices) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_01_maned_review_success(self): """Give the editor's application form a full workout""" acc = models.Account() @@ -203,6 +187,9 @@ def test_02_update_request(self): ctx.pop() + @patch('portality.models.EditorGroup.pull', editor_group_pull) + @patch('portality.lcc.lcc_choices', mock_lcc_choices) + @patch('portality.lcc.lookup_code', mock_lookup_code) def test_03_classification_required(self): acc = models.Account() acc.set_id("steve") From 041e9f59738c03f1b2c329716390d596db71febd Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 6 Jun 2024 12:12:53 +0100 Subject: [PATCH 40/65] fix editor_group ids displayed --- doajtest/fixtures/editor_groups.py | 23 +++++++ doajtest/unit/test_view_doaj.py | 68 ++++++++++++++++++- portality/dao.py | 41 +++++++---- portality/models/editors.py | 3 +- portality/static/js/doaj.fieldrender.edges.js | 23 +++++++ .../static/js/edges/admin.journals.edge.js | 1 + .../js/edges/editor.groupapplications.edge.js | 1 + .../js/edges/editor.groupjournals.edge.js | 1 + portality/static/js/formulaic.js | 2 +- portality/view/doaj.py | 22 ++++-- 10 files changed, 160 insertions(+), 25 deletions(-) create mode 100644 doajtest/fixtures/editor_groups.py diff --git a/doajtest/fixtures/editor_groups.py b/doajtest/fixtures/editor_groups.py new file mode 100644 index 0000000000..52bf0ef6cc --- /dev/null +++ b/doajtest/fixtures/editor_groups.py @@ -0,0 +1,23 @@ +from portality import models + + +def create_editor_group_en(): + eg = models.EditorGroup() + eg.set_name("English") + eg.set_id('egid') + return eg + + +def create_editor_group_cn(): + eg = models.EditorGroup() + eg.set_name("Chinese") + eg.set_id('egid2') + return eg + + +def create_editor_group_jp(): + eg = models.EditorGroup() + eg.set_name("Japanese") + eg.set_id('egid3') + return eg + diff --git a/doajtest/unit/test_view_doaj.py b/doajtest/unit/test_view_doaj.py index cdbfeeb9e1..b9a5ed10e3 100644 --- a/doajtest/unit/test_view_doaj.py +++ b/doajtest/unit/test_view_doaj.py @@ -1,12 +1,15 @@ +import time + +from doajtest.fixtures.editor_groups import create_editor_group_en, create_editor_group_cn, create_editor_group_jp from doajtest.helpers import DoajTestCase from portality import models +from portality.view.doaj import id_text_mapping class TestViewDoaj(DoajTestCase): + def test_autocomplete_pair(self): - eg = models.EditorGroup() - eg.set_name("English") - eg.set_id('egid') + eg = create_editor_group_en() eg.save(blocking=True) with self.app_test.test_client() as client: @@ -44,4 +47,63 @@ def test_autocomplete_pair__unknown_doc_type(self): assert len(resp_json['suggestions']) == 1 assert resp_json['suggestions'][0]['id'] == '' + def test_autocomplete_text(self): + eg = create_editor_group_en() + eg.save(blocking=True) + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + response = client.get(f'/autocomplete-text/{doc_type}/{field_name}/{id_field}?id={eg.id}', ) + + assert response.status_code == 200 + assert response.json == {eg.id: eg.name} + + def test_autocomplete_text__not_found(self): + eg = create_editor_group_en() + eg.save(blocking=True) + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + id_val = 'qwjeqkwjeq' + response = client.get(f'/autocomplete-text/{doc_type}/{field_name}/{id_field}?id={id_val}', ) + + assert response.status_code == 200 + assert response.json == {} + + def test_autocomplete_text__ids(self): + eg = create_editor_group_en() + eg2 = create_editor_group_cn() + eg3 = create_editor_group_jp() + + models.EditorGroup.save_all_block_last([eg, eg2, eg3]) + + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + response = client.get(f'/autocomplete-text/{doc_type}/{field_name}/{id_field}', + json={'ids': [eg.id, eg2.id]}) + + assert response.status_code == 200 + assert response.json == {eg.id: eg.name, eg2.id: eg2.name, } + + def test_autocomplete_text__ids_empty(self): + with self.app_test.test_client() as client: + doc_type = 'editor_group' + field_name = 'name' + id_field = 'id' + response = client.get(f'/autocomplete-text/{doc_type}/{field_name}/{id_field}', + json={'ids': []}) + + assert response.status_code == 200 + assert response.json == {} + + def test_id_text_mapping(self): + models.EditorGroup.save_all_block_last([create_editor_group_en(), create_editor_group_cn(), + create_editor_group_jp()]) + time.sleep(5) + assert id_text_mapping('editor_group', 'name', 'id', 'egid') == {'egid': 'English'} + assert id_text_mapping('editor_group', 'name', 'id', 'egid2') == {'egid2': 'Chinese'} diff --git a/portality/dao.py b/portality/dao.py index 629982a7a3..fbd2633cda 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -1,3 +1,4 @@ +from __future__ import annotations import json import re import sys @@ -843,7 +844,7 @@ def autocomplete(cls, field, prefix, size=5) -> List[IdText]: return result @classmethod - def autocomplete_pair(cls, query_field, query_prefix, id_field, text_field, size=5) -> List[IdText]: + def autocomplete_pair(cls, query_field, query_prefix, id_field, text_field, size=5) -> list[IdText]: query = AutocompletePairQuery(query_field, query_prefix, source_fields=[id_field, text_field], size=size).query() @@ -851,20 +852,25 @@ def autocomplete_pair(cls, query_field, query_prefix, id_field, text_field, size return [{"id": getattr(obj, id_field), "text": getattr(obj, text_field)} for obj in objs] @classmethod - def get_target_value(cls, query, field_name) -> Optional[str]: - query['_source'] = [field_name] - query['size'] = 2 - res = cls.query(q=query) - size = res.get('hits', {}).get('total', {}).get('value', 0) - - if size > 1: + def get_target_value(cls, query, id_name, field_name) -> str | None: + results = cls.get_target_values(query, id_name, field_name, size=2) + if len(results) > 1: app.logger.debug("More than one record found for query {q}".format(q=json.dumps(query, indent=2))) - if size > 0: - return res['hits']['hits'][0]['_source'].get(field_name) + if len(results) > 0: + return list(results.values())[0] return None + + @classmethod + def get_target_values(cls, query, id_name, field_name, size=1000) -> dict[str]: + query['_source'] = [id_name, field_name] + query['size'] = size + res = cls.query(q=query) + return {hit['_source'].get(id_name): hit['_source'].get(field_name) + for hit in res['hits']['hits']} + @classmethod def q2obj(cls, **kwargs) -> List['Self']: extra_trace_info = '' @@ -966,6 +972,13 @@ def save_all(cls, models, blocking=False): if blocking: cls.blockall((m.id, getattr(m, "last_updated", None)) for m in models) + @classmethod + def save_all_block_last(cls, objects): + *objs, last = objects + for obj in objects: + obj.save() + last.save(blocking=True) + def any_pending_tasks(): """ Check if there are any pending tasks in the elasticsearch task queue """ @@ -1166,9 +1179,11 @@ def __init__(self, id_field, id_value): self.id_value = id_value def query(self): - return { - "query": {"term": {self.id_field: self.id_value}} - } + if isinstance(self.id_value, list): + term = {"terms": {self.id_field: self.id_value}} + else: + term = {"term": {self.id_field: self.id_value}} + return {"query": term} def patch_model_for_bulk(obj: DomainObject): diff --git a/portality/models/editors.py b/portality/models/editors.py index 8c2c8d3a32..8e91b4b964 100644 --- a/portality/models/editors.py +++ b/portality/models/editors.py @@ -49,7 +49,8 @@ def groups_by_associate(cls, associate) -> List['Self']: @classmethod def find_name_by_id(cls, editor_group_id) -> Optional[str]: - return cls.get_target_value(IdTextTermQuery('id', editor_group_id).query(), "name") + return cls.get_target_value(IdTextTermQuery('id', editor_group_id).query(), + id_name=editor_group_id, field_name="name") @property def name(self): diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index dc3138e4f8..046fb6952d 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -128,6 +128,7 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor group", deactivateThreshold: 1, + valueFunction: new doaj.fieldRender.editorGroupNameFactory(), renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, @@ -3898,6 +3899,28 @@ $.extend(true, doaj, { return false; } }, + + editorGroupNameFactory: function() { + let resultMap = null; + this.mapCallable = function (val) { + return resultMap[val] || val + } + this.mapCallable.prepareMap = function (values) { + $.ajax({ + url: "/autocomplete-text/editor_group/name/id", + type: "POST", + contentType: "application/json", + dataType: 'json', + data: JSON.stringify({'ids': values}) , + async: false, + success: function (data) { + resultMap = data; + } + }); + } + + return this.mapCallable + } }, bulk : { diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index e612a2c7f5..f0dea20541 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -135,6 +135,7 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor group", deactivateThreshold: 1, + valueFunction: new doaj.fieldRender.editorGroupNameFactory(), renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index 5e40fd9e22..c8bd463550 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -78,6 +78,7 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor Group", deactivateThreshold: 1, + valueFunction: new doaj.fieldRender.editorGroupNameFactory(), renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index 9c145ba018..22b5b5160e 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -55,6 +55,7 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor Group", deactivateThreshold: 1, + valueFunction: new doaj.fieldRender.editorGroupNameFactory(), renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, diff --git a/portality/static/js/formulaic.js b/portality/static/js/formulaic.js index 15af4501e2..c318470932 100644 --- a/portality/static/js/formulaic.js +++ b/portality/static/js/formulaic.js @@ -2190,7 +2190,7 @@ var formulaic = { dataType: "json", url: `${current_scheme}//${current_domain}/autocomplete-text/${doc_type}/${doc_field}/${id_field}`, success: function(resp) { - setIdText(eleVal, resp?.text); + setIdText(eleVal, resp[eleVal]); } }) }; diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 8620cd8748..2f8705c6c7 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -1,3 +1,4 @@ +from __future__ import annotations import json import re import urllib.error @@ -289,14 +290,21 @@ def autocomplete_pair(doc_type, field_name, id_field): def autocomplete_text_mapping(doc_type, field_name, id_field): """ This route is used by the autocomplete widget to get the text value by id + + Json string with follow format: + {id_val: text_val} """ id_value = request.args.get('id') - text = id_text_mapping(doc_type, field_name, id_field, id_value) - return jsonify({'id': id_value, 'text': text or ''}) + if id_value is None: + id_value = request.json.get('ids', []) + else: + id_value = id_value.strip() + id_map = id_text_mapping(doc_type, field_name, id_field, id_value) + return jsonify(id_map) -def id_text_mapping(doc_type, field_name, id_field, id_value) -> Optional[str]: +def id_text_mapping(doc_type, field_name, id_field, id_value) -> dict: query_factory_mapping = { ('editor_group', 'id', 'name', ): lambda: IdTextTermQuery(id_field, id_value).query(), @@ -305,18 +313,18 @@ def id_text_mapping(doc_type, field_name, id_field, id_value) -> Optional[str]: if not query_factory: app.logger.warning(f"Unsupported id_text_mapping for " f"doc_type[{doc_type}], field_name[{field_name}], id_field[{id_field}]") - return None + return {} if not id_value: - return None + return {} m = models.lookup_model(doc_type) if not m: app.logger.warning(f"model not found for doc_type[{doc_type}]") - return None + return {} query = query_factory() - return m.get_target_value(query, field_name) + return m.get_target_values(query, id_field, field_name) From 392cee1f5bac7f55035175152a8df2e6d6afee2e Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 6 Jun 2024 13:11:42 +0100 Subject: [PATCH 41/65] fix display name in newSelectedFilters --- portality/static/js/doaj.fieldrender.edges.js | 3 +++ portality/static/js/edges/admin.applications.edge.js | 3 +++ portality/static/js/edges/admin.journals.edge.js | 3 +++ portality/static/js/edges/admin.update_requests.edge.js | 5 ++++- portality/static/js/edges/editor.groupapplications.edge.js | 5 ++++- portality/static/js/edges/editor.groupjournals.edge.js | 5 ++++- 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 046fb6952d..829e3b459a 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -3906,6 +3906,9 @@ $.extend(true, doaj, { return resultMap[val] || val } this.mapCallable.prepareMap = function (values) { + if (values.length === 0) { + return + } $.ajax({ url: "/autocomplete-text/editor_group/name/id", type: "POST", diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index c43bf55d5e..5ef545fb3f 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -229,6 +229,9 @@ $.extend(true, doaj, { "new application": "Open" } }, + valueFunctions : { + "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + }, renderer : doaj.renderers.newSelectedFiltersRenderer({ omit : [ "bibjson.apc.has_apc", diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index f0dea20541..e6c22d1238 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -493,6 +493,9 @@ $.extend(true, doaj, { false : "No" } }, + valueFunctions : { + "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + }, rangeFunctions : { "bibjson.discontinued_date" : doaj.valueMaps.displayYearPeriod } diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index cea56910fe..ea213590c0 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -229,7 +229,10 @@ $.extend(true, doaj, { "update request": "Open", "new application": "Open" } - } + }, + valueFunctions : { + "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + }, }) ]; diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index c8bd463550..5e4b3c8de7 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -383,7 +383,10 @@ $.extend(true, doaj, { 'bibjson.publisher.name.exact' : 'Publisher', 'index.license.exact' : 'Journal license', "index.has_apc.exact" : "Publication charges?" - } + }, + valueFunctions : { + "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + }, }) ]; diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index 22b5b5160e..dcce8bced4 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -382,7 +382,10 @@ $.extend(true, doaj, { true : "True", false : "False" } - } + }, + valueFunctions : { + "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + }, }) ]; From a60d662632a5c69c5db07ea7bf9693f3b9a90081 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 6 Jun 2024 13:17:47 +0100 Subject: [PATCH 42/65] add _translatePrepare SelectedFilters --- portality/static/vendor/edges | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/static/vendor/edges b/portality/static/vendor/edges index 990f422016..edfc88fc17 160000 --- a/portality/static/vendor/edges +++ b/portality/static/vendor/edges @@ -1 +1 @@ -Subproject commit 990f4220163a3e18880f0bdc3ad5c80d234d22dd +Subproject commit edfc88fc174bd7508657373e8b044ef1a6824ac5 From ef713f65ea3488ebc0a1631cc425a48d9cee7d56 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 17 Jun 2024 13:28:42 +0100 Subject: [PATCH 43/65] implement editorGroupName callback for selectedFilters, Result, leftMenu --- portality/static/js/doaj.fieldrender.edges.js | 50 ++++++++++--------- .../js/edges/admin.applications.edge.js | 8 ++- .../static/js/edges/admin.journals.edge.js | 13 +++-- .../js/edges/admin.update_requests.edge.js | 8 ++- .../js/edges/editor.groupapplications.edge.js | 13 +++-- .../js/edges/editor.groupjournals.edge.js | 13 +++-- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 829e3b459a..2af193ac67 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -128,13 +128,14 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor group", deactivateThreshold: 1, - valueFunction: new doaj.fieldRender.editorGroupNameFactory(), + valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: doaj.valueMaps.countFormat, - hideInactive: true + hideInactive: true, + noDisplayEscape: true, }) }) }, @@ -3900,29 +3901,30 @@ $.extend(true, doaj, { } }, - editorGroupNameFactory: function() { - let resultMap = null; - this.mapCallable = function (val) { - return resultMap[val] || val - } - this.mapCallable.prepareMap = function (values) { - if (values.length === 0) { - return - } - $.ajax({ - url: "/autocomplete-text/editor_group/name/id", - type: "POST", - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({'ids': values}) , - async: false, - success: function (data) { - resultMap = data; - } - }); - } + editorGroupNameCallback: function(val, resultobj, renderer) { + return `${val}` + }, + editorGroupNameTrigger: function(queryResult) { + /** + * used in edges:post-render to trigger the editor group name callback + */ - return this.mapCallable + let editorGroupIds = queryResult.data.hits.hits.map(i => i._source.admin.editor_group) + editorGroupIds.push(...queryResult.data.aggregations.editor_group.buckets.map(i => i.key)) + editorGroupIds = editorGroupIds.filter(i => i !== undefined) + $.ajax({ + url: "/autocomplete-text/editor_group/name/id", + type: "POST", + contentType: "application/json", + dataType: 'json', + data: JSON.stringify({'ids': editorGroupIds}) , + success: function (data) { + $('.editorGroupNameCallback').each(function() { + const id = $(this).data('id') + $(this).text(edges.escapeHtml(data[id] || id)) + }) + } + }); } }, diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index 5ef545fb3f..69e9c4d9fb 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -133,7 +133,8 @@ $.extend(true, doaj, { [ { "pre" : "Editor group: ", - "field" : "admin.editor_group" + "field" : "admin.editor_group", + valueFunction: doaj.fieldRender.editorGroupNameCallback, } ], [ @@ -230,7 +231,7 @@ $.extend(true, doaj, { } }, valueFunctions : { - "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, }, renderer : doaj.renderers.newSelectedFiltersRenderer({ omit : [ @@ -253,6 +254,9 @@ $.extend(true, doaj, { callbacks : { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); + }, + "edges:post-render" : function () { + doaj.fieldRender.editorGroupNameTrigger(e.result); } } }); diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index e6c22d1238..41ed54e3a2 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -135,13 +135,14 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor group", deactivateThreshold: 1, - valueFunction: new doaj.fieldRender.editorGroupNameFactory(), + valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, - hideInactive: true + hideInactive: true, + noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -377,7 +378,8 @@ $.extend(true, doaj, { [ { "pre" : "Editor group: ", - "field" : "admin.editor_group" + "field" : "admin.editor_group", + valueFunction: doaj.fieldRender.editorGroupNameCallback, } ], [ @@ -494,7 +496,7 @@ $.extend(true, doaj, { } }, valueFunctions : { - "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, }, rangeFunctions : { "bibjson.discontinued_date" : doaj.valueMaps.displayYearPeriod @@ -514,6 +516,9 @@ $.extend(true, doaj, { callbacks : { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); + }, + "edges:post-render" : function () { + doaj.fieldRender.editorGroupNameTrigger(e.result); } } }); diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index ea213590c0..453f278da3 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -144,7 +144,8 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group" + "field" : "admin.editor_group", + valueFunction: doaj.fieldRender.editorGroupNameCallback, } ], [ @@ -231,7 +232,7 @@ $.extend(true, doaj, { } }, valueFunctions : { - "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, }, }) ]; @@ -248,6 +249,9 @@ $.extend(true, doaj, { callbacks : { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); + }, + "edges:post-render" : function () { + doaj.fieldRender.editorGroupNameTrigger(e.result); } } }); diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index 5e4b3c8de7..ff0983f09b 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -78,13 +78,14 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor Group", deactivateThreshold: 1, - valueFunction: new doaj.fieldRender.editorGroupNameFactory(), + valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, - hideInactive: true + hideInactive: true, + noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -294,7 +295,8 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group" + "field" : "admin.editor_group", + valueFunction: doaj.fieldRender.editorGroupNameCallback, } ], [ @@ -385,7 +387,7 @@ $.extend(true, doaj, { "index.has_apc.exact" : "Publication charges?" }, valueFunctions : { - "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, }, }) ]; @@ -402,6 +404,9 @@ $.extend(true, doaj, { callbacks : { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); + }, + "edges:post-render" : function () { + doaj.fieldRender.editorGroupNameTrigger(e.result); } } }); diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index dcce8bced4..c33b16ef9c 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -55,13 +55,14 @@ $.extend(true, doaj, { field: "admin.editor_group.exact", display: "Editor Group", deactivateThreshold: 1, - valueFunction: new doaj.fieldRender.editorGroupNameFactory(), + valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, - hideInactive: true + hideInactive: true, + noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -275,7 +276,8 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group" + "field" : "admin.editor_group", + valueFunction: doaj.fieldRender.editorGroupNameCallback, } ], [ @@ -384,7 +386,7 @@ $.extend(true, doaj, { } }, valueFunctions : { - "admin.editor_group.exact" : new doaj.fieldRender.editorGroupNameFactory(), + "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, }, }) ]; @@ -401,6 +403,9 @@ $.extend(true, doaj, { callbacks : { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); + }, + "edges:post-render" : function () { + doaj.fieldRender.editorGroupNameTrigger(e.result); } } }); From 2cfcb288d9c911c17b359f59c115dceafad00d7d Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 28 Jun 2024 10:57:25 +0100 Subject: [PATCH 44/65] remove useless import --- portality/models/v2/application.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/portality/models/v2/application.py b/portality/models/v2/application.py index 0a17584bec..877df6c729 100644 --- a/portality/models/v2/application.py +++ b/portality/models/v2/application.py @@ -1,7 +1,6 @@ from copy import deepcopy -from typing import Iterable -from portality import constants, models +from portality import constants from portality.core import app from portality.lib import es_data_mapping, coerce, dates from portality.models.v2 import shared_structs From 18d0a1e7c3637c2ab35a4235b3e92e22f2765c31 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 28 Jun 2024 11:18:55 +0100 Subject: [PATCH 45/65] fix empty editorGroupName --- portality/static/js/doaj.fieldrender.edges.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 2af193ac67..0782cc4900 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -3902,7 +3902,11 @@ $.extend(true, doaj, { }, editorGroupNameCallback: function(val, resultobj, renderer) { - return `${val}` + if (val) { + return `${val}` + } else { + return val + } }, editorGroupNameTrigger: function(queryResult) { /** From 3faac5754baee34b8ba95d1e0c5daef83a05181c Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 1 Jul 2024 09:19:40 +0100 Subject: [PATCH 46/65] remove escape because some editor group name contain `&` --- portality/static/js/doaj.fieldrender.edges.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 0782cc4900..3dc5673902 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -3925,7 +3925,7 @@ $.extend(true, doaj, { success: function (data) { $('.editorGroupNameCallback').each(function() { const id = $(this).data('id') - $(this).text(edges.escapeHtml(data[id] || id)) + $(this).text(data[id] || id) }) } }); From 7a2cc34e45eb8fa3eb5f603f7cfb12e5044235d2 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 10 Jul 2024 11:28:33 +0100 Subject: [PATCH 47/65] add editor_group_name to index --- portality/models/v2/journal.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 6813d97a9c..00bd62a49e 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -46,7 +46,8 @@ "index": { "fields": { "publisher_ac": {"coerce": "unicode"}, - "institution_ac": {"coerce": "unicode"} + "institution_ac": {"coerce": "unicode"}, + "editor_group_name": {"coerce": "unicode"}, } } } @@ -231,6 +232,14 @@ def set_editor_group(self, eg): def remove_editor_group(self): self.__seamless__.delete("admin.editor_group") + self.__seamless__.delete("index.editor_group_name") + + def editor_group_name(self, index=True): + if index: + return self.__seamless__.get_single("index.editor_group_name") + elif self.editor_group: + from portality.models import EditorGroup + return EditorGroup.find_name_by_id(self.editor_group) @property def editor(self): @@ -453,6 +462,8 @@ def _generate_index(self): if self.editor is not None: has_editor = "Yes" + editor_group_name = self.editor_group_name(index=False) + # build the index part of the object index = {} @@ -487,6 +498,8 @@ def _generate_index(self): index["schema_code"] = schema_codes if len(schema_codes_tree) > 0: index["schema_codes_tree"] = schema_codes_tree + if editor_group_name: + index["editor_group_name"] = editor_group_name self.__seamless__.set_with_struct("index", index) From 318ca1c07aa9e58961079b335088fa8c34c211e0 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 10 Jul 2024 12:20:32 +0100 Subject: [PATCH 48/65] use editor_group_name --- portality/bll/services/todo.py | 2 +- .../application_assed_assigned_notify.py | 2 +- portality/models/v2/journal.py | 16 ++++++++++++---- portality/models/v2/shared_structs.py | 3 ++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/portality/bll/services/todo.py b/portality/bll/services/todo.py index 0cfbbed124..3b6f02ea84 100644 --- a/portality/bll/services/todo.py +++ b/portality/bll/services/todo.py @@ -136,7 +136,7 @@ def top_todo(self, account, size=25, new_applications=True, update_requests=True "boost": boost, } if ap.editor_group: - todo["editor_group_name"] = models.EditorGroup.find_name_by_id(ap.editor_group) or ap.editor_group + todo["editor_group_name"] = ap.editor_group_name(default_id=True) todos.append(todo) todos = self._rationalise_todos(todos, size) diff --git a/portality/events/consumers/application_assed_assigned_notify.py b/portality/events/consumers/application_assed_assigned_notify.py index 772b136023..c7f1ae19f3 100644 --- a/portality/events/consumers/application_assed_assigned_notify.py +++ b/portality/events/consumers/application_assed_assigned_notify.py @@ -33,7 +33,7 @@ def consume(cls, event): notification.classification = constants.NOTIFICATION_CLASSIFICATION_ASSIGN notification.long = svc.long_notification(cls.ID).format( journal_title=application.bibjson().title, - group_name=models.EditorGroup.find_name_by_id(application.editor_group) or application.editor_group + group_name=application.editor_group_name(default_id=True) ) notification.short = svc.short_notification(cls.ID).format( issns=application.bibjson().issns_as_text() diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 00bd62a49e..91f5f7d5e6 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -1,3 +1,4 @@ +from __future__ import annotations from portality.dao import DomainObject from portality.core import app from portality.lib.dates import DEFAULT_TIMESTAMP_VAL @@ -47,7 +48,6 @@ "fields": { "publisher_ac": {"coerce": "unicode"}, "institution_ac": {"coerce": "unicode"}, - "editor_group_name": {"coerce": "unicode"}, } } } @@ -234,12 +234,20 @@ def remove_editor_group(self): self.__seamless__.delete("admin.editor_group") self.__seamless__.delete("index.editor_group_name") - def editor_group_name(self, index=True): + def editor_group_name(self, index=True, default_id=False) -> str | None: + name = None if index: - return self.__seamless__.get_single("index.editor_group_name") + name = self.__seamless__.get_single("index.editor_group_name") elif self.editor_group: from portality.models import EditorGroup - return EditorGroup.find_name_by_id(self.editor_group) + name = EditorGroup.find_name_by_id(self.editor_group) + + if default_id: + name = name or self.editor_group + + return name + + @property def editor(self): diff --git a/portality/models/v2/shared_structs.py b/portality/models/v2/shared_structs.py index 6c2c031af1..71f55c73b4 100644 --- a/portality/models/v2/shared_structs.py +++ b/portality/models/v2/shared_structs.py @@ -221,7 +221,8 @@ "asciiunpunctitle" : {"coerce" : "unicode"}, "continued" : {"coerce" : "unicode"}, "has_editor_group" : {"coerce" : "unicode"}, - "has_editor" : {"coerce" : "unicode"} + "has_editor" : {"coerce" : "unicode"}, + "editor_group_name" : {'coerce' : 'unicode'}, }, "lists" : { "issn" : {"contains" : "field", "coerce" : "unicode"}, From e9de8f0a93584e184597b909ae96bc06d4b70e9a Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 10 Jul 2024 12:20:50 +0100 Subject: [PATCH 49/65] clean up --- portality/forms/application_forms.py | 9 +++++---- portality/view/doaj.py | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/portality/forms/application_forms.py b/portality/forms/application_forms.py index b440c302e6..fbdc76a64d 100644 --- a/portality/forms/application_forms.py +++ b/portality/forms/application_forms.py @@ -1803,8 +1803,8 @@ class FieldDefinitions: "label": "Group", "input": "text", "widgets": [ - {"autocomplete": {"type": "editor_group", "field": "name", "include": False, - "id_field": "id" }} # ~~^-> Autocom dget~~ + {"autocomplete": {"type": "editor_group", "field": "name", + "include": False, "id_field": "id" }} # ~~^-> Autocom dget~~ ], "contexts": { "editor": { @@ -1812,8 +1812,9 @@ class FieldDefinitions: }, "admin" : { "widgets" : [ - {"autocomplete": {"type": "editor_group", "field": "name", "include" : False, "id_field": "id" }}, # ~~^-> Autocomplete:FormWidget~~ - {"load_editors" : {"field" : "editor"}} + {"autocomplete": {"type": "editor_group", "field": "name", + "include": False, "id_field": "id" }}, # ~~^-> Autocomplete:FormWidget~~ + {"load_editors" : {"field": "editor"}} ] } } diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 985f775b1b..587ab3afb5 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -4,7 +4,6 @@ import urllib.error import urllib.parse import urllib.request -from typing import Optional from flask import Blueprint, request, make_response from flask import render_template, abort, redirect, url_for, send_file, jsonify From 9104f3907e42dbd43df936c08bacfbda5d8bc5ff Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 10 Jul 2024 13:17:26 +0100 Subject: [PATCH 50/65] show name by index --- portality/static/js/edges/admin.applications.edge.js | 3 +-- portality/static/js/edges/admin.journals.edge.js | 3 +-- portality/static/js/edges/admin.update_requests.edge.js | 3 +-- portality/static/js/edges/editor.groupapplications.edge.js | 3 +-- portality/static/js/edges/editor.groupjournals.edge.js | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index 69e9c4d9fb..c4df467407 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -133,8 +133,7 @@ $.extend(true, doaj, { [ { "pre" : "Editor group: ", - "field" : "admin.editor_group", - valueFunction: doaj.fieldRender.editorGroupNameCallback, + "field" : "index.editor_group_name", } ], [ diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index 41ed54e3a2..f09008df54 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -378,8 +378,7 @@ $.extend(true, doaj, { [ { "pre" : "Editor group: ", - "field" : "admin.editor_group", - valueFunction: doaj.fieldRender.editorGroupNameCallback, + "field" : "index.editor_group_name", } ], [ diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index 453f278da3..6701e388dc 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -144,8 +144,7 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group", - valueFunction: doaj.fieldRender.editorGroupNameCallback, + "field" : "index.editor_group_name", } ], [ diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index ff0983f09b..d5e5225f3f 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -295,8 +295,7 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group", - valueFunction: doaj.fieldRender.editorGroupNameCallback, + "field" : "index.editor_group_name", } ], [ diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index c33b16ef9c..8bf195943a 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -276,8 +276,7 @@ $.extend(true, doaj, { [ { "pre" : "Editor Group: ", - "field" : "admin.editor_group", - valueFunction: doaj.fieldRender.editorGroupNameCallback, + "field" : "index.editor_group_name", } ], [ From bc160176908c40c9290128c84af691f998167797 Mon Sep 17 00:00:00 2001 From: philip Date: Wed, 10 Jul 2024 13:27:45 +0100 Subject: [PATCH 51/65] rename mange-bgjobs.md --- docs/dev/user-guide/{user-guide.md => manage-bgjobs.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/dev/user-guide/{user-guide.md => manage-bgjobs.md} (100%) diff --git a/docs/dev/user-guide/user-guide.md b/docs/dev/user-guide/manage-bgjobs.md similarity index 100% rename from docs/dev/user-guide/user-guide.md rename to docs/dev/user-guide/manage-bgjobs.md From a50f3fa7ab30daabfff6c34bb3e8b76f5264d2f0 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 11 Jul 2024 11:04:32 +0100 Subject: [PATCH 52/65] add renew_editor_group_name --- portality/models/v2/journal.py | 56 +++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 91f5f7d5e6..66656e0306 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -12,7 +12,7 @@ from copy import deepcopy from datetime import datetime, timedelta -import string, uuid +from elasticsearch import helpers from unidecode import unidecode JOURNAL_STRUCT = { @@ -125,6 +125,41 @@ def recent(cls, max=10): records = [cls(**r.get("_source")) for r in result.get("hits", {}).get("hits", [])] return records + @classmethod + def renew_editor_group_name(cls, + editor_group_id: str, + new_editor_group_name: str, + batch_size: int = 10000, + ): + """ + Renew editor_group_name in the index. + for ALL documents with related editor_group_id. + """ + + query = ByEditorGroupIdQuery(editor_group_id).query() + query['_source'] = ['id'] + + def _to_action(_id): + return { + "_op_type": "update", + "_index": cls.index_name(), + "_id": _id, + "script": { + "source": "ctx._source.index.editor_group_name = params.new_value", + "lang": "painless", + "params": { + "new_value": new_editor_group_name + } + } + } + + ids = [j['id'] for j in cls.iterate(q=query, wrap=False, page_size=batch_size, keepalive='5m')] + app.logger.info(f"Found {len(ids)} {cls.__name__} with editor_group_id: {editor_group_id}") + + id_batches = (ids[i:i + batch_size] for i in range(0, len(ids), batch_size)) + for _sub_ids in id_batches: + helpers.bulk(dao.ES, [_to_action(_id) for _id in _sub_ids]) + ############################################ ## base property methods @@ -1191,3 +1226,22 @@ def query(self): {"created_date" : {"order" : "desc"}} ] } + + +class ByEditorGroupIdQuery: + + def __init__(self, editor_group_id): + self.editor_group_id = editor_group_id + + def query(self): + return { + 'query': { + 'bool': { + 'filter': { + 'term': { + 'admin.editor_group.exact': self.editor_group_id + } + } + } + }, + } From b1420e97b105b42d6036b7dc1fa96f28e66ed055 Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 11 Jul 2024 11:05:42 +0100 Subject: [PATCH 53/65] use huey_helper function --- portality/tasks/anon_export.py | 9 ++++----- portality/tasks/article_bulk_create.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/portality/tasks/anon_export.py b/portality/tasks/anon_export.py index 234fc7727f..5fb7ec72be 100644 --- a/portality/tasks/anon_export.py +++ b/portality/tasks/anon_export.py @@ -187,12 +187,11 @@ def submit(cls, background_job): @huey_helper.register_schedule def scheduled_anon_export(): - background_helper.submit_by_bg_task_type(AnonExportBackgroundTask, - clean=app.config.get("TASKS_ANON_EXPORT_CLEAN", False), - limit=app.config.get("TASKS_ANON_EXPORT_LIMIT", None), - batch_size=app.config.get("TASKS_ANON_EXPORT_BATCH_SIZE", 100000)) + huey_helper.scheduled_common(clean=app.config.get("TASKS_ANON_EXPORT_CLEAN", False), + limit=app.config.get("TASKS_ANON_EXPORT_LIMIT", None), + batch_size=app.config.get("TASKS_ANON_EXPORT_BATCH_SIZE", 100000)) @huey_helper.register_execute(is_load_config=False) def anon_export(job_id): - background_helper.execute_by_job_id(job_id, AnonExportBackgroundTask) + huey_helper.execute_common(job_id) diff --git a/portality/tasks/article_bulk_create.py b/portality/tasks/article_bulk_create.py index 3066ecaefe..13dbf25b20 100644 --- a/portality/tasks/article_bulk_create.py +++ b/portality/tasks/article_bulk_create.py @@ -83,4 +83,4 @@ def submit(cls, background_job): @huey_helper.register_execute(is_load_config=False) def article_bulk_create(job_id): - background_helper.execute_by_job_id(job_id, ArticleBulkCreateBackgroundTask) \ No newline at end of file + huey_helper.execute_common(job_id) \ No newline at end of file From 80e06acc2d6a7a24fef540a953753a47d317c4bf Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 11 Jul 2024 11:06:26 +0100 Subject: [PATCH 54/65] cleanup --- portality/models/v2/journal.py | 54 +++++++++++++++++----------------- portality/tasks/anon_export.py | 3 +- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index 66656e0306..fe96159d23 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -1,20 +1,24 @@ from __future__ import annotations -from portality.dao import DomainObject -from portality.core import app -from portality.lib.dates import DEFAULT_TIMESTAMP_VAL -from portality.models.v2.bibjson import JournalLikeBibJSON -from portality.models.v2 import shared_structs -from portality.models.account import Account -from portality.lib import es_data_mapping, dates, coerce -from portality.lib.seamless import SeamlessMixin -from portality.lib.coerce import COERCE_MAP +import string +import uuid from copy import deepcopy from datetime import datetime, timedelta from elasticsearch import helpers from unidecode import unidecode +from portality import dao +from portality.core import app +from portality.dao import DomainObject +from portality.lib import es_data_mapping, dates, coerce +from portality.lib.coerce import COERCE_MAP +from portality.lib.dates import DEFAULT_TIMESTAMP_VAL +from portality.lib.seamless import SeamlessMixin +from portality.models.account import Account +from portality.models.v2 import shared_structs +from portality.models.v2.bibjson import JournalLikeBibJSON + JOURNAL_STRUCT = { "objects": [ "admin", "index" @@ -256,12 +260,10 @@ def owner_account(self): return Account.pull(self.owner) return None - @property def editor_group(self): return self.__seamless__.get_single("admin.editor_group") - def set_editor_group(self, eg): self.__seamless__.set_with_struct("admin.editor_group", eg) @@ -282,8 +284,6 @@ def editor_group_name(self, index=True, default_id=False) -> str | None: return name - - @property def editor(self): return self.__seamless__.get_single("admin.editor") @@ -334,7 +334,6 @@ def add_note_by_dict(self, note): return self.add_note(note=note.get("note"), date=note.get("date"), id=note.get("id"), author_id=note.get("author_id")) - def remove_note(self, note): self.__seamless__.delete_from_list("admin.notes", matchsub=note) @@ -355,7 +354,8 @@ def ordered_notes(self): clusters = {} for note in notes: if "date" not in note: - note["date"] = DEFAULT_TIMESTAMP_VAL # this really means something is broken with note date setting, which needs to be fixed + # this really means something is broken with note date setting, which needs to be fixed + note["date"] = DEFAULT_TIMESTAMP_VAL if note["date"] not in clusters: clusters[note["date"]] = [note] else: @@ -563,7 +563,7 @@ def __init__(self, **kwargs): if "_source" in kwargs: kwargs = kwargs["_source"] # FIXME: I have taken this out for the moment, as I'm not sure it's what we should be doing - #if kwargs: + # if kwargs: # self.add_autogenerated_fields(**kwargs) super(Journal, self).__init__(raw=kwargs) @@ -593,7 +593,7 @@ def add_autogenerated_fields(cls, **kwargs): bib["pid_scheme"] = {"has_pid_scheme": False} if "preservation" in bib and bib["preservation"] != '': bib["preservation"]["has_preservation"] = (len(bib["preservation"]) != 0 or - bib["national_library"] is not None) + bib["national_library"] is not None) else: bib["preservation"] = {"has_preservation": True} @@ -822,7 +822,8 @@ def remove_related_applications(self): self.__seamless__.delete("admin.related_applications") def remove_related_application(self, application_id): - self.set_related_applications([r for r in self.related_applications if r.get("application_id") != application_id]) + self.set_related_applications([r for r in self.related_applications + if r.get("application_id") != application_id]) def related_application_record(self, application_id): for record in self.related_applications: @@ -911,7 +912,6 @@ def propagate_in_doaj_status_to_articles(self): article.set_in_doaj(self.is_in_doaj()) article.save() - def prep(self, is_update=True): self._ensure_in_doaj() self.calculate_tick() @@ -1068,7 +1068,7 @@ def query(self): ] } }, - "size" : self.max + "size": self.max } if self.in_doaj is not None: q["query"]["bool"]["must"].append({"term": {"admin.in_doaj": self.in_doaj}}) @@ -1081,9 +1081,9 @@ def __init__(self, owner, in_doaj=None): self._in_doaj = in_doaj def query(self): - musts = [{"term": { "admin.owner.exact": self._owner}}] + musts = [{"term": {"admin.owner.exact": self._owner}}] if self._in_doaj is not None: - musts.append({"term": { "admin.in_doaj": self._in_doaj}}) + musts.append({"term": {"admin.in_doaj": self._in_doaj}}) return { "track_total_hits": True, "query": { @@ -1097,7 +1097,7 @@ def query(self): "terms": { "field": "index.issn.exact", "size": 10000, - "order": { "_key": "asc" } + "order": {"_key": "asc"} } } } @@ -1220,10 +1220,10 @@ def __init__(self, max): def query(self): return { "track_total_hits": True, - "query" : {"match_all" : {}}, - "size" : self.max, - "sort" : [ - {"created_date" : {"order" : "desc"}} + "query": {"match_all": {}}, + "size": self.max, + "sort": [ + {"created_date": {"order": "desc"}} ] } diff --git a/portality/tasks/anon_export.py b/portality/tasks/anon_export.py index 5fb7ec72be..bb671ceea7 100644 --- a/portality/tasks/anon_export.py +++ b/portality/tasks/anon_export.py @@ -138,7 +138,8 @@ def run_anon_export(tmpStore, mainStore, container, clean=False, limit=None, bat out_rollover_fn = functools.partial(_copy_on_complete, logger_fn=logger_fn, tmpStore=tmpStore, mainStore=mainStore, container=container) _ = model.dump(q=iter_q, limit=limit, transform=transform, out_template=output_file, out_batch_sizes=batch_size, - out_rollover_callback=out_rollover_fn, es_bulk_fields=["_id"], scroll_keepalive=app.config.get('TASKS_ANON_EXPORT_SCROLL_TIMEOUT', '5m')) + out_rollover_callback=out_rollover_fn, es_bulk_fields=["_id"], + scroll_keepalive=app.config.get('TASKS_ANON_EXPORT_SCROLL_TIMEOUT', '5m')) logger_fn((dates.now_str() + " done\n")) From 4c97808893d6db8b0bbfa4e57254ee1167000f4d Mon Sep 17 00:00:00 2001 From: philip Date: Thu, 11 Jul 2024 11:24:13 +0100 Subject: [PATCH 55/65] apply renew_editor_group_name --- portality/view/admin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/portality/view/admin.py b/portality/view/admin.py index 38acebc523..360038e6c3 100644 --- a/portality/view/admin.py +++ b/portality/view/admin.py @@ -605,6 +605,7 @@ def editor_group(group_id=None): ae.add_role("associate_editor") ae.save() + is_name_changed = eg.name != form.name.data eg.set_name(form.name.data) eg.set_maned(form.maned.data) eg.set_editor(form.editor.data) @@ -612,6 +613,10 @@ def editor_group(group_id=None): eg.set_associates(associates) eg.save() + if is_name_changed: + models.Journal.renew_editor_group_name(eg.id, eg.name) + models.Application.renew_editor_group_name(eg.id, eg.name) + flash( "Group was updated - changes may not be reflected below immediately. Reload the page to see the update.", "success") From 8f1c3872c12c9cdea8e662a237f3a77b84c6eca7 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 09:36:05 +0100 Subject: [PATCH 56/65] use index.editor_group_name instead of callback --- portality/static/js/doaj.fieldrender.edges.js | 34 +------------------ .../js/edges/admin.applications.edge.js | 8 +---- .../static/js/edges/admin.journals.edge.js | 12 ++----- .../js/edges/admin.update_requests.edge.js | 8 +---- .../js/edges/editor.groupapplications.edge.js | 12 ++----- .../js/edges/editor.groupjournals.edge.js | 12 ++----- 6 files changed, 9 insertions(+), 77 deletions(-) diff --git a/portality/static/js/doaj.fieldrender.edges.js b/portality/static/js/doaj.fieldrender.edges.js index 3dc5673902..a056142d3b 100644 --- a/portality/static/js/doaj.fieldrender.edges.js +++ b/portality/static/js/doaj.fieldrender.edges.js @@ -125,17 +125,15 @@ $.extend(true, doaj, { return edges.newRefiningANDTermSelector({ id: "editor_group", category: "facet", - field: "admin.editor_group.exact", + field: "index.editor_group_name.exact", display: "Editor group", deactivateThreshold: 1, - valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: doaj.valueMaps.countFormat, hideInactive: true, - noDisplayEscape: true, }) }) }, @@ -3900,36 +3898,6 @@ $.extend(true, doaj, { return false; } }, - - editorGroupNameCallback: function(val, resultobj, renderer) { - if (val) { - return `${val}` - } else { - return val - } - }, - editorGroupNameTrigger: function(queryResult) { - /** - * used in edges:post-render to trigger the editor group name callback - */ - - let editorGroupIds = queryResult.data.hits.hits.map(i => i._source.admin.editor_group) - editorGroupIds.push(...queryResult.data.aggregations.editor_group.buckets.map(i => i.key)) - editorGroupIds = editorGroupIds.filter(i => i !== undefined) - $.ajax({ - url: "/autocomplete-text/editor_group/name/id", - type: "POST", - contentType: "application/json", - dataType: 'json', - data: JSON.stringify({'ids': editorGroupIds}) , - success: function (data) { - $('.editorGroupNameCallback').each(function() { - const id = $(this).data('id') - $(this).text(data[id] || id) - }) - } - }); - } }, bulk : { diff --git a/portality/static/js/edges/admin.applications.edge.js b/portality/static/js/edges/admin.applications.edge.js index c4df467407..a22cc45928 100644 --- a/portality/static/js/edges/admin.applications.edge.js +++ b/portality/static/js/edges/admin.applications.edge.js @@ -211,7 +211,7 @@ $.extend(true, doaj, { 'index.application_type.exact' : 'Application', 'index.has_editor_group.exact' : 'Editor group', 'index.has_editor.exact' : 'Associate Editor', - 'admin.editor_group.exact' : 'Editor group', + 'index.editor_group_name.exact' : 'Editor group', 'admin.editor.exact' : 'Editor', 'index.classification.exact' : 'Classification', 'index.language.exact' : 'Language', @@ -229,9 +229,6 @@ $.extend(true, doaj, { "new application": "Open" } }, - valueFunctions : { - "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, - }, renderer : doaj.renderers.newSelectedFiltersRenderer({ omit : [ "bibjson.apc.has_apc", @@ -254,9 +251,6 @@ $.extend(true, doaj, { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); }, - "edges:post-render" : function () { - doaj.fieldRender.editorGroupNameTrigger(e.result); - } } }); doaj.adminApplicationsSearch.activeEdges[selector] = e; diff --git a/portality/static/js/edges/admin.journals.edge.js b/portality/static/js/edges/admin.journals.edge.js index f09008df54..e7323ac80f 100644 --- a/portality/static/js/edges/admin.journals.edge.js +++ b/portality/static/js/edges/admin.journals.edge.js @@ -132,17 +132,15 @@ $.extend(true, doaj, { edges.newRefiningANDTermSelector({ id: "editor_group", category: "facet", - field: "admin.editor_group.exact", + field: "index.editor_group_name.exact", display: "Editor group", deactivateThreshold: 1, - valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, hideInactive: true, - noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -476,7 +474,7 @@ $.extend(true, doaj, { "admin.owner.exact" : "Owner", "index.has_editor_group.exact" : "Editor group?", "index.has_editor.exact" : "Associate Editor?", - "admin.editor_group.exact" : "Editor group", + "index.editor_group_name.exact" : "Editor group", "admin.editor.exact" : "Associate Editor", "index.license.exact" : "License", "bibjson.publisher.name.exact" : "Publisher", @@ -494,9 +492,6 @@ $.extend(true, doaj, { false : "No" } }, - valueFunctions : { - "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, - }, rangeFunctions : { "bibjson.discontinued_date" : doaj.valueMaps.displayYearPeriod } @@ -516,9 +511,6 @@ $.extend(true, doaj, { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); }, - "edges:post-render" : function () { - doaj.fieldRender.editorGroupNameTrigger(e.result); - } } }); doaj.adminJournalsSearch.activeEdges[selector] = e; diff --git a/portality/static/js/edges/admin.update_requests.edge.js b/portality/static/js/edges/admin.update_requests.edge.js index 6701e388dc..932e9fcadf 100644 --- a/portality/static/js/edges/admin.update_requests.edge.js +++ b/portality/static/js/edges/admin.update_requests.edge.js @@ -212,7 +212,7 @@ $.extend(true, doaj, { 'index.application_type.exact' : 'Update Request', 'index.has_editor_group.exact' : 'Editor Group?', 'index.has_editor.exact' : 'Associate Editor?', - 'admin.editor_group.exact' : 'Editor Group', + 'index.editor_group_name.exact' : 'Editor Group', 'admin.editor.exact' : 'Editor', 'index.classification.exact' : 'Classification', 'index.language.exact' : 'Language', @@ -230,9 +230,6 @@ $.extend(true, doaj, { "new application": "Open" } }, - valueFunctions : { - "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, - }, }) ]; @@ -249,9 +246,6 @@ $.extend(true, doaj, { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); }, - "edges:post-render" : function () { - doaj.fieldRender.editorGroupNameTrigger(e.result); - } } }); doaj.adminApplicationsSearch.activeEdges[selector] = e; diff --git a/portality/static/js/edges/editor.groupapplications.edge.js b/portality/static/js/edges/editor.groupapplications.edge.js index d5e5225f3f..dbb6a2e1e3 100644 --- a/portality/static/js/edges/editor.groupapplications.edge.js +++ b/portality/static/js/edges/editor.groupapplications.edge.js @@ -75,17 +75,15 @@ $.extend(true, doaj, { edges.newRefiningANDTermSelector({ id: "editor_group", category: "facet", - field: "admin.editor_group.exact", + field: "index.editor_group_name.exact", display: "Editor Group", deactivateThreshold: 1, - valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, hideInactive: true, - noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -375,7 +373,7 @@ $.extend(true, doaj, { 'admin.application_status.exact': 'Application Status', 'index.application_type.exact' : 'Record type', 'index.has_editor.exact' : 'Has Associate Editor?', - 'admin.editor_group.exact' : 'Editor Group', + 'index.editor_group_name.exact' : 'Editor Group', 'admin.editor.exact' : 'Editor', 'index.classification.exact' : 'Classification', 'index.language.exact' : 'Journal language', @@ -385,9 +383,6 @@ $.extend(true, doaj, { 'index.license.exact' : 'Journal license', "index.has_apc.exact" : "Publication charges?" }, - valueFunctions : { - "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, - }, }) ]; @@ -404,9 +399,6 @@ $.extend(true, doaj, { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); }, - "edges:post-render" : function () { - doaj.fieldRender.editorGroupNameTrigger(e.result); - } } }); doaj.editorGroupApplicationsSearch.activeEdges[selector] = e; diff --git a/portality/static/js/edges/editor.groupjournals.edge.js b/portality/static/js/edges/editor.groupjournals.edge.js index 8bf195943a..85d1a98494 100644 --- a/portality/static/js/edges/editor.groupjournals.edge.js +++ b/portality/static/js/edges/editor.groupjournals.edge.js @@ -52,17 +52,15 @@ $.extend(true, doaj, { edges.newRefiningANDTermSelector({ id: "editor_group", category: "facet", - field: "admin.editor_group.exact", + field: "index.editor_group_name.exact", display: "Editor Group", deactivateThreshold: 1, - valueFunction: doaj.fieldRender.editorGroupNameCallback, renderer: edges.bs3.newRefiningANDTermSelectorRenderer({ controls: true, open: false, togglable: true, countFormat: countFormat, hideInactive: true, - noDisplayEscape: true, }) }), edges.newRefiningANDTermSelector({ @@ -367,7 +365,7 @@ $.extend(true, doaj, { "admin.in_doaj" : "In DOAJ?", "admin.owner.exact" : "Owner", "index.has_editor.exact" : "Associate Editor?", - "admin.editor_group.exact" : "Editor group", + "index.editor_group_name.exact" : "Editor group", "admin.editor.exact" : "Associate Editor", "index.license.exact" : "License", "bibjson.publisher.name.exact" : "Publisher", @@ -384,9 +382,6 @@ $.extend(true, doaj, { false : "False" } }, - valueFunctions : { - "admin.editor_group.exact" : doaj.fieldRender.editorGroupNameCallback, - }, }) ]; @@ -403,9 +398,6 @@ $.extend(true, doaj, { "edges:query-fail" : function() { alert("There was an unexpected error. Please reload the page and try again. If the issue persists please contact an administrator."); }, - "edges:post-render" : function () { - doaj.fieldRender.editorGroupNameTrigger(e.result); - } } }); doaj.editorGroupJournalsSearch.activeEdges[selector] = e; From 4714392b084dfb5ff4617fb1bc868781e71c5829 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 11:08:32 +0100 Subject: [PATCH 57/65] update documents --- portality/view/doaj.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 587ab3afb5..699b8384aa 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -264,12 +264,18 @@ def autocomplete(doc_type, field_name): def autocomplete_pair(doc_type, field_name, id_field): """ different from autocomplete, in response json, the values of `id` and `text` are used different field - `id` used value of `id_field` - `text` used value of `field_name` - parameter `q` will be used to search the prefix of `field_name` value - Json string with follow format: - {suggestions: [{id: id_field, text: field_name}]} + Parameters + --------------- + `q` -- will be used to search record with value that start with `q` in `field_name` field + + Returns + --------------- + Json string with follow format: + {suggestions: [{id: id_field, text: field_name}]} + + `id` used value of `id_field` + `text` used value of `field_name` """ prefix = request.args.get('q', '') From d4c2215d4e061705db83a36d6232caf682b547ff Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 12:09:10 +0100 Subject: [PATCH 58/65] typing cleanup --- doajtest/fixtures/v2/journals.py | 12 ++++++------ doajtest/unit/test_models.py | 2 +- portality/dao.py | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/doajtest/fixtures/v2/journals.py b/doajtest/fixtures/v2/journals.py index 3b3d1123aa..58b57885c7 100644 --- a/doajtest/fixtures/v2/journals.py +++ b/doajtest/fixtures/v2/journals.py @@ -11,7 +11,7 @@ class JournalFixtureFactory(object): @staticmethod - def make_journal_source(in_doaj=False): + def make_journal_source(in_doaj=False) -> dict: template = deepcopy(JOURNAL_SOURCE) template['admin']['in_doaj'] = in_doaj return template @@ -37,23 +37,23 @@ def make_many_journal_sources(count=2, in_doaj=False) -> Iterable[dict]: return journal_sources @staticmethod - def make_journal_form(): + def make_journal_form() -> dict: return deepcopy(JOURNAL_FORM) @staticmethod - def make_journal_form_info(): + def make_journal_form_info() -> dict: return deepcopy(JOURNAL_FORM_EXPANDED) @staticmethod - def make_bulk_edit_data(): + def make_bulk_edit_data() -> dict: return deepcopy(JOURNAL_BULK_EDIT) @staticmethod - def csv_headers(): + def csv_headers() -> dict: return deepcopy(CSV_HEADERS) @staticmethod - def question_answers(): + def question_answers() -> dict: return deepcopy(JOURNAL_QUESTION_ANSWERS) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index a341fb30e3..df74914d8d 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1228,7 +1228,7 @@ def test_24_save_valid_seamless_or_dataobj(self): s.data["junk"] = "in here" with self.assertRaises(seamless.SeamlessException): s.save() - assert s.id is not None # ID is necessary for duplication check + assert s.id is not None # ID is necessary for duplication check p = models.Provenance() p.type = "suggestion" diff --git a/portality/dao.py b/portality/dao.py index 489b9648de..e0c93fae7b 100644 --- a/portality/dao.py +++ b/portality/dao.py @@ -28,7 +28,6 @@ ES_MAPPING_MISSING_REGEX = re.compile(r'.*No mapping found for \[[a-zA-Z0-9-_\.]+?\] in order to sort on.*', re.DOTALL) CONTENT_TYPE_JSON = {'Content-Type': 'application/json'} - class IdText(TypedDict): id: str text: str From eb98e395685503cbe747dff9af4beb73e86c0c99 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 12:09:38 +0100 Subject: [PATCH 59/65] rename new_name --- portality/models/v2/journal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index fe96159d23..b69014883c 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -132,7 +132,7 @@ def recent(cls, max=10): @classmethod def renew_editor_group_name(cls, editor_group_id: str, - new_editor_group_name: str, + new_name: str, batch_size: int = 10000, ): """ @@ -152,7 +152,7 @@ def _to_action(_id): "source": "ctx._source.index.editor_group_name = params.new_value", "lang": "painless", "params": { - "new_value": new_editor_group_name + "new_value": new_name } } } From 7f1d46e04aef5b21e4ea1a3d228f76d6f545f8c6 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 12:09:50 +0100 Subject: [PATCH 60/65] add editor_group_name --- portality/models/v2/journal.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index b69014883c..aec12316e9 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -267,6 +267,11 @@ def editor_group(self): def set_editor_group(self, eg): self.__seamless__.set_with_struct("admin.editor_group", eg) + @property + def editor_group_name(self): + # index.editor_group_name set by _generate_index + return self.__seamless__.get_single("index.editor_group_name") + def remove_editor_group(self): self.__seamless__.delete("admin.editor_group") self.__seamless__.delete("index.editor_group_name") From 160bbf44c1afb5744f148217817b5cc7bf88ae36 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 12:10:02 +0100 Subject: [PATCH 61/65] add test_renew_editor_group_name --- doajtest/unit/test_models.py | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index df74914d8d..51c0ba3314 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1770,3 +1770,44 @@ def test_11_find_by_issn(self): journals = models.Journal.find_by_issn_exact(["1111-1111", "3333-3333"], True) assert len(journals) == 0 + + +class TestJournal(DoajTestCase): + def test_renew_editor_group_name(self): + # preparing data + eg1 = models.EditorGroup(**{ + 'id': 'eg1', + 'name': 'Editor Group 111111', + }) + eg2 = models.EditorGroup(**{ + 'id': 'eg2', + 'name': 'Editor Group 222222', + }) + models.DomainObject.save_all_block_last([eg1, eg2]) + + journals = [models.Journal(**j) for j in JournalFixtureFactory.make_many_journal_sources(count=3, in_doaj=True)] + journals[0].set_editor_group(eg1.id) + journals[1].set_editor_group(eg1.id) + journals[2].set_editor_group(eg2.id) + + models.DomainObject.save_all_block_last(journals) + + def refresh_journals(journals): + return [models.Journal.pull(j.id) for j in journals] + + journals = refresh_journals(journals) + assert journals[0].editor_group_name == eg1.name + assert journals[1].editor_group_name == eg1.name + assert journals[2].editor_group_name == eg2.name + + # start testing + new_name = 'XXCCCCXXX' + models.Journal.renew_editor_group_name( + editor_group_id=eg1.id, + new_name=new_name, + ) + + journals = refresh_journals(journals) + assert journals[0].editor_group_name == new_name + assert journals[1].editor_group_name == new_name + assert journals[2].editor_group_name == eg2.name From 5f488ca0e75174c629f5a08c87e2ab772a60f810 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 14:09:58 +0100 Subject: [PATCH 62/65] remove duplicate editor_group_name --- doajtest/unit/test_models.py | 12 ++++++------ portality/models/v2/journal.py | 4 ---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/doajtest/unit/test_models.py b/doajtest/unit/test_models.py index 51c0ba3314..f014aa88aa 100644 --- a/doajtest/unit/test_models.py +++ b/doajtest/unit/test_models.py @@ -1796,9 +1796,9 @@ def refresh_journals(journals): return [models.Journal.pull(j.id) for j in journals] journals = refresh_journals(journals) - assert journals[0].editor_group_name == eg1.name - assert journals[1].editor_group_name == eg1.name - assert journals[2].editor_group_name == eg2.name + assert journals[0].editor_group_name() == eg1.name + assert journals[1].editor_group_name() == eg1.name + assert journals[2].editor_group_name() == eg2.name # start testing new_name = 'XXCCCCXXX' @@ -1808,6 +1808,6 @@ def refresh_journals(journals): ) journals = refresh_journals(journals) - assert journals[0].editor_group_name == new_name - assert journals[1].editor_group_name == new_name - assert journals[2].editor_group_name == eg2.name + assert journals[0].editor_group_name() == new_name + assert journals[1].editor_group_name() == new_name + assert journals[2].editor_group_name() == eg2.name diff --git a/portality/models/v2/journal.py b/portality/models/v2/journal.py index aec12316e9..447c8634a8 100644 --- a/portality/models/v2/journal.py +++ b/portality/models/v2/journal.py @@ -267,10 +267,6 @@ def editor_group(self): def set_editor_group(self, eg): self.__seamless__.set_with_struct("admin.editor_group", eg) - @property - def editor_group_name(self): - # index.editor_group_name set by _generate_index - return self.__seamless__.get_single("index.editor_group_name") def remove_editor_group(self): self.__seamless__.delete("admin.editor_group") From 2570fac1290a9ef2ad228519cb1bf619920d8f4a Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 14:38:32 +0100 Subject: [PATCH 63/65] change format for `test_openapi_schema` --- portality/view/doaj.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/portality/view/doaj.py b/portality/view/doaj.py index 699b8384aa..3b090fecfa 100644 --- a/portality/view/doaj.py +++ b/portality/view/doaj.py @@ -262,15 +262,13 @@ def autocomplete(doc_type, field_name): @blueprint.route('/autocomplete///', methods=["GET", "POST"]) def autocomplete_pair(doc_type, field_name, id_field): - """ + r""" different from autocomplete, in response json, the values of `id` and `text` are used different field - Parameters - --------------- + Parameters: `q` -- will be used to search record with value that start with `q` in `field_name` field - Returns - --------------- + Returns: Json string with follow format: {suggestions: [{id: id_field, text: field_name}]} From ba6c358636d71987436f8e3c9a1eed7571c4bc01 Mon Sep 17 00:00:00 2001 From: philip Date: Fri, 12 Jul 2024 14:40:59 +0100 Subject: [PATCH 64/65] update test cases for allow update edit group name now --- doajtest/unit/test_admin_editor_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doajtest/unit/test_admin_editor_groups.py b/doajtest/unit/test_admin_editor_groups.py index b5fb728a27..420c04af41 100644 --- a/doajtest/unit/test_admin_editor_groups.py +++ b/doajtest/unit/test_admin_editor_groups.py @@ -38,5 +38,5 @@ def test_editor_group_creation_and_update(self): # give some time for the new record to be indexed time.sleep(1) updated_group = EditorGroup.pull(editor_group_id) - self.assertEquals(updated_group.name, "Test Group") - self.assertNotEquals(updated_group.name, "New Test Group") + self.assertEquals(updated_group.name, "New Test Group") + self.assertNotEquals(updated_group.name, "Test Group") From 69c06b514290cfa55d9f3eb5ec3f85830bd8183e Mon Sep 17 00:00:00 2001 From: philip Date: Tue, 23 Jul 2024 13:13:59 +0100 Subject: [PATCH 65/65] update migrate for update mapping --- portality/core.py | 6 ++- .../3850_link_editor_groups_by_name/README.md | 4 +- portality/scripts/update_mapping_for_exact.py | 44 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 portality/scripts/update_mapping_for_exact.py diff --git a/portality/core.py b/portality/core.py index 8280edc620..37b8bc9cca 100644 --- a/portality/core.py +++ b/portality/core.py @@ -188,7 +188,11 @@ def create_es_connection(app): # return type -def put_mappings(conn, mappings): +def get_doaj_index_name(name: str) -> str: + return app.config['ELASTIC_SEARCH_DB_PREFIX'] + name + + +def put_mappings(conn: elasticsearch.Elasticsearch, mappings): # get the ES version that we're working with #es_version = app.config.get("ELASTIC_SEARCH_VERSION", "1.7.5") diff --git a/portality/migrate/3850_link_editor_groups_by_name/README.md b/portality/migrate/3850_link_editor_groups_by_name/README.md index 2b33338c05..166195c57f 100644 --- a/portality/migrate/3850_link_editor_groups_by_name/README.md +++ b/portality/migrate/3850_link_editor_groups_by_name/README.md @@ -2,4 +2,6 @@ https://github.com/DOAJ/doajPM/issues/3245 The script looks for applications which are rejected and if they have a `related_journal` then it tags them as update requests. - python portality/upgrade.py -u portality/migrate/3850_link_editor_groups_by_name/migrate.json \ No newline at end of file + python portality/upgrade.py -u portality/migrate/3850_link_editor_groups_by_name/migrate.json + python portality/scripts/update_mapping_for_exact.py journal index.editor_group_name + python portality/scripts/update_mapping_for_exact.py application index.editor_group_name diff --git a/portality/scripts/update_mapping_for_exact.py b/portality/scripts/update_mapping_for_exact.py new file mode 100644 index 0000000000..34f8ba9eb3 --- /dev/null +++ b/portality/scripts/update_mapping_for_exact.py @@ -0,0 +1,44 @@ +import argparse + +from portality import core +from portality.core import es_connection + + +def main(): + """ + Update mapping for exact field + """ + + parser = argparse.ArgumentParser(description='Update mapping for exact field') + parser.add_argument('index_type', help='Index name') + parser.add_argument('field_name', help='Field name (e.g. index.editor_group_name)') + + args = parser.parse_args() + + index = core.get_doaj_index_name(args.index_type) + field_name = args.field_name + exact_mapping = { + "exact": { + "type": "keyword", + "store": True + } + } + + print(f'Updating mapping for field: [{field_name}] in index: [{index}]') + + conn = es_connection + org_mapping = conn.indices.get_field_mapping(field_name, index=index) + org_field_mapping = org_mapping[index]['mappings'][field_name]['mapping'] + org_field_mapping = next(iter(org_field_mapping.values())) + org_field_mapping['fields'].update(**exact_mapping) + new_mapping = { + 'properties': { + field_name: org_field_mapping + } + } + print(new_mapping) + conn.indices.put_mapping(body=new_mapping, index=index) + + +if __name__ == '__main__': + main()