Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/2881 url shortener #2358

Open
wants to merge 45 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
a586a21
move wait_unit
philipkcl Mar 4, 2024
626c3f8
move wait_unit
philipkcl Mar 4, 2024
475d1a3
add urlshort
philipkcl Mar 4, 2024
6312f51
apply urlshort in frontend
philipkcl Mar 4, 2024
21ffdb5
cleanup old url shortener
philipkcl Mar 4, 2024
3f411b5
add doc `How to setup for dev with Plausible`
philipkcl Mar 5, 2024
0581a02
add setup_dev_log
philipkcl Mar 5, 2024
58b4fd2
commit edges
philipkcl Mar 5, 2024
eaba321
add plausible for urlshort
philipkcl Mar 5, 2024
6909692
add more test cases
philipkcl Mar 5, 2024
b818da0
add note
philipkcl Mar 5, 2024
90b00d4
update edges
philipkcl Mar 5, 2024
50f995f
fix docs
philipkcl Mar 5, 2024
8068d94
doc format
philipkcl Mar 5, 2024
0a5d5cc
merge develop 240318
philipkcl Mar 18, 2024
b60d6de
fix names wait_until
philipkcl Mar 18, 2024
3e4e3f3
add url limit
philipkcl Mar 18, 2024
6d271ff
limit by domain instead of paths
philipkcl Mar 18, 2024
194c8e1
fix wait_until
philipkcl Mar 18, 2024
ff699dd
fix some testcases
philipkcl Mar 18, 2024
7b173f7
fix wait_until
philipkcl Mar 18, 2024
1447101
avoid selenium test fail on circleci
philipkcl Mar 19, 2024
e3cf981
move urlshort to services
philipkcl Mar 19, 2024
085697a
change generateShortUrl interface
philipkcl Mar 20, 2024
e9be06d
change branch 2881_url_shortener
philipkcl Mar 20, 2024
37163cf
add DEBUG_DEV_LOG
philipkcl Apr 19, 2024
6b4d005
remove lock and add URLSHORT_ALIAS_LENGTH
philipkcl May 2, 2024
91fbf53
rename n_retry
philipkcl May 2, 2024
ed3eb06
reverse for old version edges urlshort interface
philipkcl May 3, 2024
6f42055
merge develop 240503
philipkcl May 3, 2024
47817a0
Removed seal in some files
RK206 Feb 12, 2025
864c53c
Merge branch 'develop' of https://github.com/DOAJ/doaj into feature/4…
RK206 Feb 20, 2025
871ba00
Removed seal
RK206 Feb 21, 2025
b5bcaa3
Migration script to remove seal from indexed records
RK206 Feb 24, 2025
e9c1b8d
Removed requested metadata
RK206 Mar 5, 2025
c38a198
Merge branch 'develop' of https://github.com/DOAJ/doaj into feature/4…
RK206 Mar 5, 2025
925a90a
merge changes from develop
RK206 Mar 5, 2025
1ca07dc
merge up from develop
richard-jones Mar 20, 2025
a6991ad
reinstate the oa statement, plagiarism url, and embedded licence link…
richard-jones Mar 21, 2025
6e31ccb
update a few remaining test references
richard-jones Mar 21, 2025
4d77264
merge up from develop
richard-jones Mar 21, 2025
af772be
refactor short url generator
richard-jones Mar 31, 2025
dc84f21
Merge branch 'develop' into feature/2881_url_shortener
richard-jones Mar 31, 2025
60beaaf
Merge branch 'feature/4007_remove_the_seal' into feature/2881_url_sho…
richard-jones Mar 31, 2025
d954a58
fix some test issues
richard-jones Mar 31, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions cms/data/nav.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,6 @@ entries:
route: apply.public_application # ~~->Apply:WebRoute~~
- label: Guide to applying
route: doaj.guide # ~~->ApplicationGuide:WebRoute~~
- label: The DOAJ Seal
route: doaj.seal # ~~->Seal:WebRoute~~
- label: Transparency & best practice
route: doaj.transparency # ~~->Transparency:WebRoute~~
- label: Publisher information
Expand Down
56 changes: 0 additions & 56 deletions cms/pages/apply/seal.md

This file was deleted.

4 changes: 1 addition & 3 deletions doajtest/example_files/incoming_application.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@
},
"article": {
"license_display": ["Embed"],
"license_display_example_url": "http://licence.embedded",
"orcid": "True",
"i4oc_open_citations": "False"
"license_display_example_url": "http://licence.embedded"
},
"boai": "True",
"copyright": {
Expand Down
2 changes: 0 additions & 2 deletions doajtest/fixtures/article.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@ def make_article_apido_struct():
"id": "abcdefghijk_article",
"admin": {
"in_doaj": True,
"seal": False,
"publisher_record_id": "some_identifier",
"upload_id": "zyxwvutsrqpo_upload_id"
},
Expand Down Expand Up @@ -240,7 +239,6 @@ def make_article_apido_struct():
"admin": {
"fields": {
"in_doaj": {"coerce": "bool", "get__default": False},
"seal": {"coerce": "bool", "get__default": False},
"publisher_record_id": {"coerce": "unicode"},
"upload_id": {"coerce": "unicode"}
}
Expand Down
4 changes: 1 addition & 3 deletions doajtest/fixtures/v2/applications.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import rstr

from portality import constants, regex
from doajtest.fixtures.v2.common import JOURNAL_LIKE_BIBJSON, EDITORIAL_FORM_EXPANDED, SUBJECT_FORM_EXPANDED, NOTES_FORM_EXPANDED, OWNER_FORM_EXPANDED, SEAL_FORM_EXPANDED
from doajtest.fixtures.v2.common import JOURNAL_LIKE_BIBJSON, EDITORIAL_FORM_EXPANDED, SUBJECT_FORM_EXPANDED, NOTES_FORM_EXPANDED, OWNER_FORM_EXPANDED
from doajtest.fixtures.v2.journals import JOURNAL_FORM_EXPANDED, JOURNAL_FORM
from portality.lib import dates
from portality.lib.dates import FMT_DATE_YM, FMT_YEAR
Expand Down Expand Up @@ -123,7 +123,6 @@ def make_application_spread(cls, desired_output, period):
],
"owner" : "publisher",
"related_journal" : "987654321123456789",
"seal": False,
"date_applied" : "2003-01-01T00:00:00Z"
},
"bibjson" : JOURNAL_LIKE_BIBJSON
Expand Down Expand Up @@ -155,7 +154,6 @@ def make_application_spread(cls, desired_output, period):
APPLICATION_FORM_EXPANDED.update(deepcopy(SUBJECT_FORM_EXPANDED))
APPLICATION_FORM_EXPANDED.update(deepcopy(OWNER_FORM_EXPANDED))
APPLICATION_FORM_EXPANDED.update(deepcopy(EDITORIAL_FORM_EXPANDED))
APPLICATION_FORM_EXPANDED.update(deepcopy(SEAL_FORM_EXPANDED))
APPLICATION_FORM_EXPANDED.update(deepcopy(WORKFLOW_FORM_EXPANDED))

from portality.crosswalks.application_form import ApplicationFormXWalk
Expand Down
10 changes: 1 addition & 9 deletions doajtest/fixtures/v2/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,6 @@
"editor": "associate"
}

SEAL_FORM_EXPANDED = {
"doaj_seal": [],
}

JOURNAL_LIKE_BIBJSON = {
"alternative_title": "Alternative Title",
"apc": {
Expand All @@ -39,9 +35,7 @@
},
"article": {
"license_display": ["Embed"],
"license_display_example_url": "http://licence.embedded",
"orcid": True,
"i4oc_open_citations": False
"license_display_example_url": "http://licence.embedded"
},
"boai": True,
"copyright": {
Expand Down Expand Up @@ -175,8 +169,6 @@
"title": "The Title",
"has_other_charges" : "y",
"has_waiver" : "y",
"orcid_ids" : "y",
"open_citations" : "n",
"deposit_policy_url" : "http://deposit.policy",
"continues": ["1111-1111"],
"continued_by": ["2222-2222"],
Expand Down
17 changes: 2 additions & 15 deletions doajtest/fixtures/v2/journals.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import rstr

from doajtest.fixtures.v2.common import EDITORIAL_FORM_EXPANDED, SUBJECT_FORM_EXPANDED, NOTES_FORM_EXPANDED, \
OWNER_FORM_EXPANDED, SEAL_FORM_EXPANDED, JOURNAL_LIKE_BIBJSON, JOURNAL_LIKE_BIBJSON_FORM_EXPANDED
OWNER_FORM_EXPANDED, JOURNAL_LIKE_BIBJSON, JOURNAL_LIKE_BIBJSON_FORM_EXPANDED
from portality.regex import ISSN_COMPILED


Expand Down Expand Up @@ -78,7 +78,6 @@ def question_answers():
{"application_id": "asdfghjkl", "date_accepted": "2018-01-01T00:00:00Z"},
{"application_id": "zxcvbnm"}
],
"seal": False,
"ticked": True
},
"bibjson": JOURNAL_LIKE_BIBJSON
Expand All @@ -87,7 +86,6 @@ def question_answers():
JOURNAL_FORM_EXPANDED = {}
JOURNAL_FORM_EXPANDED.update(JOURNAL_LIKE_BIBJSON_FORM_EXPANDED)
JOURNAL_FORM_EXPANDED.update(EDITORIAL_FORM_EXPANDED)
JOURNAL_FORM_EXPANDED.update(SEAL_FORM_EXPANDED)
JOURNAL_FORM_EXPANDED.update(SUBJECT_FORM_EXPANDED)
JOURNAL_FORM_EXPANDED.update(NOTES_FORM_EXPANDED)
JOURNAL_FORM_EXPANDED.update(OWNER_FORM_EXPANDED)
Expand All @@ -98,7 +96,6 @@ def question_answers():

JOURNAL_BULK_EDIT = {
"publisher": "Test Publisher",
"doaj_seal": True,
"country": "DZ",
"platform": "HighWire",
"contact_email": "[email protected]",
Expand All @@ -124,13 +121,11 @@ def question_answers():
"License attributes",
"URL for license terms",
"Machine-readable CC licensing information embedded or displayed in articles",
"URL to an example page with embedded licensing information",
"Author holds copyright without restrictions",
"Copyright information URL",
"Review process",
"Review process information URL",
"Journal plagiarism screening policy",
"Plagiarism information URL",
"URL for journal's aims & scope",
"URL for the Editorial Board page",
"URL for journal's instructions for authors",
Expand All @@ -148,15 +143,12 @@ def question_answers():
"Deposit policy directory",
"URL for deposit policy",
"Persistent article identifiers",
"Article metadata includes ORCIDs",
"Journal complies with I4OC standards for open citations",
"Does the journal comply to DOAJ's definition of open access?",
"URL for journal's Open Access statement",
"Continues",
"Continued By",
"LCC Codes",
"Subscribe to Open",
'Subjects', # (added outside journal2questions)
'DOAJ Seal', # (added outside journal2questions)
'Added on Date', # (added outside journal2questions)
'Last updated Date', # (added outside journal2questions)
# 'Tick: Accepted after March 2014', Removed 2020-12-11
Expand All @@ -181,13 +173,11 @@ def question_answers():
'Attribution, No Commercial Usage',
'http://licence.url',
'Yes',
'http://licence.embedded',
'Yes',
'http://copyright.com',
'Open peer review, some bloke checks it out',
'http://review.process',
'Yes',
'http://plagiarism.screening',
'http://aims.scope',
'http://editorial.board',
'http://author.instructions.com',
Expand All @@ -206,9 +196,6 @@ def question_answers():
"http://deposit.policy",
'DOI, ARK, PURL, PIDMachine',
'Yes',
'No',
'Yes',
'http://oa.statement',
"1111-1111",
"2222-2222",
"HB1-3840|H|SF600-1100",
Expand Down
3 changes: 2 additions & 1 deletion doajtest/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,8 @@ def create_app_patch(cls):
"UR_CONCURRENCY_TIMEOUT": 0,
'UPLOAD_ASYNC_DIR': paths.create_tmp_path(is_auto_mkdir=True).as_posix(),
'HUEY_IMMEDIATE': True,
'HUEY_ASYNC_DELAY': 0
'HUEY_ASYNC_DELAY': 0,
'URLSHORT_ALLOWED_SUPERDOMAINS': ['doaj.org', 'localhost', '127.0.0.1']
}

@classmethod
Expand Down
8 changes: 2 additions & 6 deletions doajtest/mocks/preservation.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ class PreservationMock:
],
"unpunctitle": "Variation in TMEM106B in chronic traumatic encephalopathy",
"asciiunpunctitle": "Variation in TMEM106B in chronic traumatic encephalopathy",
"has_seal": "Yes",
"doi": "10.1186/s40478-018-0619-9",
"fulltext": "//link.springer.com/article/10.1186/s40478-018-0619-9",
"schema_codes_tree": [
Expand All @@ -47,8 +46,7 @@ class PreservationMock:
},
"last_updated": "2020-11-24T21:55:20Z",
"admin": {
"in_doaj": "true",
"seal": "true"
"in_doaj": "true"
},
"created_date": "2018-11-04T12:37:46Z",
"id": "00003741594643f4996e2555a01e03c7",
Expand Down Expand Up @@ -260,7 +258,6 @@ class PreservationMock:
],
"unpunctitle": "Variation in TMEM106B in chronic traumatic encephalopathy",
"asciiunpunctitle": "Variation in TMEM106B in chronic traumatic encephalopathy",
"has_seal": "Yes",
"doi": "10.3389/fcosc.2022.1028295",
"fulltext": "https://frontiersin.org/articles/10.3389/fcosc.2022.1028295",
"schema_codes_tree": [
Expand All @@ -272,8 +269,7 @@ class PreservationMock:
},
"last_updated": "2020-11-24T21:55:20Z",
"admin": {
"in_doaj": "true",
"seal": "true"
"in_doaj": "true"
},
"created_date": "2018-11-04T12:37:46Z",
"id": "00005741594643f4996e2666a01e0310",
Expand Down
33 changes: 12 additions & 21 deletions doajtest/selenium_helpers.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import datetime
import logging
import multiprocessing
import time
from multiprocessing import Process, freeze_support
from typing import TYPE_CHECKING

Expand All @@ -15,6 +14,7 @@
from portality.util import patch_config
from portality import app, models, core
from portality.dao import ESMappingMissingError
from portality.lib.thread_utils import wait_until

if TYPE_CHECKING:
from selenium.webdriver.remote.webdriver import WebDriver
Expand Down Expand Up @@ -119,7 +119,7 @@ def setUp(self):
self.selenium.set_window_size(1400, 1000) # avoid something is not clickable

# wait for server to start
wait_unit(self._is_doaj_server_running, 10, 1.5, timeout_msg='doaj server not started')
wait_until(self._is_doaj_server_running, 10, 1.5, timeout_msg='doaj server not started')

fix_index_not_found_exception(self.app_test)
self.fix_es_mapping()
Expand All @@ -144,7 +144,8 @@ def _is_doaj_server_running(self):
self.selenium.find_element(By.CSS_SELECTOR, 'div.container')
log.info('doaj server is running')
return True
except selenium.common.exceptions.NoSuchElementException:
except (selenium.common.exceptions.NoSuchElementException,
selenium.common.exceptions.WebDriverException):
log.info('doaj server is not running')
return False

Expand All @@ -160,12 +161,12 @@ def tearDown(self):
print(f'{datetime.datetime.now().isoformat()} --- doaj process terminating...')
self.doaj_process.terminate()
self.doaj_process.join()
wait_unit(lambda: not self._is_doaj_server_running(), 10, 1,
timeout_msg='doaj server is still running')
wait_until(lambda: not self._is_doaj_server_running(), 10, 1,
timeout_msg='doaj server is still running')

self.selenium.quit()

wait_unit(self._is_selenium_quit, 10, 1, timeout_msg='selenium is still running')
wait_until(self._is_selenium_quit, 10, 1, timeout_msg='selenium is still running')
print('selenium terminated')

super().tearDown()
Expand Down Expand Up @@ -220,17 +221,7 @@ def login_by_acc(driver: 'WebDriver', acc: models.Account = None):
assert "/login" not in driver.current_url


def wait_unit(exit_cond_fn, timeout=10, check_interval=0.1,
timeout_msg="wait_unit but exit_cond timeout"):
start = time.time()
while (time.time() - start) < timeout:
if exit_cond_fn():
return
time.sleep(check_interval)
raise TimeoutError(timeout_msg)


def wait_unit_elements(driver: 'WebDriver', css_selector: str, timeout=10, check_interval=0.1):
def wait_until_elements(driver: 'WebDriver', css_selector: str, timeout=10, check_interval=0.1):
elements = []

def exit_cond_fn():
Expand All @@ -241,11 +232,11 @@ def exit_cond_fn():
except:
return False

wait_unit(exit_cond_fn, timeout, check_interval)
wait_until(exit_cond_fn, timeout=timeout, sleep_time=check_interval)
return elements


def wait_unit_click(driver: 'WebDriver', css_selector: str, timeout=10, check_interval=0.1):
def wait_until_click(driver: 'WebDriver', css_selector: str, timeout=10, check_interval=0.1):
def _click():
try:
ele = find_ele_by_css(driver, css_selector)
Expand All @@ -256,11 +247,11 @@ def _click():
except (StaleElementReferenceException, ElementClickInterceptedException):
return False

wait_unit(_click, timeout=10, check_interval=0.1)
wait_until(_click, timeout=timeout, sleep_time=check_interval)


def click_edges_item(driver: 'WebDriver', ele_name, item_name):
wait_unit_click(driver, f'#edges-bs3-refiningand-term-selector-toggle-{ele_name}')
wait_until_click(driver, f'#edges-bs3-refiningand-term-selector-toggle-{ele_name}')
for ele in find_eles_by_css(driver, f'.edges-bs3-refiningand-term-selector-result-{ele_name} a'):
if item_name in ele.text.strip():
ele.click()
Loading