diff --git a/.docker-compose.env b/.docker-compose.env index 9cb7a59e274..449c9747adf 100644 --- a/.docker-compose.env +++ b/.docker-compose.env @@ -7,6 +7,7 @@ INTERNAL_DOMAIN=http://192.168.168.167:5000/ API_DOMAIN=http://localhost:8000/ ELASTIC_URI=192.168.168.167:9200 ELASTIC6_URI=192.168.168.167:9201 +ELASTIC8_URI=http://192.168.168.167:9202 OSF_DB_HOST=192.168.168.167 DB_HOST=192.168.168.167 REDIS_HOST=redis://192.168.168.167:6379 diff --git a/addons/base/views.py b/addons/base/views.py index 5ff3d6e7093..ebcd662966b 100644 --- a/addons/base/views.py +++ b/addons/base/views.py @@ -14,7 +14,7 @@ import waffle from django.db import transaction from django.contrib.contenttypes.models import ContentType -from elasticsearch import exceptions as es_exceptions +from elasticsearch6 import exceptions as es_exceptions from rest_framework import status as http_status from api.caching.tasks import update_storage_usage_with_size diff --git a/api/base/elasticsearch_dsl_views.py b/api/base/elasticsearch_dsl_views.py index 6199fd82d0e..ecf2825d4e8 100644 --- a/api/base/elasticsearch_dsl_views.py +++ b/api/base/elasticsearch_dsl_views.py @@ -3,7 +3,7 @@ import datetime import typing -import elasticsearch_dsl as edsl +import elasticsearch6_dsl as edsl from rest_framework import generics, exceptions as drf_exceptions from rest_framework.settings import api_settings as drf_settings from api.base.settings.defaults import REPORT_FILENAME_FORMAT @@ -23,7 +23,7 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC): - '''abstract view class using `elasticsearch_dsl.Search` as a queryset-analogue + '''abstract view class using `elasticsearch6_dsl.Search` as a queryset-analogue builds a `Search` based on `self.get_default_search()` and the request's query parameters for filtering, sorting, and pagination -- fetches only @@ -36,7 +36,7 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, @abc.abstractmethod def get_default_search(self) -> edsl.Search | None: - '''the base `elasticsearch_dsl.Search` for this list, based on url path + '''the base `elasticsearch6_dsl.Search` for this list, based on url path (common jsonapi query parameters will be considered automatically) ''' @@ -95,7 +95,7 @@ def finalize_response(self, request, response, *args, **kwargs): # (filtering handled in-view to reuse logic from FilterMixin) filter_backends = () - # note: because elasticsearch_dsl.Search supports slicing and gives results when iterated on, + # note: because elasticsearch6_dsl.Search supports slicing and gives results when iterated on, # it works fine with default pagination # override rest_framework.generics.GenericAPIView diff --git a/api/base/settings/defaults.py b/api/base/settings/defaults.py index efddf2484b8..816586ffcfb 100644 --- a/api/base/settings/defaults.py +++ b/api/base/settings/defaults.py @@ -316,10 +316,18 @@ HASHIDS_SALT = 'pinkhimalayan' # django-elasticsearch-metrics -ELASTICSEARCH_DSL = { - 'default': { - 'hosts': osf_settings.ELASTIC6_URI, - 'retry_on_timeout': True, +DJELME_AUTOSETUP = True +DJELME_BACKENDS = { + 'osfmetrics_es6': { + 'elasticsearch_metrics.imps.elastic6': { + 'hosts': osf_settings.ELASTIC6_URI, + 'retry_on_timeout': True, + }, + }, + 'osfmetrics_es8': { + 'elasticsearch_metrics.imps.elastic8': { + 'hosts': osf_settings.ELASTIC8_URI, + }, }, } # Store yearly indices for time-series metrics diff --git a/api/metrics/views.py b/api/metrics/views.py index daaa684d13a..c6e4d56c9b9 100644 --- a/api/metrics/views.py +++ b/api/metrics/views.py @@ -6,8 +6,8 @@ from django.http import JsonResponse, HttpResponse, Http404 from django.utils import timezone -from elasticsearch.exceptions import NotFoundError, RequestError -from elasticsearch_dsl.connections import get_connection +from elasticsearch6.exceptions import NotFoundError, RequestError +from elasticsearch6_dsl.connections import get_connection from framework.auth.oauth_scopes import CoreScopes diff --git a/api_tests/institutions/views/test_institution_department_list.py b/api_tests/institutions/views/test_institution_department_list.py index c2a5c0fcf99..8b785504756 100644 --- a/api_tests/institutions/views/test_institution_department_list.py +++ b/api_tests/institutions/views/test_institution_department_list.py @@ -44,7 +44,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution): department_name='Old Department', public_project_count=1, private_project_count=1, - ).save(refresh=True) + ).save() _this_month = YearMonth.from_date(datetime.date.today()) @@ -56,7 +56,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution): department_name='New Department', public_project_count=1, private_project_count=1, - ).save(refresh=True) + ).save() # A second user entered the department InstitutionalUserReport( @@ -66,7 +66,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution): department_name='New Department', public_project_count=1, private_project_count=1, - ).save(refresh=True) + ).save() # A new department with a single user to test sorting InstitutionalUserReport( @@ -76,7 +76,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution): department_name='Smaller Department', public_project_count=1, private_project_count=1, - ).save(refresh=True) + ).save() # A user with no department InstitutionalUserReport( @@ -85,7 +85,7 @@ def populate_counts(self, user, user2, user3, user4, admin, institution): institution_id=institution._id, public_project_count=1, private_project_count=1, - ).save(refresh=True) + ).save() @pytest.fixture() def admin(self, institution): @@ -113,6 +113,7 @@ def test_auth(self, app, url, user, admin): assert resp.json['data'] == [] def test_get(self, app, url, admin, institution, populate_counts): + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) resp = app.get(url, auth=admin.auth) assert resp.json['data'] == [{ diff --git a/api_tests/institutions/views/test_institution_summary_metrics.py b/api_tests/institutions/views/test_institution_summary_metrics.py index 41983458d2e..6dd6c5bbda3 100644 --- a/api_tests/institutions/views/test_institution_summary_metrics.py +++ b/api_tests/institutions/views/test_institution_summary_metrics.py @@ -84,6 +84,7 @@ def test_get_empty(self, app, url, institutional_admin): assert resp.json['meta'] == {'version': '2.0'} def test_get_report(self, app, url, institutional_admin, institution, reports, unshown_reports): + InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern) resp = app.get(url, auth=institutional_admin.auth) assert resp.status_code == 200 @@ -149,6 +150,7 @@ def test_get_report_with_multiple_months_and_institutions( monthly_logged_in_user_count=270, monthly_active_user_count=260, ) + InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern) resp = app.get(url, auth=institutional_admin.auth) assert resp.status_code == 200 @@ -189,6 +191,7 @@ def test_get_with_valid_report_dates(self, app, url, institution, institutional_ institution, user_count=4133, ) + InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern) resp = app.get(f'{url}?report_yearmonth=2024-08', auth=institutional_admin.auth) assert resp.status_code == 200 @@ -213,6 +216,7 @@ def test_get_with_invalid_report_date(self, app, url, institution, institutional institution, user_count=999, ) + InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern) # Request with an invalid report_date format resp = app.get(f'{url}?report_yearmonth=invalid-date', auth=institutional_admin.auth) @@ -233,6 +237,7 @@ def test_get_without_report_date_uses_most_recent(self, app, url, institution, i institution, user_count=999, ) + InstitutionMonthlySummaryReport._get_connection().indices.refresh(InstitutionMonthlySummaryReport._template_pattern) resp = app.get(url, auth=institutional_admin.auth) assert resp.status_code == 200 @@ -247,5 +252,5 @@ def _summary_report_factory(yearmonth, institution, **kwargs): institution_id=institution._id, **kwargs, ) - report.save(refresh=True) + report.save() return report diff --git a/api_tests/institutions/views/test_institution_user_metric_list.py b/api_tests/institutions/views/test_institution_user_metric_list.py index 0826dcd0161..d2b99da435f 100644 --- a/api_tests/institutions/views/test_institution_user_metric_list.py +++ b/api_tests/institutions/views/test_institution_user_metric_list.py @@ -89,6 +89,7 @@ def test_get_empty(self, app, url, institutional_admin): assert _resp.json['data'] == [] def test_get_reports(self, app, url, institutional_admin, institution, reports, unshown_reports): + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) _resp = app.get(url, auth=institutional_admin.auth) assert _resp.status_code == 200 assert len(_resp.json['data']) == len(reports) @@ -100,6 +101,7 @@ def test_get_reports(self, app, url, institutional_admin, institution, reports, assert len(response_object['attributes']['contacts']) == 0 def test_filter_reports(self, app, url, institutional_admin, institution, reports, unshown_reports): + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) for _query, _expected_user_ids in ( ({'filter[department]': 'nunavum'}, set()), ({'filter[department]': 'incidentally'}, set()), @@ -135,6 +137,7 @@ def test_filter_reports(self, app, url, institutional_admin, institution, report assert set(_user_ids(_resp)) == _expected_user_ids def test_sort_reports(self, app, url, institutional_admin, institution, reports, unshown_reports): + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) for _query, _expected_user_id_list in ( ({'sort': 'storage_byte_count'}, ['u_sparse', 'u_orc', 'u_blargl', 'u_orcomma']), ({'sort': '-storage_byte_count'}, ['u_orcomma', 'u_blargl', 'u_orc', 'u_sparse']), @@ -144,6 +147,7 @@ def test_sort_reports(self, app, url, institutional_admin, institution, reports, assert list(_user_ids(_resp)) == _expected_user_id_list def test_paginate_reports(self, app, url, institutional_admin, institution, reports, unshown_reports): + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) for _query, _expected_user_id_list in ( ({'sort': 'storage_byte_count', 'page[size]': 2}, ['u_sparse', 'u_orc']), ({'sort': 'storage_byte_count', 'page[size]': 2, 'page': 2}, ['u_blargl', 'u_orcomma']), @@ -178,6 +182,7 @@ def test_get_report_formats_csv_tsv(self, app, url, institutional_admin, institu month_last_active='2018-02', month_last_login='2018-02', ) + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) resp = app.get(f'{url}?format={format_type}', auth=institutional_admin.auth) assert resp.status_code == 200 @@ -281,6 +286,7 @@ def test_csv_tsv_ignores_pagination(self, app, url, institutional_admin, institu str(736662999298 + i), f'Jalen Hurts #{i}', ]) + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) # Make request for CSV format with page[size]=10 resp = app.get(f'{url}?format={format_type}', auth=institutional_admin.auth) @@ -346,6 +352,7 @@ def test_get_report_format_table_json(self, app, url, institutional_admin, insti month_last_active='2018-02', month_last_login='2018-02', ) + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) resp = app.get(f'{url}?format=json_report', auth=institutional_admin.auth) assert resp.status_code == 200 @@ -411,6 +418,7 @@ def test_correct_number_of_contact_messages(self, app, url, institutional_admin, department_name='a department, or so, that happens, incidentally, to have commas', storage_byte_count=736662999298, ) + InstitutionalUserReport._get_connection().indices.refresh(InstitutionalUserReport._template_pattern) receiver = user1 with capture_notifications(): @@ -477,5 +485,5 @@ def _report_factory(yearmonth, institution, **kwargs): institution_id=institution._id, **kwargs, ) - _report.save(refresh=True) + _report.save() return _report diff --git a/api_tests/metrics/test_composite_query.py b/api_tests/metrics/test_composite_query.py index 0cd0b3bb180..016677c3a11 100644 --- a/api_tests/metrics/test_composite_query.py +++ b/api_tests/metrics/test_composite_query.py @@ -1,4 +1,3 @@ -import time import pytest from datetime import datetime from osf_tests.factories import ( @@ -75,7 +74,7 @@ def test_elasticsearch_agg_query(self, app, user, base_url, preprint): path=preprint.primary_file.path, timestamp=datetime(year=2020, month=2, day=1) ) - time.sleep(1) # gives ES some time to update + PreprintDownload._get_connection().indices.refresh(PreprintDownload._template_pattern) resp = app.post_json_api(post_url, payload, auth=user.auth) assert resp.status_code == 200 diff --git a/api_tests/metrics/test_preprint_metrics.py b/api_tests/metrics/test_preprint_metrics.py index 1bde8719b75..cd9b8041c2d 100644 --- a/api_tests/metrics/test_preprint_metrics.py +++ b/api_tests/metrics/test_preprint_metrics.py @@ -8,7 +8,7 @@ from django.utils import timezone from waffle.testutils import override_switch -from elasticsearch.exceptions import RequestError +from elasticsearch6.exceptions import RequestError from osf import features from api.base.settings import API_PRIVATE_BASE as API_BASE diff --git a/api_tests/metrics/test_registries_moderation_metrics.py b/api_tests/metrics/test_registries_moderation_metrics.py index 93469b1b3b5..0f3dddb79b6 100644 --- a/api_tests/metrics/test_registries_moderation_metrics.py +++ b/api_tests/metrics/test_registries_moderation_metrics.py @@ -1,7 +1,6 @@ import pytest from waffle.testutils import override_switch -import time from osf import features from osf_tests.factories import RegistrationFactory, AuthUserFactory from osf.utils.workflows import RegistrationModerationStates, RegistrationModerationTriggers @@ -32,7 +31,7 @@ def test_record_transitions(self, registration): registration.creator, 'Metrics is easy' ) - time.sleep(1) + RegistriesModerationMetrics._get_connection().indices.refresh(RegistriesModerationMetrics._template_pattern) assert RegistriesModerationMetrics.search().count() == 1 data = RegistriesModerationMetrics.search().execute()['hits']['hits'][0]['_source'] @@ -81,7 +80,7 @@ def test_registries_moderation_view(self, app, user, base_url, registration): registration.creator, 'Metrics is easy' ) - time.sleep(1) + RegistriesModerationMetrics._get_connection().indices.refresh(RegistriesModerationMetrics._template_pattern) res = app.get(base_url, auth=user.auth, expect_errors=True) data = res.json diff --git a/conftest.py b/conftest.py index 9494e3d296e..232b788c0fb 100644 --- a/conftest.py +++ b/conftest.py @@ -1,14 +1,12 @@ -import contextlib from unittest import mock import logging import os import re -from django.core.management import call_command from django.db import transaction -from elasticsearch import exceptions as es_exceptions -from elasticsearch_dsl.connections import connections -from elasticsearch_metrics.registry import registry as es_metrics_registry +from elasticsearch6_dsl.connections import connections +from website import settings as osf_settings +from elasticsearch_metrics.tests._test_util import RealElasticTestCase from faker import Factory import pytest import responses @@ -138,47 +136,29 @@ def es6_client(setup_connections): @pytest.fixture(scope='function', autouse=True) -def _es_metrics_marker(request, worker_id): +def _es_metrics_marker(request): """Clear out all indices and index templates before and after tests marked with `es_metrics`. """ marker = request.node.get_closest_marker('es_metrics') - if marker: - es6_client = request.getfixturevalue('es6_client') - _temp_prefix = 'temp_metrics_' - _temp_wildcard = f'{_temp_prefix}-{worker_id}*' - - def _teardown_es_temps(): - es6_client.indices.delete(index=_temp_wildcard) - try: - es6_client.indices.delete_template(_temp_wildcard) - except es_exceptions.NotFoundError: - pass - - @contextlib.contextmanager - def _mock_metric_names(): - with contextlib.ExitStack() as _exit: - for _metric_class in es_metrics_registry.get_metrics(): - _exit.enter_context(mock.patch.object( - _metric_class, - '_template_name', # also used to construct index names - f'{_temp_prefix}-{worker_id}{_metric_class._template_name}', - )) - _exit.enter_context(mock.patch.object( - _metric_class, - '_template', # a wildcard string for indexes and templates - f'{_temp_prefix}-{worker_id}{_metric_class._template}', - )) - yield - - _teardown_es_temps() - with _mock_metric_names(): - call_command('sync_metrics') - yield - _teardown_es_temps() - else: + + if not marker: yield + return + connections.create_connection( + alias='osfmetrics_es6', + hosts=osf_settings.ELASTIC6_URI, + ) + + class _Es6TestCase(RealElasticTestCase, autosetup_djelme_backends=True): + ... + es6_test_case = _Es6TestCase() + es6_test_case.setUp() + try: + yield + finally: + es6_test_case.tearDown() @pytest.fixture def mock_share_responses(): diff --git a/docker-compose.yml b/docker-compose.yml index f00b589f7e0..f26c3617b67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,8 @@ volumes: external: false elasticsearch6_data_vol: external: false + elasticsearch8_data_vol: + external: false rabbitmq_vol: external: false ember_osf_web_dist_vol: @@ -76,6 +78,18 @@ services: - elasticsearch6_data_vol:/usr/share/elasticsearch/data stdin_open: true + elasticsearch8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11 + platform: linux/arm64 + environment: + - xpack.security.enabled=false + - discovery.type=single-node + ports: + - 9202:9200 + volumes: + - elasticsearch8_data_vol:/usr/share/elasticsearch/data + stdin_open: true + postgres: image: postgres:15.4 command: diff --git a/osf/management/commands/monthly_reporters_go.py b/osf/management/commands/monthly_reporters_go.py index 83ed5f6d985..218b45da1df 100644 --- a/osf/management/commands/monthly_reporters_go.py +++ b/osf/management/commands/monthly_reporters_go.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from django.db import OperationalError as DjangoOperationalError -from elasticsearch.exceptions import ConnectionError as ElasticConnectionError +from elasticsearch6.exceptions import ConnectionError as ElasticConnectionError from psycopg2 import OperationalError as PostgresOperationalError from framework.celery_tasks import app as celery_app diff --git a/osf/management/commands/reindex_es6.py b/osf/management/commands/reindex_es6.py index c37d0e34f2c..8961ea6fff1 100644 --- a/osf/management/commands/reindex_es6.py +++ b/osf/management/commands/reindex_es6.py @@ -4,7 +4,7 @@ import logging from django.core.management.base import BaseCommand -from elasticsearch_dsl import connections +from elasticsearch6_dsl import connections from elasticsearch_metrics.registry import registry logger = logging.getLogger(__name__) diff --git a/osf/metrics/counted_usage.py b/osf/metrics/counted_usage.py index 39b3b74129b..41ea012fda5 100644 --- a/osf/metrics/counted_usage.py +++ b/osf/metrics/counted_usage.py @@ -4,7 +4,7 @@ from urllib.parse import urlsplit from elasticsearch6_dsl import InnerDoc, analyzer, tokenizer -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from elasticsearch_metrics.signals import pre_save from django.dispatch import receiver import pytz diff --git a/osf/metrics/metric_mixin.py b/osf/metrics/metric_mixin.py index 724ab1958da..df87d5123b1 100644 --- a/osf/metrics/metric_mixin.py +++ b/osf/metrics/metric_mixin.py @@ -2,7 +2,7 @@ from django.db import models from django.utils import timezone -from elasticsearch.exceptions import NotFoundError +from elasticsearch6.exceptions import NotFoundError import pytz diff --git a/osf/metrics/preprint_metrics.py b/osf/metrics/preprint_metrics.py index 9d02ec191a2..d284d80827e 100644 --- a/osf/metrics/preprint_metrics.py +++ b/osf/metrics/preprint_metrics.py @@ -1,5 +1,5 @@ -from elasticsearch.exceptions import NotFoundError -from elasticsearch_metrics import metrics +from elasticsearch6.exceptions import NotFoundError +import elasticsearch_metrics.imps.elastic6 as metrics from .metric_mixin import MetricMixin diff --git a/osf/metrics/registry_metrics.py b/osf/metrics/registry_metrics.py index 475dca28673..9c779fe8c0b 100644 --- a/osf/metrics/registry_metrics.py +++ b/osf/metrics/registry_metrics.py @@ -1,4 +1,4 @@ -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from osf.utils.workflows import RegistrationModerationTriggers, RegistrationModerationStates from .metric_mixin import MetricMixin diff --git a/osf/metrics/reporters/public_item_usage.py b/osf/metrics/reporters/public_item_usage.py index cc401d50bd7..7df405d385f 100644 --- a/osf/metrics/reporters/public_item_usage.py +++ b/osf/metrics/reporters/public_item_usage.py @@ -4,7 +4,7 @@ import waffle if typing.TYPE_CHECKING: - import elasticsearch_dsl as edsl + import elasticsearch6_dsl as edsl import osf.features from osf.metadata.osf_gathering import OsfmapPartition diff --git a/osf/metrics/reports.py b/osf/metrics/reports.py index ffbcfb4c9b8..9d71ea7e8c2 100644 --- a/osf/metrics/reports.py +++ b/osf/metrics/reports.py @@ -4,7 +4,7 @@ from django.dispatch import receiver from elasticsearch6_dsl import InnerDoc -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from elasticsearch_metrics.signals import pre_save as metrics_pre_save from osf.metrics.utils import stable_key, YearMonth diff --git a/osf_tests/management_commands/test_reindex_es6.py b/osf_tests/management_commands/test_reindex_es6.py index 5e01be656a8..36158c18da6 100644 --- a/osf_tests/management_commands/test_reindex_es6.py +++ b/osf_tests/management_commands/test_reindex_es6.py @@ -10,7 +10,7 @@ AuthUserFactory ) -from elasticsearch_metrics.field import Keyword +from elasticsearch6_dsl import Keyword from tests.json_api_test_app import JSONAPITestApp diff --git a/osf_tests/metrics/test_daily_report.py b/osf_tests/metrics/test_daily_report.py index 46375184f95..9301cdb114f 100644 --- a/osf_tests/metrics/test_daily_report.py +++ b/osf_tests/metrics/test_daily_report.py @@ -2,7 +2,7 @@ from unittest import mock import pytest -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from osf.metrics.reports import DailyReport, ReportInvalid diff --git a/osf_tests/metrics/test_metric_mixin.py b/osf_tests/metrics/test_metric_mixin.py index 4a2c32f7e71..ec9b2d302de 100644 --- a/osf_tests/metrics/test_metric_mixin.py +++ b/osf_tests/metrics/test_metric_mixin.py @@ -1,6 +1,6 @@ from unittest import mock import pytest -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from osf.metrics.metric_mixin import MetricMixin from osf.models import OSFUser diff --git a/osf_tests/metrics/test_monthly_report.py b/osf_tests/metrics/test_monthly_report.py index 3c841e6555c..cc8c4137cb2 100644 --- a/osf_tests/metrics/test_monthly_report.py +++ b/osf_tests/metrics/test_monthly_report.py @@ -2,7 +2,7 @@ from unittest import mock import pytest -from elasticsearch_metrics import metrics +import elasticsearch_metrics.imps.elastic6 as metrics from osf.metrics.reports import MonthlyReport, ReportInvalid, PublicItemUsageReport from osf.metrics.utils import YearMonth diff --git a/poetry.lock b/poetry.lock index 83ca13f7a00..f0dca07d95c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "amqp" @@ -1085,27 +1085,24 @@ Django = ">=2.0" [[package]] name = "django-elasticsearch-metrics" -version = "2022.0.6" +version = "2026.0.3" description = "Django app for storing time-series metrics in Elasticsearch." optional = false -python-versions = "*" +python-versions = ">=3.10,<4" groups = ["main"] files = [] develop = false -[package.dependencies] -elasticsearch6-dsl = ">=6.3.0,<7.0.0" - [package.extras] -dev = ["factory-boy (==2.11.1)", "flake8 (==5.0.4)", "flake8-bugbear (==18.8.0)", "konch (>=3.0.0)", "mock", "pre-commit (==2.17.0)", "pytest", "pytest-django (==3.10.0)", "tox"] -lint = ["flake8 (==5.0.4)", "flake8-bugbear (==18.8.0)", "pre-commit (==2.17.0)"] -tests = ["factory-boy (==2.11.1)", "mock", "pytest", "pytest-django (==3.10.0)"] +anydjango = ["django"] +elastic6 = ["elasticsearch6-dsl (>=6.3.0,<7.0.0)"] +elastic8 = ["elasticsearch8 (>=8.0.0,<9.0.0)"] [package.source] type = "git" url = "https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git" -reference = "f5b9312914154e213aa01731e934c593e3434269" -resolved_reference = "f5b9312914154e213aa01731e934c593e3434269" +reference = "bb1c84c148ac1d2b1079b2b113e52a01a861c8a6" +resolved_reference = "bb1c84c148ac1d2b1079b2b113e52a01a861c8a6" [[package]] name = "django-extensions" @@ -1189,7 +1186,7 @@ files = [ [package.dependencies] autopep8 = "*" Django = ">=3.2" -gprof2dot = ">=2017.09.19" +gprof2dot = ">=2017.9.19" sqlparse = "*" [[package]] @@ -1361,14 +1358,14 @@ stone = ">=2" [[package]] name = "elastic-transport" -version = "8.13.0" +version = "8.17.1" description = "Transport classes and utilities shared among Python Elastic client libraries" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "elastic-transport-8.13.0.tar.gz", hash = "sha256:2410ec1ff51221e8b3a01c0afa9f0d0498e1386a269283801f5c12f98e42dc45"}, - {file = "elastic_transport-8.13.0-py3-none-any.whl", hash = "sha256:aec890afdddd057762b27ff3553b0be8fa4673ec1a4fd922dfbd00325874bb3d"}, + {file = "elastic_transport-8.17.1-py3-none-any.whl", hash = "sha256:192718f498f1d10c5e9aa8b9cf32aed405e469a7f0e9d6a8923431dbb2c59fb8"}, + {file = "elastic_transport-8.17.1.tar.gz", hash = "sha256:5edef32ac864dca8e2f0a613ef63491ee8d6b8cfb52881fa7313ba9290cac6d2"}, ] [package.dependencies] @@ -1376,46 +1373,7 @@ certifi = "*" urllib3 = ">=1.26.2,<3" [package.extras] -develop = ["aiohttp", "furo", "httpx", "mock", "opentelemetry-api", "opentelemetry-sdk", "orjson", "pytest", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests", "respx", "sphinx (>2)", "sphinx-autodoc-typehints", "trustme"] - -[[package]] -name = "elasticsearch" -version = "6.8.2" -description = "Python client for Elasticsearch" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" -groups = ["main"] -files = [ - {file = "elasticsearch-6.8.2-py2.py3-none-any.whl", hash = "sha256:1aedf00b73f5d1e77cb4df70fec58f2efb664be4ce2686374239aa6c0373c65c"}, - {file = "elasticsearch-6.8.2.tar.gz", hash = "sha256:c3a560bb83e4981b5a5c82080d2ceb99686d33692ef53365656129478aa5ddb2"}, -] - -[package.dependencies] -urllib3 = ">=1.21.1" - -[package.extras] -develop = ["coverage", "mock", "nose", "nosexcover", "numpy", "pandas", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx (<1.7)", "sphinx-rtd-theme"] -requests = ["requests (>=2.4.0,<3.0.0)"] - -[[package]] -name = "elasticsearch-dsl" -version = "6.4.0" -description = "Python client for Elasticsearch" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "elasticsearch-dsl-6.4.0.tar.gz", hash = "sha256:26416f4dd46ceca43d62ef74970d9de4bdd6f4b0f163316f0b432c9e61a08bec"}, - {file = "elasticsearch_dsl-6.4.0-py2.py3-none-any.whl", hash = "sha256:f60aea7fd756ac1fbe7ce114bbf4949aefbf495dfe8896640e787c67344f12f6"}, -] - -[package.dependencies] -elasticsearch = ">=6.0.0,<7.0.0" -python-dateutil = "*" -six = "*" - -[package.extras] -develop = ["coverage (<5.0.0)", "mock", "pytest (>=3.0.0)", "pytest-cov", "pytz", "sphinx", "sphinx-rtd-theme"] +develop = ["aiohttp", "furo", "httpx", "opentelemetry-api", "opentelemetry-sdk", "orjson", "pytest", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests", "respx", "sphinx (>2)", "sphinx-autodoc-typehints", "trustme"] [[package]] name = "elasticsearch2" @@ -1471,6 +1429,32 @@ six = "*" [package.extras] develop = ["coverage (<5.0.0)", "mock", "pytest (>=3.0.0)", "pytest-cov", "pytz", "sphinx", "sphinx-rtd-theme"] +[[package]] +name = "elasticsearch8" +version = "8.19.3" +description = "Python client for Elasticsearch" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "elasticsearch8-8.19.3-py3-none-any.whl", hash = "sha256:4b52e59e68aea6f59bf37c28f6f4512333302dd8a52e26c17d0f10c076d833a1"}, + {file = "elasticsearch8-8.19.3.tar.gz", hash = "sha256:7effe95b360241b6d56ef68219037a90ad0f56723614db54bbe57d33058402f4"}, +] + +[package.dependencies] +elastic-transport = ">=8.15.1,<9" +python-dateutil = "*" +typing-extensions = "*" + +[package.extras] +async = ["aiohttp (>=3,<4)"] +dev = ["aiohttp", "black", "build", "coverage", "isort", "jinja2", "mapbox-vector-tile", "mypy", "nox", "numpy", "orjson", "pandas", "pyarrow ; python_version < \"3.14\"", "pyright", "pytest", "pytest-asyncio", "pytest-cov", "pytest-mock", "python-dateutil", "pyyaml (>=5.4)", "requests (>=2,<3)", "simsimd", "tqdm", "twine", "types-python-dateutil", "types-tqdm", "unasync"] +docs = ["sphinx", "sphinx-autodoc-typehints", "sphinx-rtd-theme (>=2.0)"] +orjson = ["orjson (>=3)"] +pyarrow = ["pyarrow (>=1)"] +requests = ["requests (>=2.4.0,!=2.32.2,<3.0.0)"] +vectorstore-mmr = ["numpy (>=1)", "simsimd (>=3)"] + [[package]] name = "email-validator" version = "2.1.1" @@ -1755,12 +1739,12 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" -proto-plus = ">=1.22.3,<2.0.0dev" +proto-plus = ">=1.22.3,<2.0.0.dev0" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] +grpc = ["grpcio (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] @@ -1836,11 +1820,11 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=1.25.0,<3.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.25.0,<3.0.dev0" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] +grpc = ["grpcio (>=1.38.0,<2.0.dev0)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" @@ -1855,15 +1839,15 @@ files = [ ] [package.dependencies] -google-api-core = ">=2.15.0,<3.0.0dev" -google-auth = ">=2.26.1,<3.0dev" -google-cloud-core = ">=2.3.0,<3.0dev" -google-crc32c = ">=1.0,<2.0dev" +google-api-core = ">=2.15.0,<3.0.0.dev0" +google-auth = ">=2.26.1,<3.0.dev0" +google-cloud-core = ">=2.3.0,<3.0.dev0" +google-crc32c = ">=1.0,<2.0.dev0" google-resumable-media = ">=2.6.0" -requests = ">=2.18.0,<3.0.0dev" +requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -protobuf = ["protobuf (<5.0.0dev)"] +protobuf = ["protobuf (<5.0.0.dev0)"] [[package]] name = "google-crc32c" @@ -1918,11 +1902,11 @@ files = [ ] [package.dependencies] -google-crc32c = ">=1.0,<2.0dev" +google-crc32c = ">=1.0,<2.0.dev0" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] -requests = ["requests (>=2.18.0,<3.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "google-auth (>=1.22.0,<2.0.dev0)"] +requests = ["requests (>=2.18.0,<3.0.0.dev0)"] [[package]] name = "googleapis-common-protos" @@ -2301,7 +2285,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -3058,7 +3042,7 @@ files = [ ] [package.dependencies] -protobuf = ">=3.19.0,<6.0.0dev" +protobuf = ">=3.19.0,<6.0.0.dev0" [package.extras] testing = ["google-api-core (>=1.31.5)"] @@ -4088,10 +4072,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.33.2,<2.0a.0" +botocore = ">=1.33.2,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] +crt = ["botocore[crt] (>=1.33.2,<2.0a0)"] [[package]] name = "schema" @@ -4412,6 +4396,18 @@ files = [ {file = "types_python_dateutil-2.9.0.20240906-py3-none-any.whl", hash = "sha256:27c8cc2d058ccb14946eebcaaa503088f4f6dbc4fb6093d3d456a49aef2753f6"}, ] +[[package]] +name = "typing-extensions" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, +] + [[package]] name = "tzdata" version = "2024.1" @@ -4715,4 +4711,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "2bc7e95f03d05e8b3335514e887b590acdab5cb2a44fc47bde870bdf8e465bf2" +content-hash = "c3108a036ae092e35f7696ffe230e271b774bb12e546db77bb8b12b5fb6eca7d" diff --git a/pyproject.toml b/pyproject.toml index b1646584209..b40cdc704ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,10 @@ Markupsafe = "2.1.5" blinker = "1.7.0" furl = "2.1.3" elasticsearch2 = "2.5.1" -elasticsearch = "6.8.2" # max version to support elasticsearch6 -elasticsearch-dsl = "6.4.0" # max version to support elasticsearch6 -elastic-transport = "8.13.0" +elasticsearch6= "6.8.2" +elasticsearch6-dsl = "6.4.0" +elasticsearch8 = "8.19.3" +elastic-transport = "8.17.1" google-api-python-client = "2.123.0" google-auth = "2.29.0" Babel = "2.14.0" @@ -90,7 +91,7 @@ datacite = "1.1.3" rdflib = "7.0.0" colorlog = "6.8.2" # Metrics -django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "f5b9312914154e213aa01731e934c593e3434269"} # branch is feature/pin-esdsl +django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "bb1c84c148ac1d2b1079b2b113e52a01a861c8a6"} # Impact Metrics CSV Export djangorestframework-csv = "3.0.2" gevent = "24.2.1" diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 2d3dcecba3b..d0ae58dc863 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -113,6 +113,7 @@ def parent_dir(path): SEARCH_ENGINE = 'elastic' # Can be 'elastic', or None ELASTIC_URI = '127.0.0.1:9200' ELASTIC6_URI = os.environ.get('ELASTIC6_URI', '127.0.0.1:9201') +ELASTIC8_URI = os.environ.get('ELASTIC8_URI', '127.0.0.1:9202') ELASTIC_TIMEOUT = 10 ELASTIC_INDEX = 'website' ELASTIC_KWARGS = {