[ENG-9702] Feature/OSF to use latest djelme

.docker-compose.env

@@ -7,6 +7,7 @@ INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
+ELASTIC8_URI=http://192.168.168.167:9202
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
 REDIS_HOST=redis://192.168.168.167:6379

api/base/settings/defaults.py

@@ -316,10 +316,18 @@
 HASHIDS_SALT = 'pinkhimalayan'
 
 # django-elasticsearch-metrics
-ELASTICSEARCH_DSL = {
-    'default': {
-        'hosts': osf_settings.ELASTIC6_URI,
-        'retry_on_timeout': True,
+DJELME_AUTOSETUP = True
+DJELME_BACKENDS = {
+    'osfmetrics_es6': {
+        'elasticsearch_metrics.imps.elastic6': {
+            'hosts': osf_settings.ELASTIC6_URI,
+            'retry_on_timeout': True,
+        },
+    },
+    'osfmetrics_es8': {
+        'elasticsearch_metrics.imps.elastic8': {
+            'hosts': osf_settings.ELASTIC8_URI,
+        },
     },
 }
 # Store yearly indices for time-series metrics

conftest.py

@@ -1,14 +1,11 @@
-import contextlib
 from unittest import mock
 import logging
 import os
 import re
 
-from django.core.management import call_command
 from django.db import transaction
-from elasticsearch import exceptions as es_exceptions
 from elasticsearch_dsl.connections import connections
-from elasticsearch_metrics.registry import registry as es_metrics_registry
+from elasticsearch_metrics.tests._test_util import RealElasticTestCase
 from faker import Factory
 import pytest
 import responses
@@ -138,47 +135,22 @@ def es6_client(setup_connections):
 
 
 @pytest.fixture(scope='function', autouse=True)
-def _es_metrics_marker(request, worker_id):
+def _es_metrics_marker(request):
     """Clear out all indices and index templates before and after
     tests marked with `es_metrics`.
     """
     marker = request.node.get_closest_marker('es_metrics')
-    if marker:
-        es6_client = request.getfixturevalue('es6_client')
-        _temp_prefix = 'temp_metrics_'
-        _temp_wildcard = f'{_temp_prefix}-{worker_id}*'
-
-        def _teardown_es_temps():
-            es6_client.indices.delete(index=_temp_wildcard)
-            try:
-                es6_client.indices.delete_template(_temp_wildcard)
-            except es_exceptions.NotFoundError:
-                pass
-
-        @contextlib.contextmanager
-        def _mock_metric_names():
-            with contextlib.ExitStack() as _exit:
-                for _metric_class in es_metrics_registry.get_metrics():
-                    _exit.enter_context(mock.patch.object(
-                        _metric_class,
-                        '_template_name',  # also used to construct index names
-                        f'{_temp_prefix}-{worker_id}{_metric_class._template_name}',
-                    ))
-                    _exit.enter_context(mock.patch.object(
-                        _metric_class,
-                        '_template',  # a wildcard string for indexes and templates
-                        f'{_temp_prefix}-{worker_id}{_metric_class._template}',
-                    ))
-                yield
-
-        _teardown_es_temps()
-        with _mock_metric_names():
-            call_command('sync_metrics')
-            yield
-        _teardown_es_temps()
-    else:
+
+    if not marker:
         yield
+        return
+
+    es6_test_case = RealElasticTestCase()
+    es6_test_case.setup_backends()
+
+    yield
 
+    es6_test_case.teardown_backends()
 
 @pytest.fixture
 def mock_share_responses():

docker-compose.yml

@@ -13,6 +13,8 @@ volumes:
     external: false
   elasticsearch6_data_vol:
     external: false
+  elasticsearch8_data_vol:
+    external: false
   rabbitmq_vol:
     external: false
   ember_osf_web_dist_vol:
@@ -76,6 +78,18 @@ services:
       - elasticsearch6_data_vol:/usr/share/elasticsearch/data
     stdin_open: true
 
+  elasticsearch8:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11
+    platform: linux/arm64
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+    ports:
+      - 9202:9200
+    volumes:
+      - elasticsearch8_data_vol:/usr/share/elasticsearch/data
+    stdin_open: true
+
   postgres:
     image: postgres:15.4
     command:

osf/metrics/counted_usage.py

@@ -4,7 +4,7 @@
 from urllib.parse import urlsplit
 
 from elasticsearch6_dsl import InnerDoc, analyzer, tokenizer
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 from elasticsearch_metrics.signals import pre_save
 from django.dispatch import receiver
 import pytz

osf/metrics/preprint_metrics.py

@@ -1,5 +1,5 @@
 from elasticsearch.exceptions import NotFoundError
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 
 from .metric_mixin import MetricMixin
 

osf/metrics/registry_metrics.py

@@ -1,4 +1,4 @@
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 
 from osf.utils.workflows import RegistrationModerationTriggers, RegistrationModerationStates
 from .metric_mixin import MetricMixin

osf/metrics/reports.py

@@ -4,7 +4,7 @@
 
 from django.dispatch import receiver
 from elasticsearch6_dsl import InnerDoc
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 from elasticsearch_metrics.signals import pre_save as metrics_pre_save
 
 from osf.metrics.utils import stable_key, YearMonth

osf_tests/metrics/test_daily_report.py

@@ -2,7 +2,7 @@
 from unittest import mock
 
 import pytest
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 
 from osf.metrics.reports import DailyReport, ReportInvalid
 

osf_tests/metrics/test_metric_mixin.py

@@ -1,6 +1,6 @@
 from unittest import mock
 import pytest
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 
 from osf.metrics.metric_mixin import MetricMixin
 from osf.models import OSFUser

osf_tests/metrics/test_monthly_report.py

@@ -2,7 +2,7 @@
 from unittest import mock
 
 import pytest
-from elasticsearch_metrics import metrics
+import elasticsearch_metrics.imps.elastic6 as metrics
 
 from osf.metrics.reports import MonthlyReport, ReportInvalid, PublicItemUsageReport
 from osf.metrics.utils import YearMonth

pyproject.toml

@@ -33,7 +33,9 @@ furl = "2.1.3"
 elasticsearch2 = "2.5.1"
 elasticsearch = "6.8.2" # max version to support elasticsearch6
 elasticsearch-dsl = "6.4.0" # max version to support elasticsearch6
-elastic-transport = "8.13.0"
+elasticsearch6-dsl = "6.4.0"
+elasticsearch8 = "8.19.3"
+elastic-transport = "8.17.1"
 google-api-python-client = "2.123.0"
 google-auth = "2.29.0"
 Babel = "2.14.0"
@@ -90,7 +92,7 @@ datacite = "1.1.3"
 rdflib = "7.0.0"
 colorlog = "6.8.2"
 # Metrics
-django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "f5b9312914154e213aa01731e934c593e3434269"} # branch is feature/pin-esdsl
+django-elasticsearch-metrics = {git ="https://github.com/CenterForOpenScience/django-elasticsearch-metrics.git", rev = "7a7f664469070dd52dc4d9401f6b6d2d9fe7ddf0"} # branch is feature/pin-esdsl
 # Impact Metrics CSV Export
 djangorestframework-csv = "3.0.2"
 gevent = "24.2.1"

website/settings/defaults.py

@@ -113,6 +113,7 @@ def parent_dir(path):
 SEARCH_ENGINE = 'elastic'  # Can be 'elastic', or None
 ELASTIC_URI = '127.0.0.1:9200'
 ELASTIC6_URI = os.environ.get('ELASTIC6_URI', '127.0.0.1:9201')
+ELASTIC8_URI = os.environ.get('ELASTIC8_URI', '127.0.0.1:9202')
 ELASTIC_TIMEOUT = 10
 ELASTIC_INDEX = 'website'
 ELASTIC_KWARGS = {