[ENG-9691][feature] osfmetrics cleanup drop es6

.docker-compose.env

@@ -6,7 +6,6 @@ DOMAIN=http://localhost:5000/
 INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
-ELASTIC6_URI=192.168.168.167:9201
 ELASTIC8_URI=http://192.168.168.167:9202
 ELASTIC8_USERNAME=elastic
 OSF_DB_HOST=192.168.168.167

.github/actions/start-build/action.yml

@@ -12,9 +12,6 @@ runs:
   - uses: ./.github/actions/build-es
     with:
       ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-  - uses: ./.github/actions/build-es6
-    with:
-      ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
   - name: Set up Python 3.12
     uses: actions/setup-python@v6
     with:

.github/workflows/test-build.yml

@@ -8,7 +8,6 @@ permissions:
 env:
   WHEELHOUSE: ~/.cache/wheelhouse
   ELASTICSEARCH_ARCHIVE: elasticsearch-2.4.5.tar.gz
-  ELASTICSEARCH6_ARCHIVE: elasticsearch-6.3.1.tar.gz
   OSF_DB_PORT: 5432
   OSF_DB_PASSWORD: postgres
   GITHUB_ACTIONS: true
@@ -140,6 +139,8 @@ jobs:
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-api3-and-osf --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report

README-docker-compose.md

@@ -116,11 +116,7 @@
 
   #### Special Instructions for Apple Chipset (M1, M2, etc.) and other ARM64 architecture
 
-  * _NOTE: The default `elasticsearch`, `elasticsearch6`, and `sharejs` containers are incompatible with ARM64._
-
-  - To run `elasticsearch6` on ARM64 architecture:
-
-    - Copy `docker-compose-dist-arm64.override.yml` into your `docker-compose.override.yml` file
+  * _NOTE: The default `elasticsearch`  and `sharejs` containers are incompatible with ARM64._
 
   - Running containers with docker compose
 
@@ -194,7 +190,7 @@
 
 - Start all containers
   ```bash
-  alias dcsa="docker compose up -d assets admin_assets mfr wb fakecas sharejs worker elasticsearch elasticsearch6 web api admin preprints gv"
+  alias dcsa="docker compose up -d assets admin_assets mfr wb fakecas sharejs worker elasticsearch elasticsearch8 web api admin preprints gv"
   ```
 
 - Shut down all containers

addons/base/views.py

@@ -14,7 +14,7 @@
 import waffle
 from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-from elasticsearch6 import exceptions as es_exceptions
+from elasticsearch8 import exceptions as es_exceptions
 from rest_framework import status as http_status
 
 from api.caching.tasks import update_storage_usage_with_size
@@ -34,7 +34,7 @@
 from framework.flask import redirect
 from framework.sentry import log_exception
 from framework.transactions.handlers import no_auto_transaction
-from osf.metrics.es8_metrics import OsfCountedUsageEvent
+from osf.metrics.events import OsfCountedUsageEvent
 from website import settings
 from addons.base import signals as file_signals
 from addons.base.utils import format_last_known_metadata, get_mfr_url
@@ -54,7 +54,6 @@
     FileVersionUserMetadata,
     FileVersion, NotificationTypeEnum
 )
-from osf.metrics import PreprintView, PreprintDownload
 from osf.utils import permissions
 from osf.external.gravy_valet import request_helpers
 from website.profile.utils import get_profile_image_url
@@ -686,12 +685,6 @@ def osfstoragefile_viewed_update_metrics(self, auth, fileversion, file_node):
         return
     if waffle.switch_is_active(features.ELASTICSEARCH_METRICS) and isinstance(resource, Preprint):
         try:
-            PreprintView.record_for_preprint(
-                preprint=resource,
-                user=auth.user,
-                version=fileversion.identifier,
-                path=file_node.path,
-            )
             OsfCountedUsageEvent.record(
                 user_id=getattr(user, '_id', None),
                 item_osfid=resource._id,
@@ -725,12 +718,6 @@ def osfstoragefile_downloaded_update_metrics(self, auth, fileversion, file_node)
         return
     if waffle.switch_is_active(features.ELASTICSEARCH_METRICS) and isinstance(resource, Preprint):
         try:
-            PreprintDownload.record_for_preprint(
-                preprint=resource,
-                user=auth.user,
-                version=fileversion.identifier,
-                path=file_node.path,
-            )
             OsfCountedUsageEvent.record(
                 user_id=getattr(user, '_id', None),
                 item_osfid=resource._id,

admin/management/urls.py

@@ -1,4 +1,4 @@
-from django.urls import re_path, path
+from django.urls import re_path
 
 from admin.management import views
 
@@ -22,5 +22,4 @@
             name='sync_notification_templates'),
     re_path(r'^remove_orcid_from_user_social', views.RemoveOrcidFromUserSocial.as_view(),
             name='remove_orcid_from_user_social'),
-    path('migrate_osfmetrics_6to8', views.MigrateOsfmetrics6to8.as_view(), name='migrate_osfmetrics_6to8'),
 ]

admin/management/views.py

@@ -1,5 +1,3 @@
-from io import StringIO
-
 from dateutil.parser import isoparse
 from django.views.generic import TemplateView, View
 from django.contrib import messages
@@ -205,22 +203,3 @@ def post(self, request):
         remove_orcid_from_user_social()
         messages.success(request, 'Orcid from user social have been successfully removed.')
         return redirect(reverse('management:commands'))
-
-
-class MigrateOsfmetrics6to8(ManagementCommandPermissionView):
-    def post(self, request):
-        _command_kwargs = {
-            'no_color': True,
-            'no_counts': request.POST.get('no_counts'),
-            'clear_state': request.POST.get('clear_state'),
-            'clear_es8_data': request.POST.get('clear_es8_data'),
-            'start': request.POST.get('start'),
-            'unchanged': request.POST.get('unchanged'),
-            'usage_reports': request.POST.get('usage_reports'),
-            'usage_events': request.POST.get('usage_events'),
-        }
-        _out_io = StringIO()
-        call_command('migrate_osfmetrics_6to8', **_command_kwargs, stdout=_out_io)
-        for _line in _out_io.getvalue().split('\n'):
-            messages.info(request, _line)
-        return redirect(reverse('management:commands'))

admin/templates/management/commands.html

@@ -190,31 +190,6 @@ <h4><u>Remove existing orcid info from user social</u></h4>
                     </nav>
                 </form>
             </section>
-            <section>
-                <h4><u>migrate osf-metrics 6to8</u></h4>
-                <p>
-                    view progress of the osf-metrics migration from elastic6 to elastic8 (or start it)
-                </p>
-                <form method="post"
-                      action="{% url 'management:migrate_osfmetrics_6to8'%}"
-                      style="display: flex; flex-direction: column;">
-                    {% csrf_token %}
-                    <label><input type="checkbox" name="no_counts"> no counts</label>
-                    <label><input type="checkbox" name="start"> start tasks (caution)</label>
-                    <label><input type="checkbox" name="clear_state"> reset migration start time (caution)</label>
-                    <label><input type="checkbox" name="clear_es8_data"> clear es8 data (big caution)</label>
-                    <fieldset>
-                        (narrow types:
-                        <label><input type="checkbox" name="unchanged"> unchanged events and reports</label>
-                        <label><input type="checkbox" name="usage_events"> usage events</label>
-                        <label><input type="checkbox" name="usage_reports"> usage reports</label>
-                        )
-                    </fieldset>
-                    <nav>
-                        <input class="btn btn-success" type="submit" value="Run" />
-                    </nav>
-                </form>
-            </section>
         </div>
     </section>
 {% endblock %}

api/base/elasticsearch_dsl_views.py

@@ -3,8 +3,9 @@
 import datetime
 import typing
 
-import elasticsearch6_dsl as edsl
+import elasticsearch8.dsl as esdsl
 from rest_framework import generics, exceptions as drf_exceptions
+from rest_framework.serializers import Serializer
 from rest_framework.settings import api_settings as drf_settings
 from api.base.settings.defaults import REPORT_FILENAME_FORMAT
 
@@ -23,7 +24,7 @@
 
 
 class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC):
-    '''abstract view class using `elasticsearch6_dsl.Search` as a queryset-analogue
+    '''abstract view class using `elasticsearch8.dsl.Search` as a queryset-analogue
 
     builds a `Search` based on `self.get_default_search()` and the request's
     query parameters for filtering, sorting, and pagination -- fetches only
@@ -35,18 +36,18 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView,
     ordering_fields: frozenset[str] = frozenset()  # serializer field names
 
     @abc.abstractmethod
-    def get_default_search(self) -> edsl.Search | None:
-        '''the base `elasticsearch6_dsl.Search` for this list, based on url path
+    def get_default_search(self) -> esdsl.Search | None:
+        '''the base `elasticsearch8.dsl.Search` for this list, based on url path
 
         (common jsonapi query parameters will be considered automatically)
         '''
         ...
 
-    FILE_RENDERER_CLASSES = {
+    FILE_RENDERER_CLASSES = (
         MetricsReportsCsvRenderer,
         MetricsReportsTsvRenderer,
         MetricsReportsJsonRenderer,
-    }
+    )
 
     def set_content_disposition(self, response, renderer: str):
         """Set the Content-Disposition header to prompt a file download with the appropriate filename.
@@ -75,7 +76,7 @@ def finalize_response(self, request, response, *args, **kwargs):
         response = super().finalize_response(request, response, *args, **kwargs)
         # Check if this is a direct download request or file renderer classes, set to the Content-Disposition header
         # so filename and attachment for browser download
-        if isinstance(request.accepted_renderer, tuple(self.FILE_RENDERER_CLASSES)):
+        if isinstance(request.accepted_renderer, self.FILE_RENDERER_CLASSES):
             self.set_content_disposition(response, request.accepted_renderer)
 
         return response
@@ -95,7 +96,7 @@ def finalize_response(self, request, response, *args, **kwargs):
     # (filtering handled in-view to reuse logic from FilterMixin)
     filter_backends = ()
 
-    # note: because elasticsearch6_dsl.Search supports slicing and gives results when iterated on,
+    # note: because elasticsearch8.dsl.Search supports slicing and gives results when iterated on,
     #       it works fine with default pagination
 
     # override rest_framework.generics.GenericAPIView
@@ -128,10 +129,17 @@ def get_queryset(self):
                 )
         return self.__add_sort(_search)
 
+    def get_serializer_context(self):
+        return (
+            super().get_serializer_context()
+            if issubclass(self.get_serializer_class(), Serializer)
+            else {}  # allow custom BaseSerializer-based serializer
+        )
+
     ###
     # private methods
 
-    def __add_sort(self, search: edsl.Search) -> edsl.Search:
+    def __add_sort(self, search: esdsl.Search) -> esdsl.Search:
         _elastic_sort = self.__get_elastic_sort()
         return (search if _elastic_sort is None else search.sort(_elastic_sort))
 
@@ -148,17 +156,20 @@ def __get_elastic_sort(self) -> str | None:
             raise drf_exceptions.ValidationError(
                 f'invalid value for {drf_settings.ORDERING_PARAM} query param (valid values: {", ".join(self.ordering_fields)})',
             )
-        _serializer_field = self.get_serializer().fields[_sort_field]
-        _elastic_sort_field = _serializer_field.source
+        _elastic_sort_field = (
+            self.get_serializer().fields[_sort_field].source
+            if issubclass(self.get_serializer_class(), Serializer)
+            else _sort_field  # allow custom BaseSerializer-based serializer
+        )
         return (_elastic_sort_field if _ascending else f'-{_elastic_sort_field}')
 
     def __add_search_filter(
         self,
-        search: edsl.Search,
+        search: esdsl.Search,
         elastic_field_name: str,
         operator: str,
         value: str,
-    ) -> edsl.Search:
+    ) -> esdsl.Search:
         match operator:  # operators from FilterMixin
             case 'eq':
                 if value == '':