Query optimization

docs/development/index.rst

@@ -15,3 +15,4 @@ developer-focussed usage information, is provided in the below docs.
    /development/releasing
    /development/api
    /development/assets
+   /development/performance

docs/development/performance.rst

@@ -0,0 +1,29 @@
+Performance testing
+===================
+
+Patchwork provides the ``generate_perf_data`` management command to create
+synthetic projects, submitters, series, patches and replies for local performance
+testing. You can use this with a disposable database, configured like the
+database backend of the instance you want to measure. You can use the command
+like so:
+
+.. code-block:: shell
+
+    python manage.py migrate
+    python manage.py loaddata default_states
+    python manage.py generate_perf_data --projects 12 --patches-per-project 100000
+
+The defaults create 1.2 million patches in batches of 1,000, with up to ten
+patches per series and 100 shared submitters. Each patch has between 1 and 10
+replies (roughly 6.6 million replies with the defaults), with randomly chosen
+submitters and dates after the patch. Reply counts and contents are reproducible
+with ``--seed``. Patch dates span a year. All patches contain the same small
+diff: this is a query-performance dataset, not a
+realistic simulation of database storage requirements or mail parsing.
+
+Generated projects have names such as ``perf-1``. The command refuses to reuse
+existing project names or generated submitter addresses. Use a different prefix
+for another dataset, or recreate the disposable database. It never deletes data.
+Batches are committed independently, so an interrupted run leaves partial data.
+Save hooks and signals are bypassed: events, tags, checks and cover letters are
+not generated.

patchwork/api/comment.py

@@ -11,6 +11,7 @@
 from rest_framework.serializers import HiddenField
 from rest_framework.serializers import SerializerMethodField
 
+from patchwork.api import utils
 from patchwork.api.base import BaseHyperlinkedModelSerializer
 from patchwork.api.base import NestedHyperlinkedIdentityField
 from patchwork.api.base import MultipleFieldLookupMixin
@@ -124,11 +125,16 @@ def get_object(self):
 
     def get_queryset(self):
         cover_id = self.kwargs['cover_id']
-        get_object_or_404(Cover, pk=cover_id)
+        get_object_or_404(Cover.objects.only('id'), pk=cover_id)
 
-        return CoverComment.objects.filter(cover=cover_id).select_related(
+        queryset = CoverComment.objects.filter(cover=cover_id).select_related(
             'submitter'
         )
+        if utils.has_version(self.request, '1.2'):
+            queryset = queryset.select_related('cover__project').defer(
+                'cover__content', 'cover__headers'
+            )
+        return queryset
 
 
 class PatchCommentSerializer(BaseCommentListSerializer):
@@ -164,11 +170,16 @@ def get_object(self):
 
     def get_queryset(self):
         patch_id = self.kwargs['patch_id']
-        get_object_or_404(Patch, id=patch_id)
+        get_object_or_404(Patch.objects.only('id'), id=patch_id)
 
-        return PatchComment.objects.filter(patch=patch_id).select_related(
+        queryset = PatchComment.objects.filter(patch=patch_id).select_related(
             'submitter'
         )
+        if utils.has_version(self.request, '1.2'):
+            queryset = queryset.select_related('patch__project').defer(
+                'patch__content', 'patch__diff', 'patch__headers'
+            )
+        return queryset
 
 
 class CoverCommentList(CoverCommentMixin, ListAPIView):

patchwork/api/embedded.py

@@ -11,6 +11,8 @@
 
 from collections import OrderedDict
 
+from django.utils.functional import cached_property
+
 from rest_framework.serializers import CharField
 from rest_framework.serializers import PrimaryKeyRelatedField
 from rest_framework.serializers import SerializerMethodField
@@ -49,8 +51,14 @@ def get_choices(self, cutoff=None):
             [(item.pk, self.display_value(item)) for item in queryset]
         )
 
+    @cached_property
+    def serializer(self):
+        # Fields are bound to a serializer for one request. Reuse the embedded
+        # serializer instead of rebuilding its fields for every related object.
+        return self._Serializer(context=self.context)
+
     def to_representation(self, data):
-        return self._Serializer(context=self.context).to_representation(data)
+        return self.serializer.to_representation(data)
 
 
 class MboxMixin(BaseHyperlinkedModelSerializer):

patchwork/api/patch.py

@@ -20,6 +20,7 @@
 from rest_framework import status
 
 from patchwork.api.base import BaseHyperlinkedModelSerializer
+from patchwork.api.base import LinkHeaderPagination
 from patchwork.api.base import PatchworkPermission
 from patchwork.api.embedded import PatchSerializer
 from patchwork.api.embedded import PersonSerializer
@@ -337,11 +338,27 @@ class Meta:
         extra_kwargs = PatchListSerializer.Meta.extra_kwargs
 
 
+class PatchListPagination(LinkHeaderPagination):
+    def paginate_queryset(self, queryset, request, view=None):
+        # Page over IDs before fetching joined rows. Otherwise a deep offset
+        # can make MySQL fetch and sort every full patch row in the project.
+        ids = super().paginate_queryset(
+            queryset.values_list('pk', flat=True), request, view=view
+        )
+        if ids is None:
+            return None
+        patches = {
+            patch.pk: patch for patch in queryset.filter(pk__in=ids).order_by()
+        }
+        return [patches[pk] for pk in ids if pk in patches]
+
+
 class PatchList(ListAPIView):
     """List patches."""
 
     permission_classes = (PatchworkPermission,)
     serializer_class = PatchListSerializer
+    pagination_class = PatchListPagination
     filter_class = filterset_class = PatchFilterSet
     search_fields = ('name',)
     ordering_fields = (

patchwork/api/series.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+from django.db.models import Prefetch
 from rest_framework.generics import ListAPIView
 from rest_framework.generics import RetrieveAPIView
 from rest_framework.serializers import (
@@ -17,6 +18,7 @@
 from patchwork.api.embedded import PatchSerializer
 from patchwork.api.embedded import PersonSerializer
 from patchwork.api.embedded import ProjectSerializer
+from patchwork.models import Patch
 from patchwork.models import Series
 
 
@@ -101,7 +103,12 @@ def get_queryset(self):
         return (
             Series.objects.all()
             .prefetch_related(
-                'patches__project',
+                Prefetch(
+                    'patches',
+                    queryset=Patch.objects.defer(
+                        'content', 'diff', 'headers'
+                    ).prefetch_related('project'),
+                ),
                 'cover_letter__project',
                 'dependencies',
                 'dependents',