Count view: async slow counts

when you count, it's usually because there are many, so this
extra complication is probaly going to be required

bsmain/migrations/0002_cachedmodelcount.py

@@ -0,0 +1,32 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bsmain", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CachedModelCount",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("app_label", models.CharField(max_length=255)),
+                ("model_name", models.CharField(max_length=255)),
+                ("count", models.PositiveIntegerField()),
+                ("last_updated", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "unique_together": {("app_label", "model_name")},
+            },
+        ),
+    ]

bsmain/models.py

@@ -18,3 +18,15 @@ class AuthToken(models.Model):
 
     def __str__(self):
         return f"AuthToken(token={self.token})"
+
+
+class CachedModelCount(models.Model):
+    """Model to cache the count of a specific model."""
+
+    app_label = models.CharField(max_length=255)
+    model_name = models.CharField(max_length=255)
+    count = models.PositiveIntegerField(null=False, blank=False)
+    last_updated = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        unique_together = ('app_label', 'model_name')

bsmain/tasks.py

@@ -0,0 +1,26 @@
+from django.apps import apps
+from django.utils import timezone
+
+from snappea.decorators import shared_task
+
+from bugsink.transaction import durable_atomic, immediate_atomic
+from .models import CachedModelCount
+
+
+@shared_task
+def count_model(app_label, model_name):
+    ModelClass = apps.get_model(app_label, model_name)
+
+    # separate transaction for the expensive counting
+    with durable_atomic():
+        count = ModelClass.objects.count()
+
+    with immediate_atomic():
+        CachedModelCount.objects.update_or_create(
+            app_label=app_label,
+            model_name=model_name,
+            defaults={
+                'count': count,
+                'last_updated': timezone.now(),
+            },
+        )

bugsink/views.py

@@ -1,4 +1,5 @@
 import sys
+from collections import namedtuple
 
 from django import apps
 from django.http import HttpResponseServerError, HttpResponseBadRequest, HttpResponseRedirect
@@ -10,6 +11,7 @@ from django.views.defaults import (
 )
 from django.shortcuts import redirect
 from django.conf import settings
+from django.db.utils import OperationalError
 
 from django.template.defaultfilters import filesizeformat
 from django.views.decorators.http import require_GET
@@ -26,11 +28,17 @@ from bugsink.version import __version__
 from bugsink.decorators import login_exempt
 from bugsink.app_settings import get_settings as get_bugsink_settings
 from bugsink.decorators import atomic_for_request_method
+from bugsink.timed_sqlite_backend.base import different_runtime_limit
 
 from phonehome.tasks import send_if_due
 from phonehome.models import Installation
 
 from ingest.views import BaseIngestAPIView
+from bsmain.models import CachedModelCount
+from bsmain.tasks import count_model
+
+
+AnnotatedCount = namedtuple("AnnotatedCount", ["count", "timestamp"])
 
 
 def cors_for_api_view(view):
@@ -149,6 +157,7 @@ def settings_view(request):
 
 
 @user_passes_test(lambda u: u.is_superuser)
+@atomic_for_request_method  # get a consistent view (barring cached, which are marked as such)
 def counts(request):
     interesting_apps = [
         # "admin",
@@ -170,11 +179,31 @@ def counts(request):
     ]
 
     counts = {}
-    for app_label in interesting_apps:
-        counts[app_label] = {}
-        app_config = apps.apps.get_app_config(app_label)
-        for model in app_config.get_models():
-            counts[app_label][model.__name__] = model.objects.count()
+
+    # when you have some 7 - 10 models (tag-related, events, issues) that can have many instances, spending max .3 on
+    # each before giving up would seem reasonable to stay below th 5s limit; the rest is via the caches anyway.
+    with different_runtime_limit(0.3):
+        for app_label in interesting_apps:
+            counts[app_label] = {}
+            app_config = apps.apps.get_app_config(app_label)
+            for model in app_config.get_models():
+                if model.__name__ == "CachedModelCount":
+                    continue  # skip the CachedModelCount model itself
+
+                try:
+                    counts[app_label][model.__name__] = AnnotatedCount(model.objects.count(), None)
+                except OperationalError as e:
+                    if e.args[0] != "interrupted":
+                        raise
+
+                    # too many to quickly count; schedule a recount, and display any we might have.
+                    count_model.delay(app_label, model.__name__)
+
+                    try:
+                        cached = CachedModelCount.objects.get(app_label=app_label, model_name=model.__name__)
+                        counts[app_label][model.__name__] = AnnotatedCount(cached.count, cached.last_updated)
+                    except CachedModelCount.DoesNotExist:
+                        counts[app_label][model.__name__] = AnnotatedCount("too many; count scheduled in snappea", None)
 
     return render(request, "bugsink/counts.html", {
         "counts": counts,

templates/bugsink/counts.html

@@ -17,11 +17,12 @@
     <h1 class="text-2xl font-bold mt-4">{{ app_name|capfirst }}</h1>
 
     <div class="mb-6">
-        {% for key, value in model_counts|items %}
+        {% for key, annotated_count in model_counts|items %}
             <div class="flex {% if forloop.first %}border-slate-300 border-t-2{% endif %}">
                 <div class="w-1/6 {% if not forloop.last %}border-b-2 border-dotted border-slate-300{% endif %}">{{ key }}</div>
-                <div class="w-1/6 {% if not forloop.last %} border-b-2 border-dotted border-slate-300{% endif %} font-mono text-right">{{ value|intcomma }}</div>
-                <div class="w-2/3 {% if not forloop.last %} border-b-2 border-dotted border-slate-300{% endif %}">&nbsp;</div>
+                <div class="w-1/6 {% if not forloop.last %} border-b-2 border-dotted border-slate-300{% endif %} text-right">{{ annotated_count.count|intcomma }}</div>
+                <div class="w-1/6 {% if not forloop.last %} border-b-2 border-dotted border-slate-300{% endif %} pl-4 text-slate-500">{% if annotated_count.timestamp %}cached {{ annotated_count.timestamp|date:"G:i T" }}{% else %}&nbsp;{% endif %}</div>
+                <div class="w-1/2 {% if not forloop.last %} border-b-2 border-dotted border-slate-300{% endif %}">&nbsp;</div>
 
             </div>
         {% endfor %}