Add first version of 'phone home'

bugsink/conf_templates/docker.py.template

@@ -11,6 +11,8 @@ _MEBIBYTE = 1024 * _KIBIBYTE
 _PORT = os.environ.get("PORT", "8000")
 
 
+IS_DOCKER = True
+
 DEBUG = os.getenv("DEBUG", "False").lower() in ("true", "1", "yes")
 DEBUG_CSRF = "USE_DEBUG" if os.getenv("DEBUG_CSRF") == "USE_DEBUG" else os.getenv("DEBUG_CSRF", "False").lower() in ("true", "1", "yes")
 

bugsink/settings/default.py

@@ -23,6 +23,10 @@ elif [s.endswith("gunicorn") for s in sys.argv[:1]] == [True]:
 else:
     I_AM_RUNNING = "OTHER"
 
+
+# Used for reporting / debugging purposes. The default docker conf template overrides this accordingly.
+IS_DOCKER = False
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
 
@@ -55,6 +59,7 @@ INSTALLED_APPS = [
 ]
 
 BUGSINK_APPS = [
+    'phonehome',
     'users',
     'theme',
     'snappea',

bugsink/views.py

@@ -14,8 +14,29 @@ from bugsink.version import __version__
 from bugsink.decorators import login_exempt
 from bugsink.app_settings import get_settings as get_bugsink_settings
 
+from phonehome.tasks import send_if_due
+
+
+def _phone_home():
+    # I need a way to cron-like run tasks that works for the setup with and without snappea. With snappea it's straight-
+    # forward (though not part of snappea _yet_). Without snappea, you'd need _some_ location to do a "poor man's cron"
+    # check. Server-start would be the first thing to consider, but how to do this across gunicorn, debugserver, and
+    # possibly even non-standard (for Bugsink) wsgi servers? Better go the "just pick some request to do the check"
+    # route. I've picked "home", because [a] it's assumed to be somewhat regularly visited [b] there's no transaction
+    # logic in it, which leaves space for transaction-logic in the phone-home task itself and [c] some alternatives are
+    # a no-go (ingestion: on a tight budget; login: not visited when a long-lived session is active).
+    #
+    # having chosen the solution for the non-snappea case, I got the crazy idea of using it for the snappea case too,
+    # i.e. just put a .delay() here and let the config choose. Not so crazy though, because [a] saves us from new
+    # features in snappea, [b] we introduce a certain symmetry of measurement between the 2 setups, i.e. the choice of
+    # lazyness does not influence counting and [c] do I really want to get pings for sites where nobody visits home()?
+
+    send_if_due.delay()  # _phone_home() wrapper serves as a place for the comment above
+
 
 def home(request):
+    _phone_home()
+
     if request.user.project_set.filter(projectmembership__accepted=True).distinct().count() == 1:
         # if the user has exactly one project, we redirect them to that project
         project = request.user.project_set.get()

phonehome/admin.py

@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+from .models import OutboundMessage
+
+
+@admin.register(OutboundMessage)
+class OutboundMessageAdmin(admin.ModelAdmin):
+    list_display = ("attempted_at", "sent_at")
+    readonly_fields = ("attempted_at", "sent_at", "message")

phonehome/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class PhonehomeConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "phonehome"

phonehome/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.16 on 2024-11-07 14:03
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Installation",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+            ],
+        ),
+    ]

phonehome/migrations/0002_create_installation_id.py

@@ -0,0 +1,17 @@
+from django.db import migrations
+
+
+def create_installation_id(apps, schema_editor):
+    Installation = apps.get_model("phonehome", "Installation")
+    Installation.objects.create()  # id is implied (it's a uuid)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("phonehome", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(create_installation_id),
+    ]

phonehome/migrations/0003_outboundmessage.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.16 on 2024-11-07 15:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("phonehome", "0002_create_installation_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="OutboundMessage",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("attempted_at", models.DateTimeField(auto_now_add=True)),
+                ("sent_at", models.DateTimeField(null=True)),
+                ("message", models.TextField()),
+            ],
+        ),
+    ]

phonehome/models.py

@@ -0,0 +1,16 @@
+import uuid
+
+from django.db import models
+
+
+class Installation(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+
+
+class OutboundMessage(models.Model):
+    attempted_at = models.DateTimeField(auto_now_add=True)
+    sent_at = models.DateTimeField(null=True)
+    message = models.TextField()
+
+    def __str__(self):
+        return f"OutboundMessage(attempted_at={self.attempted_at}, sent_at={self.sent_at})"

phonehome/tasks.py

@@ -0,0 +1,140 @@
+import logging
+import requests
+import platform
+import json
+
+from django.conf import settings
+from django.utils import timezone
+from django.contrib.auth import get_user_model
+
+from bugsink.transaction import durable_atomic, immediate_atomic
+from bugsink.version import __version__
+from bugsink.app_settings import get_settings
+
+from snappea.decorators import shared_task
+from snappea.settings import get_settings as get_snappea_settings
+
+from projects.models import Project
+from teams.models import Team
+
+from .models import Installation, OutboundMessage
+
+
+logger = logging.getLogger("bugsink.phonehome")
+
+
+User = get_user_model()
+
+INTERVAL = 60 * 60  # phone-home once an hour
+
+
+@shared_task
+def send_if_due():
+    # considered: not sending if DEBUG=True. But why? Expectation is: I'm the sole user of that setting. Better send
+    # also, to keep symmetry, and indirectly check whether my own phone-home still works.
+
+    # Note on attempted_at / sent_at distinction: attempted_at is when we first tried to send the message, and sent_at
+    # is when we actually sent it. This allows us to try only once an hour, as well as track whether sending succeeded.
+    not_due_qs = OutboundMessage.objects.filter(
+        attempted_at__gte=timezone.now() - timezone.timedelta(seconds=INTERVAL)).exists()
+
+    with durable_atomic():
+        # We check twice to see if there's any work: once in a simple read-only transaction (which doesn't block), and
+        # (below) then again in the durable transaction (which can block) to actually ensure no 2 processes
+        # simultaneously start the process of sending the message. This incurs the cost of 2 queries in the (less
+        # common) case where the work is due, but it avoids the cost of blocking the whole DB for the (more common) case
+        # where the work is not due.
+        if not_due_qs:
+            return
+
+    with immediate_atomic():
+        if not_due_qs:
+            return
+
+        # TODO: clean up old messages (perhaps older than 1 week)
+
+        message = OutboundMessage.objects.create(
+            # attempted_at is auto_now_add; will be filled in automatically
+            message=json.dumps(_make_message_body()),
+        )
+
+    if not _send_message(message):
+        return
+
+    with immediate_atomic():
+        # a fresh transaction avoids hogging the DB while doing network I/O
+        message.sent_at = timezone.now()
+        message.save()
+
+
+def _make_message_body():
+    return {
+        "installation_id": str(Installation.objects.get().id),
+        "version": __version__,
+        "python_version": platform.python_version(),
+
+        "settings": {
+            # Settings that tell us "who you are", and are relevant in the context of licensing.
+            "BASE_URL": get_settings().BASE_URL,
+            "SITE_TITLE": get_settings().SITE_TITLE,
+            "DEFAULT_FROM_EMAIL": settings.DEFAULT_FROM_EMAIL,
+
+            # we don't have these settings yet.
+            # LICENSE_KEY
+            # LICENSE_NAME
+            # LICENSE_EMAIL
+            # LICENSE_USERS
+            # LICENSE_EXPIRY
+            # LICENSE_TYPE
+
+            # Settings that tell us how production-like your usage is.
+            "SINGLE_USER": get_settings().SINGLE_USER,
+            "SINGLE_TEAM": get_settings().SINGLE_TEAM,
+            # As it stands, the 2 settings below are not used to determine production-like-ness; left here for reference
+            # "USER_REGISTRATION": get_settings().USER_REGISTRATION,
+            # "TEAM_CREATION": get_settings().TEAM_CREATION,
+
+            # Settings that tell us a bit about how Bugsink is actually deployed. Useful for support.
+            "TASK_ALWAYS_EAGER": get_snappea_settings().TASK_ALWAYS_EAGER,
+            "DIGEST_IMMEDIATELY": get_settings().DIGEST_IMMEDIATELY,
+            "IS_DOCKER": settings.IS_DOCKER,
+            "DATABASE_ENGINE": settings.DATABASES["default"]["ENGINE"],
+        },
+
+        "usage": {
+            "user_count": User.objects.count(),
+            "active_user_count": User.objects.filter(is_active=True).count(),
+            "project_count": Project.objects.count(),
+            "team_count": Team.objects.count(),
+
+            # event-counts [per some interval (e.g. 24 hours)] is a possible future enhancement. If that turns out to be
+            # expensive, one thing to consider is pulling _make_message_body() out of the immediate_atomic() block.
+        },
+    }
+
+
+def _send_message(message):
+    url = "https://www.bugsink.com/phonehome/v1/"
+
+    def post(timeout):
+        response = requests.post(url, json=json.loads(message.message), timeout=timeout)
+        response.raise_for_status()
+
+    if get_snappea_settings().TASK_ALWAYS_EAGER:
+        # Doing a switch on "am I running this in snappea or not" so deep in the code somehow feels wrong, but "if it
+        # looks stupid but works, it ain't stupid" I guess. The point is: when doing http requests inline I want them to
+        # be fast enough not to bother your flow and never fail loudly; in the async case you have a bit more time, and
+        # failing loudly (which will be picked up, or not, in the usual ways) is actually a feature.
+
+        try:
+            # 1s max wait time each hour is deemed "OK"; (counterpoint: also would be on your _first_ visit)
+            post(timeout=1)
+        except requests.RequestException as e:
+            # This is a "soft" failure; we don't want to raise an exception, but we do want to log it.
+            logger.exception("Failed to send phonehome message: %s", e)
+            return False
+
+    else:
+        post(timeout=5)
+
+    return True

phonehome/tests.py

@@ -0,0 +1 @@
+# from django.test import TestCase

phonehome/views.py

@@ -0,0 +1 @@
+# from django.shortcuts import render

requirements.txt

@@ -10,7 +10,7 @@ inotify_simple
 brotli
 python-dateutil
 whitenoise
-requests  # for sentry-sdk-extensions, which is loaded in non-dev setup too
+requests
 monofy==1.1.*
 user_agents
 fastjsonschema