Merge branch 'main' into tag-search

bsmain/management/commands/prestart.py

@@ -12,16 +12,27 @@ User = get_user_model()
 class Command(BaseCommand):
     help = "Pre-start command to run before the server starts."
 
+    def _create_superuser_if_needed(self):
+        if not os.getenv("CREATE_SUPERUSER", ""):
+            return
+
+        if ":" not in os.getenv("CREATE_SUPERUSER"):
+            raise ValueError("CREATE_SUPERUSER should be in the format 'username:password'")
+
+        username, password = os.getenv("CREATE_SUPERUSER").split(":")
+
+        if User.objects.all().exists():
+            print(
+                "Superuser not created: _any_ user(s) already exist(s). "
+                "CREATE_SUPERUSER only works for the initial user.")
+
+            return
+
+        User.objects.create_superuser(username=username, password=password)
+        print(f"Superuser created: {username}")
+
     def handle(self, *args, **options):
-        if os.getenv("CREATE_SUPERUSER", ""):
-            if ":" not in os.getenv("CREATE_SUPERUSER"):
-                raise ValueError("CREATE_SUPERUSER should be in the format 'username:password'")
-
-            username, password = os.getenv("CREATE_SUPERUSER").split(":")
-
-            if not User.objects.filter(username=username).exists():
-                User.objects.create_superuser(username=username, password=password)
-                print(f"Superuser created: {username}")
+        self._create_superuser_if_needed()
 
         # Similar considerations apply here as those which are documented in bugsink.views._phone_home().
         # By putting this in prestart, we add one more location to the list of kick-off locations; with the added

bugsink/conf_templates/docker.py.template

@@ -61,17 +61,28 @@ if os.getenv("DATABASE_URL"):
     DATABASE_URL = os.getenv("DATABASE_URL")
     parsed = urlparse(DATABASE_URL)
 
-    if parsed.scheme != "mysql":
-        raise ValueError("For DATABASE_URL, only mysql is supported.")
+    if parsed.scheme == "mysql":
+        DATABASES['default'] = {
+            'ENGINE': 'django.db.backends.mysql',
+            'NAME': parsed.path.lstrip('/'),
+            "USER": parsed.username,
+            "PASSWORD": parsed.password,
+            "HOST": parsed.hostname,
+            "PORT": parsed.port or "3306",
+        }
 
-    DATABASES['default'] = {
-        'ENGINE': 'django.db.backends.mysql',
-        'NAME': parsed.path.lstrip('/'),
-        "USER": parsed.username,
-        "PASSWORD": parsed.password,
-        "HOST": parsed.hostname,
-        "PORT": parsed.port or "3306",
-    }
+    elif parsed.scheme == ["postgres", "postgresql"]:
+        DATABASES['default'] = {
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': parsed.path.lstrip('/'),
+            "USER": parsed.username,
+            "PASSWORD": parsed.password,
+            "HOST": parsed.hostname,
+            "PORT": parsed.port or "5432",
+        }
+
+    else:
+        raise ValueError("For DATABASE_URL, only mysql and postgres are supported.")
 
 else:
     # similar to the default, but the fallback is /data/db.sqlite3; we also create the directory if it doesn't exist,

events/retention.py

@@ -166,23 +166,26 @@ def filter_for_work(epoch_bounds_with_irrelevance, pairs, max_total_irrelevance)
 
 
 def eviction_target(max_event_count, stored_event_count):
-    # We always evict at least 500 events, or 5% of the max event count, whichever is less. The reason is: we want to
-    # avoid having to evict again immediately after we've just evicted. Because eviction is relatively expensive, we
-    # want to avoid doing it too often. For the completely reasonable quota of 10_000 events or more, this just means
-    # 500; for lower quota we take 5% to avoid evicting too much (at a small performance penalty).
+    # Calculate a target number of events to evict, which is a balancing act between 2 things:
     #
-    # One reason to pick no higher than 500 (and to delete_with_limit) is that we want to avoid blocking too long on a
-    # single eviction. (both on a single query, to avoid timeouts, but also on the eviciton as a whole, because it
-    # would block other threads/processes and trigger timeouts there). On the slow VPS we've observed deletions taking
-    # in the order of 1-4ms per event, so 500 would put us at 2s, which is still less than 50% of the timeout value.
+    # 1. large enough to avoid having to evict again immediately after we've just evicted (eviction is relatively
+    #    expensive, so we want to avoid doing it too often)
+    # 2. not too large to avoid [a] throwing too much data away unnecessarily and [b] avoid blocking too
+    #    long on a single eviction (both on a single query, to avoid timeouts, but also on the eviction as a whole,
+    #    because it would block other threads/processes and trigger timeouts there).
+    #
+    # Inputs into the calculation are (see test_eviction_target to understand the min/maxing):
+    #
+    # * the constant value of 500
+    # * the over-targetness
+    # * and a percentage of 5% of the target
+    #
+    # On the slow VPS we've observed deletions taking in the order of 1-4ms per event, so 500 would put us at 2s, which
+    # is still less than 50% of the timeout value.
     #
     # Although eviction triggers "a lot" of queries, "a lot" is in the order of 25, so after amortization this is far
     # less than 1 query extra per event (as a result of the actual eviction, checking for the need to evict is a
-    # different story). 5% seems close enough to the limit to stem the "why was so much deleted" questions.
-    #
-    # We also evict at least the number of events that we are over the max event count; this takes care of 2 scenarios:
-    # * a quota that has been adjusted downwards (we want to get rid of the excess);
-    # * quota so ridiculously low that 5% rounds down to 0, in those cases at least delete 1
+    # different story). 5% seems small enough to stem the "why was so much deleted" questions.
     return min(
                max(
                    int(max_event_count * 0.05),

issues/templates/issues/issue_list.html

@@ -146,7 +146,11 @@
 </svg>&nbsp;&nbsp;{% endif %}{{ issue.title|truncatechars:100 }}</a>
                 </div>
                 <div class="text-sm">from <b>{{ issue.first_seen|date:"j M G:i T" }}</b> | last <b>{{ issue.last_seen|date:"j M G:i T" }}</b> | with <b>{{ issue.digested_event_count|intcomma }}</b> events
-                    {% if issue.transaction %}| <span class="font-bold">{{ issue.transaction }} </span>{% endif %}
+                {% if issue.digested_event_count != issue.stored_event_count %}
+                    <span class="text-xs">({{ issue.stored_event_count|intcomma }}&thinsp;av{#ilable#})<span>
+                {% endif %}
+
+                {% if issue.transaction %}| <span class="font-bold">{{ issue.transaction }} </span>{% endif %}
                 </div>
             </td>
 

issues/views.py

@@ -503,7 +503,7 @@ def issue_event_details(request, issue, event_pk=None, digest_order=None, nav=No
     ]
 
     logentry_info = []
-    if parsed_data.get("logger") or parsed_data.get("logentry", {}).get("message") or parsed_data.get("message"):
+    if parsed_data.get("logger") or parsed_data.get("logentry") or parsed_data.get("message"):
         if "level" in parsed_data:
             # Sentry gives "level" a front row seat in the UI; but we don't: in an Error Tracker, the default is just
             # "error" (and we don't want to pollute the UI with this info). Sentry's documentation is also very sparse
@@ -519,16 +519,20 @@ def issue_event_details(request, issue, event_pk=None, digest_order=None, nav=No
         # past. see https://github.com/bugsink/bugsink/issues/43
         logentry_key = "logentry" if "logentry" in parsed_data else "message"
 
-        if parsed_data.get(logentry_key, {}).get("message"):
-            logentry_info.append(("message", parsed_data[logentry_key]["message"]))
+        if isinstance(parsed_data.get(logentry_key), dict):
+            if parsed_data.get(logentry_key, {}).get("message"):
+                logentry_info.append(("message", parsed_data[logentry_key]["message"]))
 
-        params = parsed_data.get(logentry_key, {}).get("params", {})
-        if isinstance(params, list):
-            for param_i, param_v in enumerate(params):
-                logentry_info.append(("#%s" % param_i, param_v))
-        elif isinstance(params, dict):
-            for param_k, param_v in params.items():
-                logentry_info.append((param_k, param_v))
+            params = parsed_data.get(logentry_key, {}).get("params", {})
+            if isinstance(params, list):
+                for param_i, param_v in enumerate(params):
+                    logentry_info.append(("#%s" % param_i, param_v))
+            elif isinstance(params, dict):
+                for param_k, param_v in params.items():
+                    logentry_info.append((param_k, param_v))
+
+        elif isinstance(parsed_data.get(logentry_key), str):
+            logentry_info.append(("message", parsed_data[logentry_key]))
 
     key_info += [
         ("grouping key", event.grouping.grouping_key),