diff --git a/bsmain/management/commands/prestart.py b/bsmain/management/commands/prestart.py index c83062e..f9ae97b 100644 --- a/bsmain/management/commands/prestart.py +++ b/bsmain/management/commands/prestart.py @@ -12,16 +12,27 @@ User = get_user_model() class Command(BaseCommand): help = "Pre-start command to run before the server starts." + def _create_superuser_if_needed(self): + if not os.getenv("CREATE_SUPERUSER", ""): + return + + if ":" not in os.getenv("CREATE_SUPERUSER"): + raise ValueError("CREATE_SUPERUSER should be in the format 'username:password'") + + username, password = os.getenv("CREATE_SUPERUSER").split(":") + + if User.objects.all().exists(): + print( + "Superuser not created: _any_ user(s) already exist(s). " + "CREATE_SUPERUSER only works for the initial user.") + + return + + User.objects.create_superuser(username=username, password=password) + print(f"Superuser created: {username}") + def handle(self, *args, **options): - if os.getenv("CREATE_SUPERUSER", ""): - if ":" not in os.getenv("CREATE_SUPERUSER"): - raise ValueError("CREATE_SUPERUSER should be in the format 'username:password'") - - username, password = os.getenv("CREATE_SUPERUSER").split(":") - - if not User.objects.filter(username=username).exists(): - User.objects.create_superuser(username=username, password=password) - print(f"Superuser created: {username}") + self._create_superuser_if_needed() # Similar considerations apply here as those which are documented in bugsink.views._phone_home(). # By putting this in prestart, we add one more location to the list of kick-off locations; with the added diff --git a/bugsink/conf_templates/docker.py.template b/bugsink/conf_templates/docker.py.template index 7551ae0..918b7f8 100644 --- a/bugsink/conf_templates/docker.py.template +++ b/bugsink/conf_templates/docker.py.template @@ -61,17 +61,28 @@ if os.getenv("DATABASE_URL"): DATABASE_URL = os.getenv("DATABASE_URL") parsed = urlparse(DATABASE_URL) - if parsed.scheme != "mysql": - raise ValueError("For DATABASE_URL, only mysql is supported.") + if parsed.scheme == "mysql": + DATABASES['default'] = { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': parsed.path.lstrip('/'), + "USER": parsed.username, + "PASSWORD": parsed.password, + "HOST": parsed.hostname, + "PORT": parsed.port or "3306", + } - DATABASES['default'] = { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': parsed.path.lstrip('/'), - "USER": parsed.username, - "PASSWORD": parsed.password, - "HOST": parsed.hostname, - "PORT": parsed.port or "3306", - } + elif parsed.scheme == ["postgres", "postgresql"]: + DATABASES['default'] = { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': parsed.path.lstrip('/'), + "USER": parsed.username, + "PASSWORD": parsed.password, + "HOST": parsed.hostname, + "PORT": parsed.port or "5432", + } + + else: + raise ValueError("For DATABASE_URL, only mysql and postgres are supported.") else: # similar to the default, but the fallback is /data/db.sqlite3; we also create the directory if it doesn't exist, diff --git a/events/retention.py b/events/retention.py index 7f49f65..5a8f73a 100644 --- a/events/retention.py +++ b/events/retention.py @@ -166,23 +166,26 @@ def filter_for_work(epoch_bounds_with_irrelevance, pairs, max_total_irrelevance) def eviction_target(max_event_count, stored_event_count): - # We always evict at least 500 events, or 5% of the max event count, whichever is less. The reason is: we want to - # avoid having to evict again immediately after we've just evicted. Because eviction is relatively expensive, we - # want to avoid doing it too often. For the completely reasonable quota of 10_000 events or more, this just means - # 500; for lower quota we take 5% to avoid evicting too much (at a small performance penalty). + # Calculate a target number of events to evict, which is a balancing act between 2 things: # - # One reason to pick no higher than 500 (and to delete_with_limit) is that we want to avoid blocking too long on a - # single eviction. (both on a single query, to avoid timeouts, but also on the eviciton as a whole, because it - # would block other threads/processes and trigger timeouts there). On the slow VPS we've observed deletions taking - # in the order of 1-4ms per event, so 500 would put us at 2s, which is still less than 50% of the timeout value. + # 1. large enough to avoid having to evict again immediately after we've just evicted (eviction is relatively + # expensive, so we want to avoid doing it too often) + # 2. not too large to avoid [a] throwing too much data away unnecessarily and [b] avoid blocking too + # long on a single eviction (both on a single query, to avoid timeouts, but also on the eviction as a whole, + # because it would block other threads/processes and trigger timeouts there). + # + # Inputs into the calculation are (see test_eviction_target to understand the min/maxing): + # + # * the constant value of 500 + # * the over-targetness + # * and a percentage of 5% of the target + # + # On the slow VPS we've observed deletions taking in the order of 1-4ms per event, so 500 would put us at 2s, which + # is still less than 50% of the timeout value. # # Although eviction triggers "a lot" of queries, "a lot" is in the order of 25, so after amortization this is far # less than 1 query extra per event (as a result of the actual eviction, checking for the need to evict is a - # different story). 5% seems close enough to the limit to stem the "why was so much deleted" questions. - # - # We also evict at least the number of events that we are over the max event count; this takes care of 2 scenarios: - # * a quota that has been adjusted downwards (we want to get rid of the excess); - # * quota so ridiculously low that 5% rounds down to 0, in those cases at least delete 1 + # different story). 5% seems small enough to stem the "why was so much deleted" questions. return min( max( int(max_event_count * 0.05), diff --git a/issues/templates/issues/issue_list.html b/issues/templates/issues/issue_list.html index 68173f3..68eb6e3 100644 --- a/issues/templates/issues/issue_list.html +++ b/issues/templates/issues/issue_list.html @@ -146,7 +146,11 @@   {% endif %}{{ issue.title|truncatechars:100 }}
from {{ issue.first_seen|date:"j M G:i T" }} | last {{ issue.last_seen|date:"j M G:i T" }} | with {{ issue.digested_event_count|intcomma }} events - {% if issue.transaction %}| {{ issue.transaction }} {% endif %} + {% if issue.digested_event_count != issue.stored_event_count %} + ({{ issue.stored_event_count|intcomma }} av{#ilable#}) + {% endif %} + + {% if issue.transaction %}| {{ issue.transaction }} {% endif %}
diff --git a/issues/views.py b/issues/views.py index 66b0d2a..9adac2c 100644 --- a/issues/views.py +++ b/issues/views.py @@ -503,7 +503,7 @@ def issue_event_details(request, issue, event_pk=None, digest_order=None, nav=No ] logentry_info = [] - if parsed_data.get("logger") or parsed_data.get("logentry", {}).get("message") or parsed_data.get("message"): + if parsed_data.get("logger") or parsed_data.get("logentry") or parsed_data.get("message"): if "level" in parsed_data: # Sentry gives "level" a front row seat in the UI; but we don't: in an Error Tracker, the default is just # "error" (and we don't want to pollute the UI with this info). Sentry's documentation is also very sparse @@ -519,16 +519,20 @@ def issue_event_details(request, issue, event_pk=None, digest_order=None, nav=No # past. see https://github.com/bugsink/bugsink/issues/43 logentry_key = "logentry" if "logentry" in parsed_data else "message" - if parsed_data.get(logentry_key, {}).get("message"): - logentry_info.append(("message", parsed_data[logentry_key]["message"])) + if isinstance(parsed_data.get(logentry_key), dict): + if parsed_data.get(logentry_key, {}).get("message"): + logentry_info.append(("message", parsed_data[logentry_key]["message"])) - params = parsed_data.get(logentry_key, {}).get("params", {}) - if isinstance(params, list): - for param_i, param_v in enumerate(params): - logentry_info.append(("#%s" % param_i, param_v)) - elif isinstance(params, dict): - for param_k, param_v in params.items(): - logentry_info.append((param_k, param_v)) + params = parsed_data.get(logentry_key, {}).get("params", {}) + if isinstance(params, list): + for param_i, param_v in enumerate(params): + logentry_info.append(("#%s" % param_i, param_v)) + elif isinstance(params, dict): + for param_k, param_v in params.items(): + logentry_info.append((param_k, param_v)) + + elif isinstance(parsed_data.get(logentry_key), str): + logentry_info.append(("message", parsed_data[logentry_key])) key_info += [ ("grouping key", event.grouping.grouping_key),