From 9a96ab767a8833137b3010391b8e8958afdf3444 Mon Sep 17 00:00:00 2001 From: Klaas van Schelven Date: Wed, 26 Jun 2024 09:38:34 +0200 Subject: [PATCH] retention insights: don't ignore never_evict=True --- events/retention.py | 8 ++++---- events/retention_insight.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/events/retention.py b/events/retention.py index ef56ce0..bcfe7ff 100644 --- a/events/retention.py +++ b/events/retention.py @@ -104,10 +104,10 @@ def get_age_for_irrelevance(age_based_irrelevance): return pow(2, age_based_irrelevance) - 1 -def get_epoch_bounds_with_irrelevance(project, current_timestamp): +def get_epoch_bounds_with_irrelevance(project, current_timestamp, qs_kwargs={"never_evict": False}): from .models import Event - oldest = Event.objects.filter(project=project, never_evict=False).aggregate(val=Min('server_side_timestamp'))['val'] + oldest = Event.objects.filter(project=project, **qs_kwargs).aggregate(val=Min('server_side_timestamp'))['val'] first_epoch = get_epoch(oldest) if oldest is not None else get_epoch(current_timestamp) current_epoch = get_epoch(current_timestamp) @@ -121,7 +121,7 @@ def get_epoch_bounds_with_irrelevance(project, current_timestamp): return [((lb, ub), age_based_irrelevance) for age_based_irrelevance, (ub, lb) in enumerate(swapped_bounds)] -def get_irrelevance_pairs(project, epoch_bounds_with_irrelevance): +def get_irrelevance_pairs(project, epoch_bounds_with_irrelevance, qs_kwargs={"never_evict": False}): """tuples of `age_based_irrelevance` and, per associated period, the max observed (evictable) event irrelevance""" from .models import Event @@ -129,7 +129,7 @@ def get_irrelevance_pairs(project, epoch_bounds_with_irrelevance): d = Event.objects.filter( get_epoch_bounds(lower_bound, upper_bound), project=project, - never_evict=False, + **qs_kwargs, ).aggregate(Max('irrelevance_for_retention')) max_event_irrelevance = d["irrelevance_for_retention__max"] or 0 diff --git a/events/retention_insight.py b/events/retention_insight.py index 2b7c3a1..51fa044 100644 --- a/events/retention_insight.py +++ b/events/retention_insight.py @@ -7,8 +7,11 @@ from .models import Event def retention_insight_values(project): timestamp = datetime.now(tz=timezone.utc) - epoch_bounds_with_irrelevance = get_epoch_bounds_with_irrelevance(project, timestamp) - pairs = list(get_irrelevance_pairs(project, epoch_bounds_with_irrelevance)) + # we call with qs_kwargs={} here, because we want to see what actually exists including any never_evict=True events. + # (the default (used during actual eviction) is to ignore such events when creating bounds and pairs, because they + # cannot be deleted anyway) + epoch_bounds_with_irrelevance = get_epoch_bounds_with_irrelevance(project, timestamp, qs_kwargs={}) + pairs = list(get_irrelevance_pairs(project, epoch_bounds_with_irrelevance, qs_kwargs={})) print("epoch_bounds_with_irrelevance") for x in epoch_bounds_with_irrelevance: