diff --git a/bugsink/settings/development.py b/bugsink/settings/development.py index 1fe7bef..a104532 100644 --- a/bugsink/settings/development.py +++ b/bugsink/settings/development.py @@ -94,8 +94,11 @@ BUGSINK = { # set MAX_EVENTS* very high to be able to do serious performance testing (which I do often in my dev environment) "MAX_EVENTS_PER_PROJECT_PER_5_MINUTES": 1_000_000, "MAX_EVENTS_PER_PROJECT_PER_HOUR": 50_000_000, +} - "EVENT_STORAGES": { + +if not I_AM_RUNNING == "TEST": + BUGSINK["EVENT_STORAGES"] = { "local_flat_files": { "STORAGE": "events.storage.FileEventStorage", "OPTIONS": { @@ -104,7 +107,6 @@ BUGSINK = { "USE_FOR_WRITE": True, }, } -} # performance development settings: show inline in the console, with a nice little arrow diff --git a/events/storage.py b/events/storage.py index 2ba1d98..6ffc3b0 100644 --- a/events/storage.py +++ b/events/storage.py @@ -1,5 +1,6 @@ import contextlib import os.path +from pathlib import Path class EventStorage(object): @@ -46,6 +47,11 @@ class FileEventStorage(EventStorage): # strict about what we allow; we further imply "text mode" and "utf-8 encoding" given the JSON context. raise ValueError("EventStorage.open() mode must be 'r' or 'w'") + if mode == 'w' and not os.path.exists(self.basepath): + # only if we're writing does this make sense (when reading, a newly created directoy won't have files in it, + # and fail in the next step) + Path(self.basepath).mkdir(parents=True, exist_ok=True) + # We open with utf-8 encoding explicitly to pre-empt the future of pep-0686 (it's also the only thing that makes # sense in the context of JSON) with open(self._event_path(event_id), mode, encoding="utf-8") as f: diff --git a/events/storage_registry.py b/events/storage_registry.py index c60c464..7443ba0 100644 --- a/events/storage_registry.py +++ b/events/storage_registry.py @@ -1,6 +1,7 @@ +from contextlib import contextmanager import importlib -from bugsink.app_settings import get_settings +from bugsink.app_settings import get_settings, override_settings _storages = None @@ -48,3 +49,23 @@ def _resolve(name, conf): clazz = getattr(module, class_name) return clazz(name, **conf.get("OPTIONS", {})) + + +@contextmanager +def override_event_storages(storage_conf): + """ + Temporarily override the event storage for the duration of the context (for tests). + """ + global _storages + global _write_storage + + _storages = None + _write_storage = None + + try: + with override_settings(EVENT_STORAGES=storage_conf): + yield + + finally: + _storages = None + _write_storage = None diff --git a/ingest/tests.py b/ingest/tests.py index 7aea1f3..eaa0819 100644 --- a/ingest/tests.py +++ b/ingest/tests.py @@ -4,6 +4,7 @@ import json import io import uuid import time +import tempfile import datetime from unittest.mock import patch @@ -18,6 +19,8 @@ from django.core.exceptions import ValidationError from bugsink.test_utils import TransactionTestCase25251 as TransactionTestCase from projects.models import Project from events.factories import create_event_data, create_event +from events.retention import evict_for_max_events +from events.storage_registry import override_event_storages from issues.factories import get_or_create_issue from issues.models import IssueStateManager, Issue, TurningPoint, TurningPointKind from bugsink.app_settings import override_settings @@ -397,6 +400,29 @@ class IngestViewTestCase(TransactionTestCase): with override_settings(DIGEST_IMMEDIATELY=False): self.test_envelope_endpoint() + @tag("samples") + def test_filestore(self): + # quick & dirty way to test the filestore; in absence of a proper test for it, we just run a more-or-less + # integration test with the FileEventStorage activated. This will at least show the absence of the most obvious + # errors. We then run + with tempfile.TemporaryDirectory() as tempdir: + with override_event_storages({"local_flat_files": { + "STORAGE": "events.storage.FileEventStorage", + "OPTIONS": { + "basepath": tempdir, + }, + "USE_FOR_WRITE": True, + }, + }): + self.test_envelope_endpoint() + self.assertEqual(len(os.listdir(tempdir)), 2) # test_envelope_endpoint creates 2 events + + project = Project.objects.get(name="test") + project.retention_max_event_count = 1 + evict_for_max_events(project, timezone.now(), stored_event_count=2) + + self.assertEqual(len(os.listdir(tempdir)), 1) # we set the max to 1, so one should remain + @override_settings(MAX_EVENTS_PER_PROJECT_PER_5_MINUTES=0) @patch("ingest.views.check_for_thresholds") def test_count_project_periods_and_act_on_it_zero(self, patched_check_for_thresholds):