FileEventStorage: create dir on-demand; fix and add tests

bugsink/settings/development.py

@@ -94,8 +94,11 @@ BUGSINK = {
     # set MAX_EVENTS* very high to be able to do serious performance testing (which I do often in my dev environment)
     "MAX_EVENTS_PER_PROJECT_PER_5_MINUTES": 1_000_000,
     "MAX_EVENTS_PER_PROJECT_PER_HOUR": 50_000_000,
+}
 
-    "EVENT_STORAGES": {
+
+if not I_AM_RUNNING == "TEST":
+    BUGSINK["EVENT_STORAGES"] = {
         "local_flat_files": {
             "STORAGE": "events.storage.FileEventStorage",
             "OPTIONS": {
@@ -104,7 +107,6 @@ BUGSINK = {
             "USE_FOR_WRITE": True,
         },
     }
-}
 
 
 # performance development settings: show inline in the console, with a nice little arrow

events/storage.py

@@ -1,5 +1,6 @@
 import contextlib
 import os.path
+from pathlib import Path
 
 
 class EventStorage(object):
@@ -46,6 +47,11 @@ class FileEventStorage(EventStorage):
             # strict about what we allow; we further imply "text mode" and "utf-8 encoding" given the JSON context.
             raise ValueError("EventStorage.open() mode must be 'r' or 'w'")
 
+        if mode == 'w' and not os.path.exists(self.basepath):
+            # only if we're writing does this make sense (when reading, a newly created directoy won't have files in it,
+            # and fail in the next step)
+            Path(self.basepath).mkdir(parents=True, exist_ok=True)
+
         # We open with utf-8 encoding explicitly to pre-empt the future of pep-0686 (it's also the only thing that makes
         # sense in the context of JSON)
         with open(self._event_path(event_id), mode, encoding="utf-8") as f:

events/storage_registry.py

@@ -1,6 +1,7 @@
+from contextlib import contextmanager
 import importlib
 
-from bugsink.app_settings import get_settings
+from bugsink.app_settings import get_settings, override_settings
 
 
 _storages = None
@@ -48,3 +49,23 @@ def _resolve(name, conf):
     clazz = getattr(module, class_name)
 
     return clazz(name, **conf.get("OPTIONS", {}))
+
+
+@contextmanager
+def override_event_storages(storage_conf):
+    """
+    Temporarily override the event storage for the duration of the context (for tests).
+    """
+    global _storages
+    global _write_storage
+
+    _storages = None
+    _write_storage = None
+
+    try:
+        with override_settings(EVENT_STORAGES=storage_conf):
+            yield
+
+    finally:
+        _storages = None
+        _write_storage = None

ingest/tests.py

@@ -4,6 +4,7 @@ import json
 import io
 import uuid
 import time
+import tempfile
 
 import datetime
 from unittest.mock import patch
@@ -18,6 +19,8 @@ from django.core.exceptions import ValidationError
 from bugsink.test_utils import TransactionTestCase25251 as TransactionTestCase
 from projects.models import Project
 from events.factories import create_event_data, create_event
+from events.retention import evict_for_max_events
+from events.storage_registry import override_event_storages
 from issues.factories import get_or_create_issue
 from issues.models import IssueStateManager, Issue, TurningPoint, TurningPointKind
 from bugsink.app_settings import override_settings
@@ -397,6 +400,29 @@ class IngestViewTestCase(TransactionTestCase):
         with override_settings(DIGEST_IMMEDIATELY=False):
             self.test_envelope_endpoint()
 
+    @tag("samples")
+    def test_filestore(self):
+        # quick & dirty way to test the filestore; in absence of a proper test for it, we just run a more-or-less
+        # integration test with the FileEventStorage activated. This will at least show the absence of the most obvious
+        # errors. We then run
+        with tempfile.TemporaryDirectory() as tempdir:
+            with override_event_storages({"local_flat_files": {
+                        "STORAGE": "events.storage.FileEventStorage",
+                        "OPTIONS": {
+                            "basepath": tempdir,
+                        },
+                        "USE_FOR_WRITE": True,
+                    },
+                    }):
+                self.test_envelope_endpoint()
+                self.assertEqual(len(os.listdir(tempdir)), 2)  # test_envelope_endpoint creates 2 events
+
+                project = Project.objects.get(name="test")
+                project.retention_max_event_count = 1
+                evict_for_max_events(project, timezone.now(), stored_event_count=2)
+
+                self.assertEqual(len(os.listdir(tempdir)), 1)  # we set the max to 1, so one should remain
+
     @override_settings(MAX_EVENTS_PER_PROJECT_PER_5_MINUTES=0)
     @patch("ingest.views.check_for_thresholds")
     def test_count_project_periods_and_act_on_it_zero(self, patched_check_for_thresholds):