diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..c1c24be --- /dev/null +++ b/.bandit @@ -0,0 +1,10 @@ +[bandit] +# Skip B108 ("hardcoded temp dir"), see https://github.com/bugsink/bugsink/issues/174 +skips = B108 + +# Exclude any file named tests.py anywhere under the tree +exclude = tests.py + +# include everything, even LOW +confidence-level = LOW +severity-level = LOW diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2406cc6 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,59 @@ +Bugsink is a Django-based error tracker. It's maintained by one developer and favors simple, predictable code over +abstract generality. Python 3.12 is the standard environment. + +## Coding Guidance + +* Keep it clear and simple. When in doubt: shorter. +* Use descriptive names, short functions, minimal boilerplate. +* Error-handling: avoid catching every possible error; in many cases "fail early" + is in fact the idiom which gives much more clarity, especially in utilities. +* Avoid overly clever or verbose code +* Keep comments absolutely minimal: only comment to explain unusual or complex + (which there shouldn't be anyway) +* Follow PEP8 and ensure `flake8` passes (CI ignores E741, E731). width: 120 columns. +* Use Django's function-based views + +### Tests + +Tests should be either of 2 kinds: + +1. Tight unit-tests around easy-to-test, small, functions. These should express intent + clearly and enumerate the relevant cases (and no others) very carefully. The data + here will be very "synthetic", focussed on expressing intent + +2. Broader "integration-like" tests: much more end-to-end, with the goal of covering + [a] the integration of parts and [b] the happy paths for those. These tests should + not try to enumerate every single case. They should reflect a more typical flow fitting + "what would typically happen" (more natural inputs/outputs, no extensive edge-cases) + +### Database + +Bugsink assumes a single-writer DB model: + +* Keep writes as short as possible +* Do not worry about write-concurrency (there is none) +* Use the helpers in `bugsink/transaction.py` helpers like `@durable_atomic` + +### Frontend (Tailwind) + +* Bugsink does NOT have a "modern" backend/frontend split: "classic django instead" +* Bugsink uses Tailwind via `django-tailwind` + +Test the frontend by running the server like so and checking in a browser: + +`python manage.py runserver` (uses `bugsink.settings.development`) + +(an admin user is available in your environment. login with admin@example.org/admin) + +### Before committing + +A pre-commit hook is installed in your environment and will block any illegal commit. +Before commiting, _always_ run the following: + +``` +python manage.py tailwind build +git add theme/static/css/dist/styles.css +tools/strip-trailing-whitespace.sh +``` + +If you fail to do so the pre-commit hook will trigger, and you will not be able to commit. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3572a39..ee86b4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,16 @@ name: Continuous Integration on: push: - branches: [ "main" ] + # hardcoded list; GitHub does not support wildcards in branch names AFAICT + branches: [ "main", "1.4.x", "1.5.x", "1.6.x", "1.7.x", "1.8.x", "1.9.x", "1.10.x", "1.11.x" ] pull_request: branches: [ "main" ] + workflow_dispatch: # Enables manual invocation via "Run workflow" button in the Actions UI + inputs: + target_branch: + description: 'Branch to run workflow on' + required: false + default: 'main' env: DJANGO_SETTINGS_MODULE: "bugsink.settings.development" @@ -34,6 +41,56 @@ jobs: # so we just specify it on the command line flake8 --extend-ignore=E127,E741,E501,E731 `git ls-files | grep py$` + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Bandit and Plugins + run: | + pip install bandit spoils + + - name: Run Bandit and format results + shell: bash + run: | + # set +e disables "exit on any non-zero command" behavior + # set +o pipefail disables GH's default "fail the whole pipeline if any stage fails" + set +e +o pipefail + + # Note: .py files only; at the time of writing I checked the conf_templates/*.template + # also; but they had 2 False positives only (SECRET_KEY lives there by design) and I + # don't want to pollute templates that other people deal with with "nosec". + bandit_json_output=$( \ + git ls-files \ + | grep '\.py$' \ + | xargs bandit -q -f json --ini .bandit \ + ) + bandit_exit_code=$? + + echo "$bandit_json_output" \ + | jq -r ' + .results[] + | [ .filename + , .line_number + , .test_id + , .issue_confidence + , .issue_severity + , .test_name + ] + | @tsv + ' \ + | column -t + + if [[ $bandit_exit_code -ne 0 ]]; then + echo "Bandit found issues (exit code $bandit_exit_code)" + exit $bandit_exit_code + fi + test: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000..30b0aab --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,59 @@ +name: "Copilot Setup Steps" +on: + push: + paths: + - ".github/workflows/copilot-setup-steps.yml" + workflow_dispatch: + # The "on" section defines when this workflow runs independently. + # It has no effect on Copilot itself — Copilot runs this job on demand + # if the job name is exactly "copilot-setup-steps". + # These triggers just let us test the workflow manually if needed. + +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + permissions: + contents: read + env: + DJANGO_SETTINGS_MODULE: bugsink.settings.development + SAMPLES_DIR: ${{ github.workspace }}/event-samples + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install project `pre-commit` hook + run: | + cp pre-commit .git/hooks/pre-commit + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements.development.txt + # We install directly from source using pip — no wheels, no Docker. + # This keeps the setup fast and simple for Copilot. + # To ensure correctness for more complex scenarios (e.g. multiple DBs), + # we rely on the actual CI pipeline. + + - name: Run migrations + run: | + python manage.py migrate + # Ensures the SQLite database is initialized. + + - name: Create superuser + env: + DJANGO_SUPERUSER_USERNAME: admin@example.com + DJANGO_SUPERUSER_EMAIL: admin@example.com + DJANGO_SUPERUSER_PASSWORD: admin + run: | + python manage.py createsuperuser --noinput + + - name: Check out event-samples outside workspace + # by using a plain-old `git-clone` we are able to "ecape from the working dir" + run: | + git clone https://github.com/bugsink/event-samples.git ../event-samples diff --git a/CHANGELOG.md b/CHANGELOG.md index 2932685..667eb01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changes +## 1.7.6 (1 August 2025) + +* envelope-headers `sent_at` check should allow 00+00 (See #179) +* evenlope-header validation failure should not lead to envelope-rejection (See #179) + +## 1.7.5 (31 July 2025) + +### General Improvements + +* Add failure visibility for alert backends (See #169) +* Add per-month quota for email-sending (Fix #34) +* Store `remote_addr` on the event (Fix #165) +* Use `remote_addr` for `'{{auto}}'` `ip_addr` tags (See #165) +* `PID_FILE` check: make optional (See #99) +* `PID_FILE` check: don't use in docker/systemd (Fix #99) +* Breadcrumb timestamps: display harmonized w/ rest of application (ceca12940bd5) + +### Sourcemaps: better debugging + +* sourcemaps: Uploaded, but ignored, files: warn (See #158) +* Sourcemaps: Warn (in the logs) on multiple-debug-ids source uploads (See #157, #158) +* Debug IDs for missing sourcemaps: show them right in the stacktrace (See #158) +* Sourcemap Images IDs: show those in event details (See #158) + +### Configuration / Settings + +* `SINGLE_USER` implies `SINGLE_TEAM` and more (Fix #162) +* Docker config: `BEHIND_PLAIN_HTTP_PROXY` (Fix #164) +* Development setting: keep artifact bundles (1aef4a45c2dc) + +### Security Hardening + +* CI pipeline security checks with Bandit (See #175) +* Envelope parsing validates headers strictly (See #173) +* Use `django.utils._os.safe_join` to construct paths (see #173) + +### Internal Tooling + +* Remove the Django Debug Toolbar entirely (Fix #168) +* semaphore-for-db-write-lock: sqlite only (See #117) +* `send_json` utility: make envelope API the default (13226603ec7a) + +## 1.7.4, 1.6.4, 1.5.5, 1.4.3 (29 July 2025) + +Security release. Upgrading is highly recommended. See [this +notice](https://github.com/bugsink/bugsink/security/advisories/GHSA-q78p-g86f-jg6q) + ## 1.7.3 (17 July 2025) Migration fix: delete TurningPoints w/ project=None (Fix #155) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44f966e..892313a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,6 +40,8 @@ python manage.py tailwind build git add theme/static/css/dist/styles.css ``` +The pre-commit hook in the project's root does this automatically if needed, copy it to .git/hooks +to auto-run. ### Security diff --git a/Dockerfile b/Dockerfile index cc52519..cd75fbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,4 +51,4 @@ RUN ["bugsink-manage", "migrate", "snappea", "--database=snappea"] HEALTHCHECK CMD python -c 'import requests; requests.get("http://localhost:8000/health/ready").raise_for_status()' -CMD [ "monofy", "bugsink-manage", "check", "--deploy", "--fail-level", "WARNING", "&&", "bugsink-manage", "migrate", "&&", "bugsink-manage", "prestart", "&&", "gunicorn", "--config", "gunicorn.docker.conf.py", "--bind=0.0.0.0:$PORT", "--access-logfile", "-", "bugsink.wsgi", "|||", "bugsink-runsnappea"] +CMD [ "monofy", "bugsink-show-version", "&&", "bugsink-manage", "check", "--deploy", "--fail-level", "WARNING", "&&", "bugsink-manage", "migrate", "&&", "bugsink-manage", "prestart", "&&", "gunicorn", "--config", "bugsink/gunicorn.docker.conf.py", "--bind=0.0.0.0:$PORT", "--access-logfile", "-", "bugsink.wsgi", "|||", "bugsink-runsnappea"] diff --git a/Dockerfile.fromwheel b/Dockerfile.fromwheel index 69c5aad..ed65197 100644 --- a/Dockerfile.fromwheel +++ b/Dockerfile.fromwheel @@ -28,11 +28,10 @@ RUN --mount=type=cache,target=/var/cache/buildkit/pip \ # * this moves the dependency on the bugsink wheel up in the build-order, and # that's precisely the most changing part, i.e. the thing that breaks caching. # -# * all current (Apr 2025) dependencies .whl files are available on PyPI anyway. -# Exception: inotify_simple, but it's a pure python tar.gz; Nothing much -# is gained by fetch-first-install-later. And if we ever depend on further -# packages that require a build step, explicit action is probably needed -# anyway b/c of the build deps. +# * all current (Aug 2025) dependencies .whl files are available on PyPI anyway. +# Nothing much is gained by fetch-first-install-later. And if we ever +# depend on packages that require a build step, explicit action is probably +# needed anyway b/c of the build deps. # # * pointing to requirements.txt here instead of the wheel is tempting, but # breaks the idea of "just build the wheel" (requirements.txt is whatever @@ -70,11 +69,11 @@ COPY dist/$WHEEL_FILE /wheels/ RUN --mount=type=cache,target=/var/cache/buildkit/pip \ pip install /wheels/$WHEEL_FILE -COPY bugsink/conf_templates/docker.py.template bugsink_conf.py -COPY gunicorn.docker.conf.py /app/ +RUN cp /usr/local/lib/python3.12/site-packages/bugsink/conf_templates/docker.py.template /app/bugsink_conf.py && \ + cp /usr/local/lib/python3.12/site-packages/bugsink/gunicorn.docker.conf.py /app/gunicorn.docker.conf.py RUN ["bugsink-manage", "migrate", "snappea", "--database=snappea"] HEALTHCHECK CMD python -c 'import requests; requests.get("http://localhost:8000/health/ready").raise_for_status()' -CMD [ "monofy", "bugsink-manage", "check", "--deploy", "--fail-level", "WARNING", "&&", "bugsink-manage", "migrate", "&&", "bugsink-manage", "prestart", "&&", "gunicorn", "--config", "gunicorn.docker.conf.py", "--bind=0.0.0.0:$PORT", "--access-logfile", "-", "bugsink.wsgi", "|||", "bugsink-runsnappea"] +CMD [ "monofy", "bugsink-show-version", "&&", "bugsink-manage", "check", "--deploy", "--fail-level", "WARNING", "&&", "bugsink-manage", "migrate", "&&", "bugsink-manage", "prestart", "&&", "gunicorn", "--config", "gunicorn.docker.conf.py", "--bind=0.0.0.0:$PORT", "--access-logfile", "-", "bugsink.wsgi", "|||", "bugsink-runsnappea"] diff --git a/README.md b/README.md index b713152..f953d4e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Bugsink: Self-hosted Error Tracking +# Bugsink: Self-hosted Error Tracking * [Error Tracking](https://www.bugsink.com/error-tracking/) * [Built to self-host](https://www.bugsink.com/built-to-self-host/) @@ -30,6 +30,6 @@ are `admin`. Now, you can [set up your first project](https://www.bugsink.com/docs/quickstart/) and start tracking errors. -[Detailed installation instructions](https://www.bugsink.com/docs/installation/) are on the Bugsink website. +[Detailed installation instructions](https://www.bugsink.com/docs/installation/) are on the Bugsink website. [More information and documentation](https://www.bugsink.com/) diff --git a/alerts/admin.py b/alerts/admin.py index 846f6b4..3ccbddc 100644 --- a/alerts/admin.py +++ b/alerts/admin.py @@ -1 +1,10 @@ -# Register your models here. +from django.contrib import admin + +from .models import MessagingServiceConfig + + +@admin.register(MessagingServiceConfig) +class MessagingServiceConfigAdmin(admin.ModelAdmin): + list_display = ('project', 'display_name', 'kind', 'last_failure_timestamp') + search_fields = ('name', 'service_type') + list_filter = ('kind',) diff --git a/alerts/migrations/0003_messagingserviceconfig_last_failure_error_message_and_more.py b/alerts/migrations/0003_messagingserviceconfig_last_failure_error_message_and_more.py new file mode 100644 index 0000000..2fff8ea --- /dev/null +++ b/alerts/migrations/0003_messagingserviceconfig_last_failure_error_message_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.23 on 2025-07-28 14:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0002_alter_messagingserviceconfig_project'), + ] + + operations = [ + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_error_message', + field=models.TextField(blank=True, help_text='Error message from the exception', null=True), + ), + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_error_type', + field=models.CharField(blank=True, help_text="Type of error that occurred (e.g., 'requests.HTTPError')", max_length=100, null=True), + ), + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_is_json', + field=models.BooleanField(blank=True, help_text='Whether the response was valid JSON', null=True), + ), + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_response_text', + field=models.TextField(blank=True, help_text='Response text from the failed request', null=True), + ), + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_status_code', + field=models.IntegerField(blank=True, help_text='HTTP status code of the failed request', null=True), + ), + migrations.AddField( + model_name='messagingserviceconfig', + name='last_failure_timestamp', + field=models.DateTimeField(blank=True, help_text='When the last failure occurred', null=True), + ), + ] diff --git a/alerts/models.py b/alerts/models.py index e7eec32..2c3cec2 100644 --- a/alerts/models.py +++ b/alerts/models.py @@ -13,6 +13,33 @@ class MessagingServiceConfig(models.Model): config = models.TextField(blank=False) + # Alert backend failure tracking + last_failure_timestamp = models.DateTimeField(null=True, blank=True, + help_text="When the last failure occurred") + last_failure_status_code = models.IntegerField(null=True, blank=True, + help_text="HTTP status code of the failed request") + last_failure_response_text = models.TextField(null=True, blank=True, + help_text="Response text from the failed request") + last_failure_is_json = models.BooleanField(null=True, blank=True, + help_text="Whether the response was valid JSON") + last_failure_error_type = models.CharField(max_length=100, null=True, blank=True, + help_text="Type of error that occurred (e.g., 'requests.HTTPError')") + last_failure_error_message = models.TextField(null=True, blank=True, + help_text="Error message from the exception") + def get_backend(self): # once we have multiple backends: lookup by kind. return SlackBackend(self) + + def clear_failure_status(self): + """Clear all failure tracking fields on successful operation""" + self.last_failure_timestamp = None + self.last_failure_status_code = None + self.last_failure_response_text = None + self.last_failure_is_json = None + self.last_failure_error_type = None + self.last_failure_error_message = None + + def has_recent_failure(self): + """Check if this config has a recent failure""" + return self.last_failure_timestamp is not None diff --git a/alerts/service_backends/slack.py b/alerts/service_backends/slack.py index be4c9ad..805e526 100644 --- a/alerts/service_backends/slack.py +++ b/alerts/service_backends/slack.py @@ -1,11 +1,13 @@ import json import requests +from django.utils import timezone from django import forms from django.template.defaultfilters import truncatechars from snappea.decorators import shared_task from bugsink.app_settings import get_settings +from bugsink.transaction import immediate_atomic from issues.models import Issue @@ -32,8 +34,57 @@ def _safe_markdown(text): return text.replace("&", "&").replace("<", "<").replace(">", ">").replace("*", "\\*").replace("_", "\\_") +def _store_failure_info(service_config_id, exception, response=None): + """Store failure information in the MessagingServiceConfig with immediate_atomic""" + from alerts.models import MessagingServiceConfig + + with immediate_atomic(only_if_needed=True): + try: + config = MessagingServiceConfig.objects.get(id=service_config_id) + + config.last_failure_timestamp = timezone.now() + config.last_failure_error_type = type(exception).__name__ + config.last_failure_error_message = str(exception) + + # Handle requests-specific errors + if response is not None: + config.last_failure_status_code = response.status_code + config.last_failure_response_text = response.text[:2000] # Limit response text size + + # Check if response is JSON + try: + json.loads(response.text) + config.last_failure_is_json = True + except (json.JSONDecodeError, ValueError): + config.last_failure_is_json = False + else: + # Non-HTTP errors + config.last_failure_status_code = None + config.last_failure_response_text = None + config.last_failure_is_json = None + + config.save() + except MessagingServiceConfig.DoesNotExist: + # Config was deleted while task was running + pass + + +def _store_success_info(service_config_id): + """Clear failure information on successful operation""" + from alerts.models import MessagingServiceConfig + + with immediate_atomic(only_if_needed=True): + try: + config = MessagingServiceConfig.objects.get(id=service_config_id) + config.clear_failure_status() + config.save() + except MessagingServiceConfig.DoesNotExist: + # Config was deleted while task was running + pass + + @shared_task -def slack_backend_send_test_message(webhook_url, project_name, display_name): +def slack_backend_send_test_message(webhook_url, project_name, display_name, service_config_id): # See Slack's Block Kit Builder data = {"blocks": [ @@ -67,17 +118,29 @@ def slack_backend_send_test_message(webhook_url, project_name, display_name): ]} - result = requests.post( - webhook_url, - data=json.dumps(data), - headers={"Content-Type": "application/json"}, - ) + try: + result = requests.post( + webhook_url, + data=json.dumps(data), + headers={"Content-Type": "application/json"}, + timeout=5, + ) - result.raise_for_status() + result.raise_for_status() + + _store_success_info(service_config_id) + except requests.RequestException as e: + response = getattr(e, 'response', None) + _store_failure_info(service_config_id, e, response) + + except Exception as e: + _store_failure_info(service_config_id, e) @shared_task -def slack_backend_send_alert(webhook_url, issue_id, state_description, alert_article, alert_reason, unmute_reason=None): +def slack_backend_send_alert( + webhook_url, issue_id, state_description, alert_article, alert_reason, service_config_id, unmute_reason=None): + issue = Issue.objects.get(id=issue_id) issue_url = get_settings().BASE_URL + issue.get_absolute_url() @@ -134,13 +197,23 @@ def slack_backend_send_alert(webhook_url, issue_id, state_description, alert_art }, ]} - result = requests.post( - webhook_url, - data=json.dumps(data), - headers={"Content-Type": "application/json"}, - ) + try: + result = requests.post( + webhook_url, + data=json.dumps(data), + headers={"Content-Type": "application/json"}, + timeout=5, + ) - result.raise_for_status() + result.raise_for_status() + + _store_success_info(service_config_id) + except requests.RequestException as e: + response = getattr(e, 'response', None) + _store_failure_info(service_config_id, e, response) + + except Exception as e: + _store_failure_info(service_config_id, e) class SlackBackend: @@ -156,9 +229,10 @@ class SlackBackend: json.loads(self.service_config.config)["webhook_url"], self.service_config.project.name, self.service_config.display_name, + self.service_config.id, ) def send_alert(self, issue_id, state_description, alert_article, alert_reason, **kwargs): slack_backend_send_alert.delay( json.loads(self.service_config.config)["webhook_url"], - issue_id, state_description, alert_article, alert_reason, **kwargs) + issue_id, state_description, alert_article, alert_reason, self.service_config.id, **kwargs) diff --git a/alerts/templates/mails/issue_alert.html b/alerts/templates/mails/issue_alert.html index a1ff7d1..144446c 100644 --- a/alerts/templates/mails/issue_alert.html +++ b/alerts/templates/mails/issue_alert.html @@ -9,7 +9,7 @@