diff --git a/docs/config.md b/docs/config.md index 07267fca..0e2e9b6b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1111,16 +1111,26 @@ doing, and/or secure access to the endpoint in your reverse proxy. - `metrics-listen-http` exposes the metrics endpoint via a dedicated `[IP]:port`. If set, this option implicitly enables metrics as well, e.g. "10.0.1.1:9090" or ":9090" -=== "Using default port" +=== "server.yml (Using default port)" ```yaml enable-metrics: true ``` -=== "Using dedicated IP/port" +=== "server.yml (Using dedicated IP/port)" ```yaml metrics-listen-http: "10.0.1.1:9090" ``` +In Prometheus, an example scrape config would look like this: + +=== "prometheus.yml" + ```yaml + scrape_configs: + - job_name: "ntfy" + static_configs: + - targets: ["10.0.1.1:9090"] + ``` + Here's an example Grafana dashboard built from the metrics (see [Grafana JSON on GitHub](https://raw.githubusercontent.com/binwiederhier/ntfy/main/examples/grafana-dashboard/ntfy-grafana.json)):
diff --git a/docs/integrations.md b/docs/integrations.md index 5c646e5c..93880ff9 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -116,6 +116,7 @@ and uptime of third party servers, so use of each server is **at your own discre - [nodebb-plugin-ntfy](https://github.com/NodeBB/nodebb-plugin-ntfy) - Push notifications for NodeBB forums - [n8n-ntfy](https://github.com/raghavanand98/n8n-ntfy.sh) - n8n community node that lets you use ntfy in your workflows - [nlog-ntfy](https://github.com/MichelMichels/nlog-ntfy) - Send NLog messages over ntfy (C# / .NET / NLog) +- [helm-charts](https://github.com/sarab97/helm-charts) - Helm charts of some of the selfhosted services, incl. ntfy ## Blog + forum posts diff --git a/docs/static/img/android-screenshot-logs.jpg b/docs/static/img/android-screenshot-logs.jpg new file mode 100644 index 00000000..e5f1d8e8 Binary files /dev/null and b/docs/static/img/android-screenshot-logs.jpg differ diff --git a/docs/static/img/grafana-dashboard.png b/docs/static/img/grafana-dashboard.png index 4f2cfc8a..6cb12c80 100644 Binary files a/docs/static/img/grafana-dashboard.png and b/docs/static/img/grafana-dashboard.png differ diff --git a/docs/static/img/web-logs.png b/docs/static/img/web-logs.png new file mode 100644 index 00000000..2dfebdcc Binary files /dev/null and b/docs/static/img/web-logs.png differ diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..9f882b23 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,113 @@ +# Troubleshooting +This page lists a few suggestions of what to do when things don't work as expected. This is not a complete list. +If this page does not help, feel free to drop by the [Discord](https://discord.gg/cT7ECsZj9w) or [Matrix](https://matrix.to/#/#ntfy:matrix.org) +and ask there. We're happy to help. + +## ntfy server +If you host your own ntfy server, and you're having issues with any component, it is always helpful to enable debugging/tracing +in the server. You can find detailed instructions in the [Logging & Debugging](config.md#logging-debugging) section, but it ultimately +boils down to setting `log-level: debug` or `log-level: trace` in the `server.yml` file: + +=== "server.yml (debug)" + ``` yaml + log-level: debug + ``` + +=== "server.yml (trace)" + ``` yaml + log-level: trace + ``` + +If you're using environment variables, set `NTFY_LOG_LEVEL=debug` (or `trace`) instead. You can also pass `--debug` or `--trace` +to the `ntfy serve` command, e.g. `ntfy serve --trace`. If you're using systemd (i.e. `systemctl`) to run ntfy, you can look at the logs using `journalctl -u ntfy -f`. + +Assuming trace is enabled, the logs will look something like this: + +=== "Example logs (debug)" + ``` + $ ntfy serve --debug + 2023/03/20 14:45:38 INFO Listening on :2586[http] :1025[smtp], ntfy 2.1.2, log level is DEBUG (tag=startup) + 2023/03/20 14:45:38 DEBUG Waiting until 2023-03-21 00:00:00 +0000 UTC to reset visitor stats (tag=resetter) + 2023/03/20 14:45:39 DEBUG Rate limiters reset for visitor (visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=0, visitor_messages_limit=500, visitor_messages_remaining=500, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=60, visitor_seen=2023-03-20T14:45:39.7-04:00) + 2023/03/20 14:45:39 DEBUG HTTP request started (http_method=POST, http_path=/mytopic, tag=http, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=0, visitor_messages_limit=500, visitor_messages_remaining=500, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=60, visitor_seen=2023-03-20T14:45:39.7-04:00) + 2023/03/20 14:45:39 DEBUG Received message (http_method=POST, http_path=/mytopic, message_body_size=2, message_delayed=false, message_email=, message_event=message, message_firebase=true, message_id=EZu6i2WZjH0v, message_sender=127.0.0.1, message_time=1679337939, message_unifiedpush=false, tag=publish, topic=mytopic, topic_last_access=2023-03-20T14:45:38.319-04:00, topic_subscribers=0, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.0002132248, visitor_seen=2023-03-20T14:45:39.7-04:00) + 2023/03/20 14:45:39 DEBUG Adding message to cache (http_method=POST, http_path=/mytopic, message_body_size=2, message_event=message, message_id=EZu6i2WZjH0v, message_sender=127.0.0.1, message_time=1679337939, tag=publish, topic=mytopic, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.000259165, visitor_seen=2023-03-20T14:45:39.7-04:00) + 2023/03/20 14:45:39 DEBUG HTTP request finished (http_method=POST, http_path=/mytopic, tag=http, time_taken_ms=2, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.0004147334, visitor_seen=2023-03-20T14:45:39.7-04:00) + 2023/03/20 14:45:39 DEBUG Wrote 1 message(s) in 8.285712ms (tag=message_cache) + ... + ``` + +=== "Example logs (trace)" + ``` + $ ntfy serve --trace + 2023/03/20 14:40:42 INFO Listening on :2586[http] :1025[smtp], ntfy 2.1.2, log level is TRACE (tag=startup) + 2023/03/20 14:40:42 DEBUG Waiting until 2023-03-21 00:00:00 +0000 UTC to reset visitor stats (tag=resetter) + 2023/03/20 14:40:59 DEBUG Rate limiters reset for visitor (visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=0, visitor_messages_limit=500, visitor_messages_remaining=500, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=60, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:40:59 TRACE HTTP request started (http_method=POST, http_path=/mytopic, http_request=POST /mytopic HTTP/1.1 + User-Agent: curl/7.81.0 + Accept: */* + Content-Length: 2 + Content-Type: application/x-www-form-urlencoded + + hi, tag=http, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=0, visitor_messages_limit=500, visitor_messages_remaining=500, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=60, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:40:59 TRACE Received message (http_method=POST, http_path=/mytopic, message_body={ + "id": "Khaup1RVclU3", + "time": 1679337659, + "expires": 1679380859, + "event": "message", + "topic": "mytopic", + "message": "hi" + }, message_body_size=2, message_delayed=false, message_email=, message_event=message, message_firebase=true, message_id=Khaup1RVclU3, message_sender=127.0.0.1, message_time=1679337659, message_unifiedpush=false, tag=publish, topic=mytopic, topic_last_access=2023-03-20T14:40:59.893-04:00, topic_subscribers=0, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.0001785048, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:40:59 DEBUG Adding message to cache (http_method=POST, http_path=/mytopic, message_body_size=2, message_event=message, message_id=Khaup1RVclU3, message_sender=127.0.0.1, message_time=1679337659, tag=publish, topic=mytopic, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.0002044368, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:40:59 DEBUG HTTP request finished (http_method=POST, http_path=/mytopic, tag=http, time_taken_ms=1, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.000220502, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:40:59 TRACE No stream or WebSocket subscribers, not forwarding (message_body_size=2, message_event=message, message_id=Khaup1RVclU3, message_sender=127.0.0.1, message_time=1679337659, tag=publish, topic=mytopic, visitor_auth_limiter_limit=0.016666666666666666, visitor_auth_limiter_tokens=10, visitor_emails=0, visitor_emails_limit=12, visitor_emails_remaining=12, visitor_id=ip:127.0.0.1, visitor_ip=127.0.0.1, visitor_messages=1, visitor_messages_limit=500, visitor_messages_remaining=499, visitor_request_limiter_limit=0.2, visitor_request_limiter_tokens=59.0002369212, visitor_seen=2023-03-20T14:40:59.893-04:00) + 2023/03/20 14:41:00 DEBUG Wrote 1 message(s) in 9.529196ms (tag=message_cache) + ... + ``` + +## Android app +On Android, you can turn on logging in the settings under **Settings → Record logs**. This will store up to 1,000 log +entries, which you can then copy or upload. + +
+ ![Recording logs on Android](static/img/android-screenshot-logs.jpg){ width=400 } +
Recording logs on Android
+
+ +When you copy or upload the logs, you can censor them to make it easier to share them with others. ntfy will replace all +topics and hostnames with fruits. Here's an example: + +=== "Android log (censored)" + ``` + This is a log of the ntfy Android app. The log shows up to 1,000 entries. + Server URLs (aside from ntfy.sh) and topics have been replaced with fruits 🍌🥝🍋🥥🥑🍊🍎🍑. + + Device info: + -- + ntfy: 1.16.0 (play) + OS: 4.19.157-perf+ + Android: 13 (SDK 33) + ... + + Logs + -- + + 1679339199507 2023-03-20 15:06:39.507 D NtfyMainActivity Battery: ignoring optimizations = true (we want this to be true); instant subscriptions = true; remind time reached = true; banner = false + 1679339199507 2023-03-20 15:06:39.507 D NtfySubscriberMgr Enqueuing work to refresh subscriber service + 1679339199589 2023-03-20 15:06:39.589 D NtfySubscriberMgr ServiceStartWorker: Starting foreground service with action START (work ID: a7eeeae9-9356-40df-afbd-236e5ed10a0b) + 1679339199602 2023-03-20 15:06:39.602 D NtfySubscriberService onStartCommand executed with startId: 262 + 1679339199602 2023-03-20 15:06:39.602 D NtfySubscriberService using an intent with action START + 1679339199629 2023-03-20 15:06:39.629 D NtfySubscriberService Refreshing subscriptions + 1679339199629 2023-03-20 15:06:39.629 D NtfySubscriberService - Desired connections: [ConnectionId(baseUrl=https://ntfy.sh, topicsToSubscriptionIds={avocado=23801492, lemon=49013182, banana=1309176509201171073, peach=573300885184666424, pineapple=-5956897229801209316, durian=81453333, starfruit=30489279, fruit12=82532869}), ConnectionId(baseUrl=https://orange.example.com, topicsToSubscriptionIds={apple=4971265, dragonfruit=66809328})] + 1679339199629 2023-03-20 15:06:39.629 D NtfySubscriberService - Active connections: [ConnectionId(baseUrl=https://orange.example.com, topicsToSubscriptionIds={apple=4971265, dragonfruit=66809328}), ConnectionId(baseUrl=https://ntfy.sh, topicsToSubscriptionIds={avocado=23801492, lemon=49013182, banana=1309176509201171073, peach=573300885184666424, pineapple=-5956897229801209316, durian=81453333, starfruit=30489279, fruit12=82532869})] + ... + ``` + +## Web app +The web app logs everything to the **developer console**, which you can open by **pressing the F12 key** on your +keyboard. + +
+ ![Web app logs](static/img/web-logs.png) +
Web app logs in the developer console
+
diff --git a/examples/grafana-dashboard/ntfy-grafana.json b/examples/grafana-dashboard/ntfy-grafana.json index a5cc5d2d..11273da3 100644 --- a/examples/grafana-dashboard/ntfy-grafana.json +++ b/examples/grafana-dashboard/ntfy-grafana.json @@ -129,8 +129,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_messages_published_success", + "editorMode": "code", + "expr": "ntfy_messages_published_success{job=\"$job\"}", "legendFormat": "Messages cached", "range": true, "refId": "A" @@ -191,7 +191,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_messages_cached_total", + "expr": "ntfy_messages_cached_total{job=\"$job\"}", "legendFormat": "Messages cached", "range": true, "refId": "A" @@ -252,7 +252,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_visitors_total", + "expr": "ntfy_visitors_total{job=\"$job\"}", "legendFormat": "Visitors", "range": true, "refId": "A" @@ -318,7 +318,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_users_total", + "expr": "ntfy_users_total{job=\"$job\"}", "legendFormat": "Visitors", "range": true, "refId": "A" @@ -380,7 +380,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_topics_total", + "expr": "ntfy_topics_total{job=\"$job\"}", "legendFormat": "Topics", "range": true, "refId": "A" @@ -467,6 +467,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Number of successfully published messages, and messages that could not be published (due to rate limiting, bad formatting, etc.)", "fieldConfig": { "defaults": { "color": { @@ -509,15 +510,31 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "custom.axisColorMode", + "value": "text" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -525,7 +542,7 @@ "x": 0, "y": 5 }, - "id": 14, + "id": 42, "options": { "legend": { "calcs": [], @@ -544,9 +561,9 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_visitors_total", - "legendFormat": "Visitors", + "editorMode": "code", + "expr": "rate(ntfy_messages_published_success{job=\"$job\"}[$rate])", + "legendFormat": "Success", "range": true, "refId": "A" }, @@ -555,39 +572,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_topics_total", + "editorMode": "code", + "expr": "rate(ntfy_messages_published_failure{job=\"$job\"}[$rate])", "hide": false, - "legendFormat": "Topics", + "legendFormat": "Failed", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "ntfy_subscribers_total", - "hide": false, - "legendFormat": "Subscribers", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "ntfy_users_total", - "hide": false, - "legendFormat": "Users", - "range": true, - "refId": "D" } ], - "title": "Visitors, subscribers, topics", + "title": "Messages published (per second)", "type": "timeseries" }, { @@ -595,7 +588,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Number of messages published since last ntfy server restrart", + "description": "Number of messages published since last ntfy server restart", "fieldConfig": { "defaults": { "color": { @@ -646,7 +639,23 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -674,7 +683,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_messages_published_success", + "expr": "ntfy_messages_published_success{job=\"$job\"}", "legendFormat": "Successful", "range": true, "refId": "A" @@ -685,7 +694,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_messages_published_failure", + "expr": "ntfy_messages_published_failure{job=\"$job\"}", "hide": false, "legendFormat": "Failed", "range": true, @@ -700,6 +709,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "Number of messages currently stored in message cache", "fieldConfig": { "defaults": { "color": { @@ -758,7 +768,7 @@ "x": 12, "y": 5 }, - "id": 42, + "id": 2, "options": { "legend": { "calcs": [], @@ -778,25 +788,13 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "rate(ntfy_messages_published_success[10m])", - "legendFormat": "Successful", + "expr": "ntfy_messages_cached_total{job=\"$job\"}", + "legendFormat": "Messages in database", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(ntfy_messages_published_failure[10m])", - "hide": false, - "legendFormat": "Failed", - "range": true, - "refId": "B" } ], - "title": "Messages published (per second)", + "title": "Messages cached", "type": "timeseries" }, { @@ -804,7 +802,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -863,7 +860,7 @@ "x": 18, "y": 5 }, - "id": 2, + "id": 14, "options": { "legend": { "calcs": [], @@ -883,13 +880,49 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_messages_cached_total", - "legendFormat": "Messages in database", + "expr": "ntfy_visitors_total{job=\"$job\"}", + "legendFormat": "Visitors", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "ntfy_topics_total{job=\"$job\"}", + "hide": false, + "legendFormat": "Topics", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "ntfy_subscribers_total{job=\"$job\"}", + "hide": false, + "legendFormat": "Subscribers", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "ntfy_users_total{job=\"$job\"}", + "hide": false, + "legendFormat": "Users", + "range": true, + "refId": "D" } ], - "title": "Messages cached", + "title": "Visitors, subscribers, topics", "type": "timeseries" }, { @@ -974,8 +1007,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "sum by(job) (rate(ntfy_http_requests_total[10m]))", + "editorMode": "code", + "expr": "sum by(job) (rate(ntfy_http_requests_total{job=\"$job\"}[$rate]))", "legendFormat": "Requests per second", "range": true, "refId": "A" @@ -1043,109 +1076,21 @@ }, "gridPos": { "h": 7, - "w": 6, + "w": 9, "x": 6, "y": 12 }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "sum by(http_code) (ntfy_http_requests_total{http_code=\"200\"})", - "legendFormat": "{{http_code}}", - "range": true, - "refId": "A" - } - ], - "title": "HTTP requests (success only)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 12 - }, "id": 41, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, "tooltip": { "mode": "single", @@ -1158,14 +1103,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "sum by(http_code) (ntfy_http_requests_total{http_code!=\"200\"})", + "editorMode": "code", + "expr": "sum by(http_code) (rate(ntfy_http_requests_total{job=\"$job\", http_code!=\"200\", http_code!=\"429\", http_code!=\"507\"}[$rate]))", "legendFormat": "{{http_code}}", "range": true, "refId": "A" } ], - "title": "HTTP requests (error only)", + "title": "HTTP errors (per second, excl. 429/507)", "type": "timeseries" }, { @@ -1227,17 +1172,21 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 18, + "w": 9, + "x": 15, "y": 12 }, "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, "tooltip": { "mode": "single", @@ -1250,14 +1199,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "sum by(ntfy_code) (ntfy_http_requests_total{http_code!=\"200\"})", + "editorMode": "code", + "expr": "sum by(ntfy_code) (rate(ntfy_http_requests_total{http_code!=\"200\", job=\"$job\"}[$rate]))", "legendFormat": "{{http_method}} {{http_code}} {{ntfy_code}}", "range": true, "refId": "A" } ], - "title": "HTTP requests (error only, ntfy code)", + "title": "HTTP errors (per second, ntfy code)", "type": "timeseries" }, { @@ -1344,7 +1293,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_attachments_total_size", + "expr": "ntfy_attachments_total_size{job=\"$job\"}", "legendFormat": "Total size in MB", "range": true, "refId": "A" @@ -1369,7 +1318,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1409,7 +1358,23 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -1436,8 +1401,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_firebase_published_success", + "editorMode": "code", + "expr": "rate(ntfy_firebase_published_success{job=\"$job\"}[$rate])", "legendFormat": "Success", "range": true, "refId": "A" @@ -1447,8 +1412,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_firebase_published_failure", + "editorMode": "code", + "expr": "rate(ntfy_firebase_published_failure{job=\"$job\"}[$rate])", "hide": false, "legendFormat": "Failure", "range": true, @@ -1505,15 +1470,27 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Rejected (HTTP 507)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -1540,8 +1517,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_unifiedpush_published_success", + "editorMode": "code", + "expr": "rate(ntfy_unifiedpush_published_success{job=\"$job\"}[$rate])", "legendFormat": "Success", "range": true, "refId": "A" @@ -1551,10 +1528,10 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_http_requests_total{ntfy_code=\"50701\"}", + "editorMode": "code", + "expr": "rate(ntfy_http_requests_total{job=\"$job\",http_code=\"507\"}[$rate])", "hide": false, - "legendFormat": "Rejected", + "legendFormat": "Rejected (HTTP 507)", "range": true, "refId": "B" } @@ -1617,7 +1594,23 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -1644,8 +1637,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_matrix_published_success", + "editorMode": "code", + "expr": "rate(ntfy_matrix_published_success{job=\"$job\"}[$rate])", "legendFormat": "Success", "range": true, "refId": "A" @@ -1655,8 +1648,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "ntfy_matrix_published_failure", + "editorMode": "code", + "expr": "rate(ntfy_matrix_published_failure{job=\"$job\"}[$rate])", "hide": false, "legendFormat": "Failure", "range": true, @@ -1721,7 +1714,23 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -1749,8 +1758,8 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_emails_sent_success", - "legendFormat": "Successful", + "expr": "ntfy_emails_sent_success{job=\"$job\"}", + "legendFormat": "Success", "range": true, "refId": "A" }, @@ -1760,7 +1769,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_emails_sent_failure", + "expr": "ntfy_emails_sent_failure{job=\"$job\"}", "hide": false, "legendFormat": "Failure", "range": true, @@ -1825,7 +1834,23 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, @@ -1853,7 +1878,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_emails_received_success", + "expr": "ntfy_emails_received_success{job=\"$job\"}", "legendFormat": "Success", "range": true, "refId": "A" @@ -1864,7 +1889,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_emails_received_failure", + "expr": "ntfy_emails_received_failure{job=\"$job\"}", "hide": false, "legendFormat": "Failure", "range": true, @@ -1958,7 +1983,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "ntfy_message_publish_duration_ms", + "expr": "ntfy_message_publish_duration_ms{job=\"$job\"}", "legendFormat": "Duration", "range": true, "refId": "A" @@ -2063,8 +2088,8 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "go_goroutines{job=\"ntfy\"}", - "legendFormat": "__auto", + "expr": "go_goroutines{job=\"$job\"}", + "legendFormat": "Go routines", "range": true, "refId": "A" } @@ -2100,7 +2125,8 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "symlog" }, "showPoints": "auto", "spanNulls": false, @@ -2156,7 +2182,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "process_open_fds{job=\"ntfy\"}", + "expr": "process_open_fds{job=\"$job\"}", "legendFormat": "Open", "range": true, "refId": "A" @@ -2167,7 +2193,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "process_max_fds{job=\"ntfy\"}", + "expr": "process_max_fds{job=\"$job\"}", "hide": false, "legendFormat": "Max", "range": true, @@ -2261,8 +2287,8 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "process_resident_memory_bytes{job=\"ntfy\"}", - "legendFormat": "__auto", + "expr": "process_resident_memory_bytes{job=\"$job\"}", + "legendFormat": "Resident memory used by ntfy (RSS)", "range": true, "refId": "A" }, @@ -2272,9 +2298,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "process_virtual_memory_bytes{job=\"ntfy\"}", + "expr": "process_virtual_memory_bytes{job=\"$job\"}", "hide": false, - "legendFormat": "__auto", + "legendFormat": "Virtual memory used by ntfy (VSS)", "range": true, "refId": "B" } @@ -2283,13 +2309,83 @@ "type": "timeseries" } ], - "refresh": false, + "refresh": "10s", "revision": 1, "schemaVersion": 38, "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(ntfy_visitors_total, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(ntfy_visitors_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30m", + "value": "30m" + }, + "description": "Average per-second rates over values from this time span", + "hide": 0, + "label": "Rate", + "name": "rate", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": true, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,10m,30m,1h", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] }, "time": { "from": "now-24h", @@ -2299,6 +2395,6 @@ "timezone": "", "title": "ntfy App", "uid": "TO6HgexVz", - "version": 11, + "version": 24, "weekStart": "" } \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 66fc4c84..b49931d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -93,6 +93,7 @@ nav: - "Integrations + projects": integrations.md - "Release notes": releases.md - "Emojis 🥳 🎉": emojis.md + - "Troubleshooting": troubleshooting.md - "Known issues": known-issues.md - "Deprecation notices": deprecations.md - "Development": develop.md