Bump deps

Batch message INSERTs

.github/workflows/docs.yaml

@@ -0,0 +1,36 @@
+name: docs
+on:
+  push:
+    branches:
+      - main
+jobs:
+  publish-docs:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout ntfy code
+        uses: actions/checkout@v3
+      -
+        name: Checkout docs pages code
+        uses: actions/checkout@v3
+        with:
+          repository: binwiederhier/ntfy-docs.github.io
+          path: build/ntfy-docs.github.io
+          token: ${{secrets.NTFY_DOCS_PUSH_TOKEN}}
+          # Expires after 1 year, re-generate via
+          # User -> Settings -> Developer options -> Personal Access Tokens -> Fine Grained Token
+      -
+        name: Build docs
+        run: make docs
+      -
+        name: Copy generated docs
+        run: rsync -av --exclude CNAME --delete server/docs/ build/ntfy-docs.github.io/docs/
+      -
+        name: Publish docs
+        run: |
+          cd build/ntfy-docs.github.io
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"          
+          git add docs/
+          git commit -m "Updated docs"
+          git push origin main

README.md

@@ -96,6 +96,9 @@ appreciated. A big fat **Thank You** to the folks already sponsoring ntfy:
 <a href="https://github.com/nwithan8"><img src="https://github.com/nwithan8.png" width="40px" /></a>
 <a href="https://github.com/peterleiser"><img src="https://github.com/peterleiser.png" width="40px" /></a>
 <a href="https://github.com/portothree"><img src="https://github.com/portothree.png" width="40px" /></a>
+<a href="https://github.com/finngreig"><img src="https://github.com/finngreig.png" width="40px" /></a>
+<a href="https://github.com/skrollme"><img src="https://github.com/skrollme.png" width="40px" /></a>
+<a href="https://github.com/gergepalfi"><img src="https://github.com/gergepalfi.png" width="40px" /></a>
 
 ## License
 Made with ❤️ by [Philipp C. Heckel](https://heckel.io).   

cmd/publish_test.go

@@ -17,6 +17,7 @@ func TestCLI_Publish_Subscribe_Poll_Real_Server(t *testing.T) {
 
 	app, _, _, _ := newTestApp()
 	require.Nil(t, app.Run([]string{"ntfy", "publish", "ntfytest", "ntfy unit test " + testMessage}))
+	time.Sleep(3 * time.Second) // Since #502, ntfy.sh writes messages to the cache asynchronously, after a timeout of ~1.5s
 
 	app2, _, stdout, _ := newTestApp()
 	require.Nil(t, app2.Run([]string{"ntfy", "subscribe", "--poll", "ntfytest"}))

cmd/serve.go

@@ -44,6 +44,8 @@ var flagsServe = append(
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "firebase-key-file", Aliases: []string{"firebase_key_file", "F"}, EnvVars: []string{"NTFY_FIREBASE_KEY_FILE"}, Usage: "Firebase credentials file; if set additionally publish to FCM topic"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"cache_file", "C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}),
 	altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-duration", Aliases: []string{"cache_duration", "b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: server.DefaultCacheDuration, Usage: "buffer messages for this time to allow `since` requests"}),
+	altsrc.NewIntFlag(&cli.IntFlag{Name: "cache-batch-size", Aliases: []string{"cache_batch_size"}, EnvVars: []string{"NTFY_BATCH_SIZE"}, Usage: "max size of messages to batch together when writing to message cache (if zero, writes are synchronous)"}),
+	altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-batch-timeout", Aliases: []string{"cache_batch_timeout"}, EnvVars: []string{"NTFY_CACHE_BATCH_TIMEOUT"}, Usage: "timeout for batched async writes to the message cache (if zero, writes are synchronous)"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-startup-queries", Aliases: []string{"cache_startup_queries"}, EnvVars: []string{"NTFY_CACHE_STARTUP_QUERIES"}, Usage: "queries run when the cache database is initialized"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "auth-file", Aliases: []string{"auth_file", "H"}, EnvVars: []string{"NTFY_AUTH_FILE"}, Usage: "auth database file used for access control"}),
 	altsrc.NewStringFlag(&cli.StringFlag{Name: "auth-default-access", Aliases: []string{"auth_default_access", "p"}, EnvVars: []string{"NTFY_AUTH_DEFAULT_ACCESS"}, Value: "read-write", Usage: "default permissions if no matching entries in the auth database are found"}),
@@ -110,6 +112,8 @@ func execServe(c *cli.Context) error {
 	cacheFile := c.String("cache-file")
 	cacheDuration := c.Duration("cache-duration")
 	cacheStartupQueries := c.String("cache-startup-queries")
+	cacheBatchSize := c.Int("cache-batch-size")
+	cacheBatchTimeout := c.Duration("cache-batch-timeout")
 	authFile := c.String("auth-file")
 	authDefaultAccess := c.String("auth-default-access")
 	attachmentCacheDir := c.String("attachment-cache-dir")
@@ -233,6 +237,8 @@ func execServe(c *cli.Context) error {
 	conf.CacheFile = cacheFile
 	conf.CacheDuration = cacheDuration
 	conf.CacheStartupQueries = cacheStartupQueries
+	conf.CacheBatchSize = cacheBatchSize
+	conf.CacheBatchTimeout = cacheBatchTimeout
 	conf.AuthFile = authFile
 	conf.AuthDefaultRead = authDefaultRead
 	conf.AuthDefaultWrite = authDefaultWrite

docs/config.md

@@ -309,6 +309,25 @@ with the given username/password. Be sure to use HTTPS to avoid eavesdropping an
     ]));
     ```
 
+### Example: UnifiedPush
+[UnifiedPush](https://unifiedpush.org) requires that the [application server](https://unifiedpush.org/spec/definitions/#application-server) (e.g. Synapse, Fediverse Server, …) 
+has anonymous write access to the [topic](https://unifiedpush.org/spec/definitions/#endpoint) used for push messages. 
+The topic names used by UnifiedPush all start with the `up*` prefix. Please refer to the 
+**[UnifiedPush documentation](https://unifiedpush.org/users/distributors/ntfy/#limit-access-to-some-users)** for more details.
+
+To enable support for UnifiedPush for private servers (i.e. `auth-default-access: "deny-all"`), you should either 
+allow anonymous write access for the entire prefix or explicitly per topic:
+
+=== "Prefix"
+    ```
+    $ ntfy access '*' 'up*' write-only
+    ```
+
+=== "Explicitly"
+    ```
+    $ ntfy access '*' upYzMtZGZiYTY5 write-only
+    ```
+
 ## E-mail notifications
 To allow forwarding messages via e-mail, you can configure an **SMTP server for outgoing messages**. Once configured, 
 you can set the `X-Email` header to [send messages via e-mail](publish.md#e-mail-notifications) (e.g. 
@@ -806,19 +825,27 @@ out [this discussion on Reddit](https://www.reddit.com/r/golang/comments/r9u4ee/
 
 Depending on *how you run it*, here are a few limits that are relevant:
 
-### WAL for message cache
+### Message cache
 By default, the [message cache](#message-cache) (defined by `cache-file`) uses the SQLite default settings, which means it
 syncs to disk on every write. For personal servers, this is perfectly adequate. For larger installations, such as ntfy.sh,
 the [write-ahead log (WAL)](https://sqlite.org/wal.html) should be enabled, and the sync mode should be adjusted. 
 See [this article](https://phiresky.github.io/blog/2020/sqlite-performance-tuning/) for details.
 
+In addition to that, for very high load servers (such as ntfy.sh), it may be beneficial to write messages to the cache
+in batches, and asynchronously. This can be enabled with the `cache-batch-size` and `cache-batch-timeout`. If you start
+seeing `database locked` messages in the logs, you should probably enable that.
+
 Here's how ntfy.sh has been tuned in the `server.yml` file:
 
 ``` yaml
+cache-batch-size: 25
+cache-batch-timeout: "1s"
 cache-startup-queries: |
     pragma journal_mode = WAL;
     pragma synchronous = normal;
     pragma temp_store = memory;
+    pragma busy_timeout = 15000;
+    vacuum;
 ```
 
 ### For systemd services
@@ -880,7 +907,7 @@ and [here](https://easyengine.io/tutorials/nginx/block-wp-login-php-bruteforce-a
     ```
     # Rate limit all IP addresses
     http {
-	  limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s;
+	  limit_req_zone $binary_remote_addr zone=one:10m rate=45r/m;
     }
 
     # Alternatively, whitelist certain IP addresses
@@ -895,7 +922,7 @@ and [here](https://easyengine.io/tutorials/nginx/block-wp-login-php-bruteforce-a
         1 $binary_remote_addr;
         0 "";
       }
-      limit_req_zone $limitkey zone=one:10m rate=1r/s;
+      limit_req_zone $limitkey zone=one:10m rate=45r/m;
     }
     ```
 
@@ -924,7 +951,7 @@ and [here](https://easyengine.io/tutorials/nginx/block-wp-login-php-bruteforce-a
     action = iptables-multiport[name=ReqLimit, port="http,https", protocol=tcp]
     logpath = /var/log/nginx/error.log
     findtime = 600
-    bantime = 7200
+    bantime = 14400
     maxretry = 10
     ```
 
@@ -971,6 +998,8 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`).
 | `cache-file`                               | `NTFY_CACHE_FILE`                               | *filename*                                          | -                 | If set, messages are cached in a local SQLite database instead of only in-memory. This allows for service restarts without losing messages in support of the since= parameter. See [message cache](#message-cache).             |
 | `cache-duration`                           | `NTFY_CACHE_DURATION`                           | *duration*                                          | 12h               | Duration for which messages will be buffered before they are deleted. This is required to support the `since=...` and `poll=1` parameter. Set this to `0` to disable the cache entirely.                                        |
 | `cache-startup-queries`                    | `NTFY_CACHE_STARTUP_QUERIES`                    | *string (SQL queries)*                              | -                 | SQL queries to run during database startup; this is useful for tuning and [enabling WAL mode](#wal-for-message-cache)                                                                                                           |
+| `cache-batch-size`                         | `NTFY_CACHE_BATCH_SIZE`                         | *int*                                               | 0                 | Max size of messages to batch together when writing to message cache (if zero, writes are synchronous)                                                                                                                          |
+| `cache-batch-timeout`                      | `NTFY_CACHE_BATCH_TIMEOUT`                      | *duration*                                          | 0s                | Timeout for batched async writes to the message cache (if zero, writes are synchronous)                                                                                                                                         |
 | `auth-file`                                | `NTFY_AUTH_FILE`                                | *filename*                                          | -                 | Auth database file used for access control. If set, enables authentication and access control. See [access control](#access-control).                                                                                           |
 | `auth-default-access`                      | `NTFY_AUTH_DEFAULT_ACCESS`                      | `read-write`, `read-only`, `write-only`, `deny-all` | `read-write`      | Default permissions if no matching entries in the auth database are found. Default is `read-write`.                                                                                                                             |
 | `behind-proxy`                             | `NTFY_BEHIND_PROXY`                             | *bool*                                              | false             | If set, the X-Forwarded-For header is used to determine the visitor IP address instead of the remote address of the connection.                                                                                                 |
@@ -1035,6 +1064,8 @@ OPTIONS:
    --behind-proxy, --behind_proxy, -P                                                                  if set, use X-Forwarded-For header to determine visitor IP address (for rate limiting) (default: false) [$NTFY_BEHIND_PROXY]
    --cache-duration since, --cache_duration since, -b since                                            buffer messages for this time to allow since requests (default: 12h0m0s) [$NTFY_CACHE_DURATION]
    --cache-file value, --cache_file value, -C value                                                    cache file used for message caching [$NTFY_CACHE_FILE]
+   --cache-batch-size value, --cache_batch_size value                                                  max size of messages to batch together when writing to message cache (if zero, writes are synchronous) (default: 0) [$NTFY_BATCH_SIZE]
+   --cache-batch-timeout value, --cache_batch_timeout value                                            timeout for batched async writes to the message cache (if zero, writes are synchronous) (default: 0s) [$NTFY_CACHE_BATCH_TIMEOUT]   
    --cache-startup-queries value, --cache_startup_queries value                                        queries run when the cache database is initialized [$NTFY_CACHE_STARTUP_QUERIES]
    --cert-file value, --cert_file value, -E value                                                      certificate file, if listen-https is set [$NTFY_CERT_FILE]
    --config value, -c value                                                                            config file (default: /etc/ntfy/server.yml) [$NTFY_CONFIG_FILE]

docs/examples.md

@@ -122,6 +122,19 @@ to ntfy at its default URL (`attrs` and other attributes are optional):
           priority: 1
 ```
 
+## GitHub Actions
+You can send a message during a workflow run with curl. Here is an example sending info about the repo, commit and job status.
+``` yaml
+- name: Actions Ntfy
+  run: |
+    curl \
+      -u ${{ secrets.NTFY_CRED }} \
+      -H "Title: Title here" \
+      -H "Content-Type: text/plain" \
+      -d $'Repo: ${{ github.repository }}\nCommit: ${{ github.sha }}\nRef: ${{ github.ref }}\nStatus: ${{ job.status}}' \
+      ${{ secrets.NTFY_URL }}
+```
+
 ## Watchtower (shoutrrr)
 You can use [shoutrrr](https://github.com/containrrr/shoutrrr) generic webhook support to send 
 [Watchtower](https://github.com/containrrr/watchtower/) notifications to your ntfy topic.

docs/install.md

@@ -26,37 +26,37 @@ deb/rpm packages.
 
 === "x86_64/amd64"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_x86_64.tar.gz
-    tar zxvf ntfy_1.29.0_linux_x86_64.tar.gz
-    sudo cp -a ntfy_1.29.0_linux_x86_64/ntfy /usr/bin/ntfy
-    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.0_linux_x86_64/{client,server}/*.yml /etc/ntfy
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_x86_64.tar.gz
+    tar zxvf ntfy_1.29.1_linux_x86_64.tar.gz
+    sudo cp -a ntfy_1.29.1_linux_x86_64/ntfy /usr/bin/ntfy
+    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.1_linux_x86_64/{client,server}/*.yml /etc/ntfy
     sudo ntfy serve
     ```
 
 === "armv6"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv6.tar.gz
-    tar zxvf ntfy_1.29.0_linux_armv6.tar.gz
-    sudo cp -a ntfy_1.29.0_linux_armv6/ntfy /usr/bin/ntfy
-    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.0_linux_armv6/{client,server}/*.yml /etc/ntfy
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv6.tar.gz
+    tar zxvf ntfy_1.29.1_linux_armv6.tar.gz
+    sudo cp -a ntfy_1.29.1_linux_armv6/ntfy /usr/bin/ntfy
+    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.1_linux_armv6/{client,server}/*.yml /etc/ntfy
     sudo ntfy serve
     ```
 
 === "armv7/armhf"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv7.tar.gz
-    tar zxvf ntfy_1.29.0_linux_armv7.tar.gz
-    sudo cp -a ntfy_1.29.0_linux_armv7/ntfy /usr/bin/ntfy
-    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.0_linux_armv7/{client,server}/*.yml /etc/ntfy
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv7.tar.gz
+    tar zxvf ntfy_1.29.1_linux_armv7.tar.gz
+    sudo cp -a ntfy_1.29.1_linux_armv7/ntfy /usr/bin/ntfy
+    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.1_linux_armv7/{client,server}/*.yml /etc/ntfy
     sudo ntfy serve
     ```
 
 === "arm64"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_arm64.tar.gz
-    tar zxvf ntfy_1.29.0_linux_arm64.tar.gz
-    sudo cp -a ntfy_1.29.0_linux_arm64/ntfy /usr/bin/ntfy
-    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.0_linux_arm64/{client,server}/*.yml /etc/ntfy
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_arm64.tar.gz
+    tar zxvf ntfy_1.29.1_linux_arm64.tar.gz
+    sudo cp -a ntfy_1.29.1_linux_arm64/ntfy /usr/bin/ntfy
+    sudo mkdir /etc/ntfy && sudo cp ntfy_1.29.1_linux_arm64/{client,server}/*.yml /etc/ntfy
     sudo ntfy serve
     ```
 
@@ -106,7 +106,7 @@ Manually installing the .deb file:
 
 === "x86_64/amd64"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_amd64.deb
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_amd64.deb
     sudo dpkg -i ntfy_*.deb
     sudo systemctl enable ntfy
     sudo systemctl start ntfy
@@ -114,7 +114,7 @@ Manually installing the .deb file:
 
 === "armv6"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv6.deb
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv6.deb
     sudo dpkg -i ntfy_*.deb
     sudo systemctl enable ntfy
     sudo systemctl start ntfy
@@ -122,7 +122,7 @@ Manually installing the .deb file:
 
 === "armv7/armhf"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv7.deb
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv7.deb
     sudo dpkg -i ntfy_*.deb
     sudo systemctl enable ntfy
     sudo systemctl start ntfy
@@ -130,7 +130,7 @@ Manually installing the .deb file:
 
 === "arm64"
     ```bash
-    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_arm64.deb
+    wget https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_arm64.deb
     sudo dpkg -i ntfy_*.deb
     sudo systemctl enable ntfy
     sudo systemctl start ntfy
@@ -140,28 +140,28 @@ Manually installing the .deb file:
 
 === "x86_64/amd64"
     ```bash
-    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_amd64.rpm
+    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_amd64.rpm
     sudo systemctl enable ntfy 
     sudo systemctl start ntfy
     ```
 
 === "armv6"
     ```bash
-    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv6.rpm
+    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv6.rpm
     sudo systemctl enable ntfy
     sudo systemctl start ntfy
     ```
 
 === "armv7/armhf"
     ```bash
-    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_armv7.rpm
+    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_armv7.rpm
     sudo systemctl enable ntfy 
     sudo systemctl start ntfy
     ```
 
 === "arm64"
     ```bash
-    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_linux_arm64.rpm
+    sudo rpm -ivh https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_linux_arm64.rpm
     sudo systemctl enable ntfy 
     sudo systemctl start ntfy
     ```
@@ -189,18 +189,18 @@ NixOS also supports [declarative setup of the ntfy server](https://search.nixos.
 
 ## macOS
 The [ntfy CLI](subscribe/cli.md) (`ntfy publish` and `ntfy subscribe` only) is supported on macOS as well. 
-To install, please [download the tarball](https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_macOS_all.tar.gz), 
+To install, please [download the tarball](https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_macOS_all.tar.gz), 
 extract it and place it somewhere in your `PATH` (e.g. `/usr/local/bin/ntfy`). 
 
 If run as `root`, ntfy will look for its config at `/etc/ntfy/client.yml`. For all other users, it'll look for it at 
 `~/Library/Application Support/ntfy/client.yml` (sample included in the tarball).
 
 ```bash
-curl -L https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_macOS_all.tar.gz > ntfy_1.29.0_macOS_all.tar.gz
-tar zxvf ntfy_1.29.0_macOS_all.tar.gz
-sudo cp -a ntfy_1.29.0_macOS_all/ntfy /usr/local/bin/ntfy
+curl -L https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_macOS_all.tar.gz > ntfy_1.29.1_macOS_all.tar.gz
+tar zxvf ntfy_1.29.1_macOS_all.tar.gz
+sudo cp -a ntfy_1.29.1_macOS_all/ntfy /usr/local/bin/ntfy
 mkdir ~/Library/Application\ Support/ntfy 
-cp ntfy_1.29.0_macOS_all/client/client.yml ~/Library/Application\ Support/ntfy/client.yml
+cp ntfy_1.29.1_macOS_all/client/client.yml ~/Library/Application\ Support/ntfy/client.yml
 ntfy --help
 ```
 
@@ -212,7 +212,7 @@ ntfy --help
 
 ## Windows
 The [ntfy CLI](subscribe/cli.md) (`ntfy publish` and `ntfy subscribe` only) is supported on Windows as well.
-To install, please [download the latest ZIP](https://github.com/binwiederhier/ntfy/releases/download/v1.29.0/ntfy_1.29.0_windows_x86_64.zip),
+To install, please [download the latest ZIP](https://github.com/binwiederhier/ntfy/releases/download/v1.29.1/ntfy_1.29.1_windows_x86_64.zip),
 extract it and place the `ntfy.exe` binary somewhere in your `%Path%`. 
 
 The default path for the client config file is at `%AppData%\ntfy\client.yml` (not created automatically, sample in the ZIP file).
@@ -300,8 +300,8 @@ This image can be pushed to a container registry and shipped independently. All
 ## Kubernetes
 
 The setup for Kubernetes is very similar to that for Docker, and requires a fairly minimal deployment or pod definition to function. There
-are a few options to mix and match, including a deployment without a cache file, a stateful set with a persistant cache, and a standalone
-unmanaged pod.
+are a few options to mix and match, including a deployment without a cache file, a stateful set with a persistent cache, and a standalone
+unmanned pod.
 
 
 === "deployment"
@@ -422,7 +422,7 @@ unmanaged pod.
             name: ntfy
     ```
 
-Configuration is relatively straightforward. As an exmaple, a minimal configuration is provided.
+Configuration is relatively straightforward. As an example, a minimal configuration is provided.
 
 === "resource definition"
     ```yaml
@@ -431,7 +431,7 @@ Configuration is relatively straightforward. As an exmaple, a minimal configurat
     metadata:
       name: ntfy
     data:
-    server.yml: |
+      server.yml: |
         # Template: https://github.com/binwiederhier/ntfy/blob/main/server/server.yml
         base-url: https://ntfy.sh
     ```
@@ -440,3 +440,154 @@ Configuration is relatively straightforward. As an exmaple, a minimal configurat
     ```bash
     kubectl create configmap ntfy --from-file=server.yml 
     ```
+
+## Kustomize
+
+ntfy can be deployed in a Kubernetes cluster with [Kustomize](https://github.com/kubernetes-sigs/kustomize), a tool used
+to customize Kubernetes objects using a `kustomization.yaml` file.
+
+1. Create new folder - `ntfy`
+2. Add all files listed below 
+    1. `kustomization.yaml` - stores all configmaps and resources used in a deployment
+    2. `ntfy-deployment.yaml` - define deployment type and its parameters
+    3. `ntfy-pvc.yaml` - describes how [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) will be created 
+    4. `ntfy-svc.yaml` - expose application to the internal kubernetes network
+    5. `ntfy-ingress.yaml` - expose service to outside the network using [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/)
+    6. `server.yaml` - simple server configuration
+3. Replace **TESTNAMESPACE** within `kustomization.yaml` with designated namespace 
+4. Replace **ntfy.test** within `ntfy-ingress.yaml` with desired DNS name
+5. Apply configuration to cluster set in current context: 
+
+```bash
+kubectl apply -k /ntfy
+```
+
+=== "kustomization.yaml"
+    ```yaml
+    apiVersion: kustomize.config.k8s.io/v1beta1
+    kind: Kustomization
+    resources:
+      - ntfy-deployment.yaml # deployment definition
+      - ntfy-svc.yaml # service connecting pods to cluster network
+      - ntfy-pvc.yaml # pvc used to store cache and attachment
+      - ntfy-ingress.yaml # ingress definition
+    configMapGenerator: # will parse config from raw config to configmap,it allows for dynamic reload of application if additional app is deployed ie https://github.com/stakater/Reloader
+        - name: server-config
+          files: 
+            - server.yml
+    namespace: TESTNAMESPACE # select namespace for whole application 
+    ```
+=== "ntfy-deployment.yaml"
+    ```yaml
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: ntfy-deployment
+      labels:
+        app: ntfy-deployment
+    spec:
+      revisionHistoryLimit: 1
+      replicas: 1
+      selector:
+        matchLabels:
+          app: ntfy-pod
+      template:
+        metadata:
+          labels:
+            app: ntfy-pod
+        spec:
+          containers:
+            - name: ntfy 
+              image: binwiederhier/ntfy:v1.28.0 # set deployed version
+              args: ["serve"]
+              env:  #example of adjustments made in environmental variables
+                - name: TZ # set timezone
+                  value: XXXXXXX
+                - name: NTFY_DEBUG # enable/disable debug
+                  value: "false"
+                - name: NTFY_LOG_LEVEL # adjust log level
+                  value: INFO
+                - name: NTFY_BASE_URL # add base url
+                  value: XXXXXXXXXX 
+              ports: 
+                - containerPort: 80
+                  name: http-ntfy
+              resources:
+                limits:
+                  memory: 300Mi
+                  cpu:  200m
+                requests:
+                      cpu: 150m
+                      memory: 150Mi
+              volumeMounts:
+                  - mountPath: /etc/ntfy/server.yml
+                    subPath: server.yml
+                    name: config-volume # generated vie configMapGenerator from kustomization file
+                  - mountPath: /var/cache/ntfy
+                    name: cache-volume #cache volume mounted to persistent volume
+            volumes:
+              - name: config-volume
+                configMap:  # uses configmap generator to parse server.yml to configmap
+                  name: server-config
+              - name: cache-volume
+                persistentVolumeClaim: # stores /cache/ntfy in defined pv
+                  claimName: ntfy-pvc
+    ```
+  
+=== "ntfy-pvc.yaml"
+    ```yaml
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: ntfy-pvc
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      storageClassName: local-path # adjust storage if needed
+      resources:
+        requests:
+          storage: 1Gi
+    ```
+
+=== "ntfy-svc.yaml"
+    ```yaml
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ntfy-svc  
+    spec:
+      type: ClusterIP
+      selector:
+        app: ntfy-pod
+      ports:
+        - name: http-ntfy-out
+          protocol: TCP
+          port: 80
+          targetPort:  http-ntfy
+    ```
+
+=== "ntfy-ingress.yaml"
+    ```yaml
+    apiVersion: networking.k8s.io/v1
+    kind: Ingress
+    metadata:
+      name: ntfy-ingress
+    spec:
+      rules:
+        - host: ntfy.test #select own
+          http:
+            paths:
+              - path: /
+                pathType: Prefix
+                backend:
+                  service:
+                    name:  ntfy-svc
+                    port:
+                      number: 80
+    ```
+
+=== "server.yml"
+    ```yaml
+    cache-file: "/var/cache/ntfy/cache.db"
+    attachment-cache-dir: "/var/cache/ntfy/attachments"
+    ```

docs/integrations.md

@@ -47,6 +47,7 @@ messages until I finally finish implementing end-to-end encryption.
 - [ntfy-middleman](https://github.com/nachotp/ntfy-middleman) - Wraps APIs and send notifications using ntfy.sh on schedule (Python)
 - [ntfy-dotnet](https://github.com/nwithan8/ntfy-dotnet) - .NET client library to interact with a ntfy server (C# / .NET)
 - [node-ntfy-publish](https://github.com/cityssm/node-ntfy-publish) - A Node package to publish notifications to an ntfy server (Node)
+- [ntfy](https://github.com/jonocarroll/ntfy) - Wraps the ntfy API with pipe-friendly tooling (R)
 
 ## CLIs + GUIs
 
@@ -93,6 +94,10 @@ messages until I finally finish implementing end-to-end encryption.
 
 ## Blog + forum posts
 
+- [Tracking layoffs, tech worker demand still high, ntfy, devenv, Markdoc & Mike Bifulco](https://changelog.com/news/tracking-layoffs-tech-worker-demand-still-high-ntfy-devenv-markdoc-mike-bifulco-Y1jW) - 11/2022
+- [Pointer | Issue #367](https://www.pointer.io/archives/a9495a2a6f/) - 11/2022
+- [Envie Push Notifications por POST (de graça e sem cadastro)](https://www.tabnews.com.br/filipedeschamps/envie-push-notifications-por-post-de-graca-e-sem-cadastro) - 11/2022
+- [Push Notifications for KDE](https://volkerkrause.eu/2022/11/12/kde-unifiedpush-push-notifications.html) - 11/2022
 - [TLDR Newsletter Daily Update 2022-11-09](https://tldr.tech/tech/newsletter/2022-11-09) - 11/2022
 - [Ntfy.sh – Send push notifications to your phone via PUT/POST](https://news.ycombinator.com/item?id=33517944) ⭐ - 11/2022
 - [Ntfy et Jeedom : un plugin](https://lunarok-domotique.com/2022/11/ntfy-et-jeedom/) - 11/2022

docs/releases.md

@@ -2,13 +2,38 @@
 Binaries for all releases can be found on the GitHub releases pages for the [ntfy server](https://github.com/binwiederhier/ntfy/releases)
 and the [ntfy Android app](https://github.com/binwiederhier/ntfy-android/releases).
 
-<!--
 ## ntfy Android app v1.14.0 (UNRELEASED)
 
+**Bug fixes:**
+
+* Remove timestamp when copying message text ([#471](https://github.com/binwiederhier/ntfy/issues/471), thanks to [@wunter8](https://github.com/wunter8))
+
 **Additional translations:**
 
 * Korean (thanks to [@YJSofta0f97461d82447ac](https://hosted.weblate.org/user/YJSofta0f97461d82447ac/))
--->
+
+## ntfy server v1.29.1
+Released November 17, 2022
+
+This is mostly a bugfix release to address the high load on ntfy.sh. There are now two new options that allow
+synchronous batch-writing of messages to the cache. This avoids database locking, and subsequent pileups of waiting
+requests.
+
+**Bug fixes:**
+
+* High-load servers: Allow asynchronous batch-writing of messages to cache via `cache-batch-*` options ([#498](https://github.com/binwiederhier/ntfy/issues/498)/[#502](https://github.com/binwiederhier/ntfy/pull/502))
+* Sender column in cache.db shows invalid IP ([#503](https://github.com/binwiederhier/ntfy/issues/503))
+
+**Documentation:**
+
+* GitHub Actions example ([#492](https://github.com/binwiederhier/ntfy/pull/492), thanks to [@ksurl](https://github.com/ksurl))
+* UnifiedPush ACL clarification ([#497](https://github.com/binwiederhier/ntfy/issues/497), thanks to [@bt90](https://github.com/bt90)) 
+* Install instructions for Kustomize ([#463](https://github.com/binwiederhier/ntfy/pull/463), thanks to [@l-maciej](https://github.com/l-maciej))
+
+**Other things:**
+
+* Put ntfy.sh docs on GitHub pages to reduce AWS outbound traffic cost ([#491](https://github.com/binwiederhier/ntfy/issues/491))
+* The ntfy.sh server hardware was upgraded to a bigger box. If you'd like to help out carrying the server cost, **[sponsorships and donations](https://github.com/sponsors/binwiederhier)** 💸 would be very much appreciated
 
 ## ntfy server v1.29.0
 Released November 12, 2022

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/olebedev/when v0.0.0-20211212231525-59bd4edcf9d6
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.23.5
-	golang.org/x/crypto v0.2.0
+	golang.org/x/crypto v0.3.0
 	golang.org/x/oauth2 v0.2.0 // indirect
 	golang.org/x/sync v0.1.0
 	golang.org/x/term v0.2.0
@@ -28,7 +28,7 @@ require github.com/pkg/errors v0.9.1 // indirect
 require firebase.google.com/go/v4 v4.10.0
 
 require (
-	cloud.google.com/go v0.106.0 // indirect
+	cloud.google.com/go v0.107.0 // indirect
 	cloud.google.com/go/compute v1.12.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	cloud.google.com/go/iam v0.7.0 // indirect
@@ -54,7 +54,7 @@ require (
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/appengine/v2 v2.0.2 // indirect
-	google.golang.org/genproto v0.0.0-20221111202108-142d8a6fa32e // indirect
+	google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029 // indirect
 	google.golang.org/grpc v1.50.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,6 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.106.0 h1:AWaMWuZb2oFeiV91OfNHZbmwUhMVuXEaLPm9sqDAOl8=
 cloud.google.com/go v0.106.0/go.mod h1:5NEGxGuIeMQiPaWLwLYZ7kfNWiP6w1+QJK+xqyIT+dw=
+cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww=
+cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I=
 cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
 cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
 cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
@@ -111,6 +113,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
 golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -174,6 +178,10 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20221111202108-142d8a6fa32e h1:azcyH5lGzGy7pkLCbhPe0KkKxsM7c6UA/FZIXImKE7M=
 google.golang.org/genproto v0.0.0-20221111202108-142d8a6fa32e/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
+google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472 h1:kIfItBRE5gkUKpH4H5lNGciZbka1JrmRli3ArqrKFkA=
+google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
+google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029 h1:zS8DNtiDX68/osEpazR86KM1vnDELdnRgpK6/fwlQTs=
+google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

server/config.go

@@ -61,6 +61,8 @@ type Config struct {
 	CacheFile                            string
 	CacheDuration                        time.Duration
 	CacheStartupQueries                  string
+	CacheBatchSize                       int
+	CacheBatchTimeout                    time.Duration
 	AuthFile                             string
 	AuthDefaultRead                      bool
 	AuthDefaultWrite                     bool
@@ -114,6 +116,8 @@ func NewConfig() *Config {
 		FirebaseKeyFile:                      "",
 		CacheFile:                            "",
 		CacheDuration:                        DefaultCacheDuration,
+		CacheBatchSize:                       0,
+		CacheBatchTimeout:                    0,
 		AuthFile:                             "",
 		AuthDefaultRead:                      true,
 		AuthDefaultWrite:                     true,

server/message_cache.go

@@ -44,6 +44,7 @@ const (
 			published INT NOT NULL
 		);
 		CREATE INDEX IF NOT EXISTS idx_mid ON messages (mid);
+		CREATE INDEX IF NOT EXISTS idx_time ON messages (time);
 		CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic);
 		COMMIT;
 	`
@@ -92,7 +93,7 @@ const (
 
 // Schema management queries
 const (
-	currentSchemaVersion          = 8
+	currentSchemaVersion          = 9
 	createSchemaVersionTableQuery = `
 		CREATE TABLE IF NOT EXISTS schemaVersion (
 			id INT PRIMARY KEY,
@@ -185,15 +186,21 @@ const (
 	migrate7To8AlterMessagesTableQuery = `
 		ALTER TABLE messages ADD COLUMN icon TEXT NOT NULL DEFAULT('');
 	`
+
+	// 8 -> 9
+	migrate8To9AlterMessagesTableQuery = `
+		CREATE INDEX IF NOT EXISTS idx_time ON messages (time);	
+	`
 )
 
 type messageCache struct {
-	db  *sql.DB
-	nop bool
+	db    *sql.DB
+	queue *util.BatchingQueue[*message]
+	nop   bool
 }
 
 // newSqliteCache creates a SQLite file-backed cache
-func newSqliteCache(filename, startupQueries string, nop bool) (*messageCache, error) {
+func newSqliteCache(filename, startupQueries string, batchSize int, batchTimeout time.Duration, nop bool) (*messageCache, error) {
 	db, err := sql.Open("sqlite3", filename)
 	if err != nil {
 		return nil, err
@@ -201,21 +208,28 @@ func newSqliteCache(filename, startupQueries string, nop bool) (*messageCache, e
 	if err := setupCacheDB(db, startupQueries); err != nil {
 		return nil, err
 	}
-	return &messageCache{
-		db:  db,
-		nop: nop,
-	}, nil
+	var queue *util.BatchingQueue[*message]
+	if batchSize > 0 || batchTimeout > 0 {
+		queue = util.NewBatchingQueue[*message](batchSize, batchTimeout)
+	}
+	cache := &messageCache{
+		db:    db,
+		queue: queue,
+		nop:   nop,
+	}
+	go cache.processMessageBatches()
+	return cache, nil
 }
 
 // newMemCache creates an in-memory cache
 func newMemCache() (*messageCache, error) {
-	return newSqliteCache(createMemoryFilename(), "", false)
+	return newSqliteCache(createMemoryFilename(), "", 0, 0, false)
 }
 
 // newNopCache creates an in-memory cache that discards all messages;
 // it is always empty and can be used if caching is entirely disabled
 func newNopCache() (*messageCache, error) {
-	return newSqliteCache(createMemoryFilename(), "", true)
+	return newSqliteCache(createMemoryFilename(), "", 0, 0, true)
 }
 
 // createMemoryFilename creates a unique memory filename to use for the SQLite backend.
@@ -228,14 +242,23 @@ func createMemoryFilename() string {
 	return fmt.Sprintf("file:%s?mode=memory&cache=shared", util.RandomString(10))
 }
 
+// AddMessage stores a message to the message cache synchronously, or queues it to be stored at a later date asyncronously.
+// The message is queued only if "batchSize" or "batchTimeout" are passed to the constructor.
 func (c *messageCache) AddMessage(m *message) error {
+	if c.queue != nil {
+		c.queue.Enqueue(m)
+		return nil
+	}
 	return c.addMessages([]*message{m})
 }
 
+// addMessages synchronously stores a match of messages. If the database is locked, the transaction waits until
+// SQLite's busy_timeout is exceeded before erroring out.
 func (c *messageCache) addMessages(ms []*message) error {
 	if c.nop {
 		return nil
 	}
+	start := time.Now()
 	tx, err := c.db.Begin()
 	if err != nil {
 		return err
@@ -264,6 +287,10 @@ func (c *messageCache) addMessages(ms []*message) error {
 			}
 			actionsStr = string(actionsBytes)
 		}
+		var sender string
+		if m.Sender.IsValid() {
+			sender = m.Sender.String()
+		}
 		_, err := tx.Exec(
 			insertMessageQuery,
 			m.ID,
@@ -281,7 +308,7 @@ func (c *messageCache) addMessages(ms []*message) error {
 			attachmentSize,
 			attachmentExpires,
 			attachmentURL,
-			m.Sender.String(),
+			sender,
 			m.Encoding,
 			published,
 		)
@@ -289,7 +316,12 @@ func (c *messageCache) addMessages(ms []*message) error {
 			return err
 		}
 	}
-	return tx.Commit()
+	if err := tx.Commit(); err != nil {
+		log.Error("Cache: Writing %d message(s) failed (took %v)", len(ms), time.Since(start))
+		return err
+	}
+	log.Debug("Cache: Wrote %d message(s) in %v", len(ms), time.Since(start))
+	return nil
 }
 
 func (c *messageCache) Messages(topic string, since sinceMarker, scheduled bool) ([]*message, error) {
@@ -395,8 +427,12 @@ func (c *messageCache) Topics() (map[string]*topic, error) {
 }
 
 func (c *messageCache) Prune(olderThan time.Time) error {
-	_, err := c.db.Exec(pruneMessagesQuery, olderThan.Unix())
-	return err
+	start := time.Now()
+	if _, err := c.db.Exec(pruneMessagesQuery, olderThan.Unix()); err != nil {
+		log.Warn("Cache: Pruning failed (after %v): %s", time.Since(start), err.Error())
+	}
+	log.Debug("Cache: Pruning successful (took %v)", time.Since(start))
+	return nil
 }
 
 func (c *messageCache) AttachmentBytesUsed(sender string) (int64, error) {
@@ -417,6 +453,17 @@ func (c *messageCache) AttachmentBytesUsed(sender string) (int64, error) {
 	return size, nil
 }
 
+func (c *messageCache) processMessageBatches() {
+	if c.queue == nil {
+		return
+	}
+	for messages := range c.queue.Dequeue() {
+		if err := c.addMessages(messages); err != nil {
+			log.Error("Cache: %s", err.Error())
+		}
+	}
+}
+
 func readMessages(rows *sql.Rows) ([]*message, error) {
 	defer rows.Close()
 	messages := make([]*message, 0)
@@ -458,9 +505,8 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
 		}
 		senderIP, err := netip.ParseAddr(sender)
 		if err != nil {
-			senderIP = netip.IPv4Unspecified() // if no IP stored in database, 0.0.0.0
+			senderIP = netip.Addr{} // if no IP stored in database, return invalid address
 		}
-
 		var att *attachment
 		if attachmentName != "" && attachmentURL != "" {
 			att = &attachment{
@@ -542,6 +588,8 @@ func setupCacheDB(db *sql.DB, startupQueries string) error {
 		return migrateFrom6(db)
 	} else if schemaVersion == 7 {
 		return migrateFrom7(db)
+	} else if schemaVersion == 8 {
+		return migrateFrom8(db)
 	}
 	return fmt.Errorf("unexpected schema version found: %d", schemaVersion)
 }
@@ -647,5 +695,16 @@ func migrateFrom7(db *sql.DB) error {
 	if _, err := db.Exec(updateSchemaVersion, 8); err != nil {
 		return err
 	}
+	return migrateFrom8(db)
+}
+
+func migrateFrom8(db *sql.DB) error {
+	log.Info("Migrating cache database schema: from 8 to 9")
+	if _, err := db.Exec(migrate8To9AlterMessagesTableQuery); err != nil {
+		return err
+	}
+	if _, err := db.Exec(updateSchemaVersion, 9); err != nil {
+		return err
+	}
 	return nil // Update this when a new version is added
 }

server/message_cache_test.go

@@ -450,7 +450,7 @@ func TestSqliteCache_StartupQueries_WAL(t *testing.T) {
 	startupQueries := `pragma journal_mode = WAL; 
 pragma synchronous = normal; 
 pragma temp_store = memory;`
-	db, err := newSqliteCache(filename, startupQueries, false)
+	db, err := newSqliteCache(filename, startupQueries, 0, 0, false)
 	require.Nil(t, err)
 	require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
 	require.FileExists(t, filename)
@@ -461,7 +461,7 @@ pragma temp_store = memory;`
 func TestSqliteCache_StartupQueries_None(t *testing.T) {
 	filename := newSqliteTestCacheFile(t)
 	startupQueries := ""
-	db, err := newSqliteCache(filename, startupQueries, false)
+	db, err := newSqliteCache(filename, startupQueries, 0, 0, false)
 	require.Nil(t, err)
 	require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
 	require.FileExists(t, filename)
@@ -472,10 +472,33 @@ func TestSqliteCache_StartupQueries_None(t *testing.T) {
 func TestSqliteCache_StartupQueries_Fail(t *testing.T) {
 	filename := newSqliteTestCacheFile(t)
 	startupQueries := `xx error`
-	_, err := newSqliteCache(filename, startupQueries, false)
+	_, err := newSqliteCache(filename, startupQueries, 0, 0, false)
 	require.Error(t, err)
 }
 
+func TestSqliteCache_Sender(t *testing.T) {
+	testSender(t, newSqliteTestCache(t))
+}
+
+func TestMemCache_Sender(t *testing.T) {
+	testSender(t, newMemTestCache(t))
+}
+
+func testSender(t *testing.T, c *messageCache) {
+	m1 := newDefaultMessage("mytopic", "mymessage")
+	m1.Sender = netip.MustParseAddr("1.2.3.4")
+	require.Nil(t, c.AddMessage(m1))
+
+	m2 := newDefaultMessage("mytopic", "mymessage without sender")
+	require.Nil(t, c.AddMessage(m2))
+
+	messages, err := c.Messages("mytopic", sinceAllMessages, false)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(messages))
+	require.Equal(t, messages[0].Sender, netip.MustParseAddr("1.2.3.4"))
+	require.Equal(t, messages[1].Sender, netip.Addr{})
+}
+
 func checkSchemaVersion(t *testing.T, db *sql.DB) {
 	rows, err := db.Query(`SELECT version FROM schemaVersion`)
 	require.Nil(t, err)
@@ -501,7 +524,7 @@ func TestMemCache_NopCache(t *testing.T) {
 }
 
 func newSqliteTestCache(t *testing.T) *messageCache {
-	c, err := newSqliteCache(newSqliteTestCacheFile(t), "", false)
+	c, err := newSqliteCache(newSqliteTestCacheFile(t), "", 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -513,7 +536,7 @@ func newSqliteTestCacheFile(t *testing.T) string {
 }
 
 func newSqliteTestCacheFromFile(t *testing.T, filename, startupQueries string) *messageCache {
-	c, err := newSqliteCache(filename, startupQueries, false)
+	c, err := newSqliteCache(filename, startupQueries, 0, 0, false)
 	if err != nil {
 		t.Fatal(err)
 	}

server/server.go

@@ -159,7 +159,7 @@ func createMessageCache(conf *Config) (*messageCache, error) {
 	if conf.CacheDuration == 0 {
 		return newNopCache()
 	} else if conf.CacheFile != "" {
-		return newSqliteCache(conf.CacheFile, conf.CacheStartupQueries, false)
+		return newSqliteCache(conf.CacheFile, conf.CacheStartupQueries, conf.CacheBatchSize, conf.CacheBatchTimeout, false)
 	}
 	return newMemCache()
 }
@@ -491,6 +491,7 @@ func (s *Server) handlePublishWithoutResponse(r *http.Request, v *visitor) (*mes
 		log.Debug("%s Message delayed, will process later", logMessagePrefix(v, m))
 	}
 	if cache {
+		log.Debug("%s Adding message to cache", logMessagePrefix(v, m))
 		if err := s.messageCache.AddMessage(m); err != nil {
 			return nil, err
 		}

server/server.yml

@@ -53,6 +53,12 @@
 #       pragma journal_mode = WAL;
 #       pragma synchronous = normal;
 #       pragma temp_store = memory;
+#       pragma busy_timeout = 15000;
+#       vacuum;
+#
+# The "cache-batch-size" and "cache-batch-timeout" parameter allow enabling async batch writing
+# of messages. If set, messages will be queued and written to the database in batches of the given
+# size, or after the given timeout. This is only required for high volume servers.
 #
 # Debian/RPM package users:
 #   Use /var/cache/ntfy/cache.db as cache file to avoid permission issues. The package
@@ -65,6 +71,8 @@
 # cache-file: <filename>
 # cache-duration: "12h"
 # cache-startup-queries:
+# cache-batch-size: 0
+# cache-batch-timeout: "0ms"
 
 # If set, access to the ntfy server and API can be controlled on a granular level using
 # the 'ntfy user' and 'ntfy access' commands. See the --help pages for details, or check the docs.
@@ -173,8 +181,9 @@
 # Rate limiting: Allowed GET/PUT/POST requests per second, per visitor:
 # - visitor-request-limit-burst is the initial bucket of requests each visitor has
 # - visitor-request-limit-replenish is the rate at which the bucket is refilled
-# - visitor-request-limit-exempt-hosts is a comma-separated list of hostnames and IPs to be
-#   exempt from request rate limiting; hostnames are resolved at the time the server is started
+# - visitor-request-limit-exempt-hosts is a comma-separated list of hostnames, IPs or CIDRs to be
+#   exempt from request rate limiting. Hostnames are resolved at the time the server is started.
+#   Example: "1.2.3.4,ntfy.example.com,8.7.6.0/24"
 #
 # visitor-request-limit-burst: 60
 # visitor-request-limit-replenish: "5s"

util/batching_queue.go

@@ -0,0 +1,86 @@
+package util
+
+import (
+	"sync"
+	"time"
+)
+
+// BatchingQueue is a queue that creates batches of the enqueued elements based on a
+// max batch size and a batch timeout.
+//
+// Example:
+//
+//	q := NewBatchingQueue[int](2, 500 * time.Millisecond)
+//	go func() {
+//	  for batch := range q.Dequeue() {
+//	    fmt.Println(batch)
+//	  }
+//	}()
+//	q.Enqueue(1)
+//	q.Enqueue(2)
+//	q.Enqueue(3)
+//	time.Sleep(time.Second)
+//
+// This example will emit batch [1, 2] immediately (because the batch size is 2), and
+// a batch [3] after 500ms.
+type BatchingQueue[T any] struct {
+	batchSize int
+	timeout   time.Duration
+	in        []T
+	out       chan []T
+	mu        sync.Mutex
+}
+
+// NewBatchingQueue creates a new BatchingQueue
+func NewBatchingQueue[T any](batchSize int, timeout time.Duration) *BatchingQueue[T] {
+	q := &BatchingQueue[T]{
+		batchSize: batchSize,
+		timeout:   timeout,
+		in:        make([]T, 0),
+		out:       make(chan []T),
+	}
+	go q.timeoutTicker()
+	return q
+}
+
+// Enqueue enqueues an element to the queue. If the configured batch size is reached,
+// the batch will be emitted immediately.
+func (q *BatchingQueue[T]) Enqueue(element T) {
+	q.mu.Lock()
+	q.in = append(q.in, element)
+	var elements []T
+	if len(q.in) == q.batchSize {
+		elements = q.dequeueAll()
+	}
+	q.mu.Unlock()
+	if len(elements) > 0 {
+		q.out <- elements
+	}
+}
+
+// Dequeue returns a channel emitting batches of elements
+func (q *BatchingQueue[T]) Dequeue() <-chan []T {
+	return q.out
+}
+
+func (q *BatchingQueue[T]) dequeueAll() []T {
+	elements := make([]T, len(q.in))
+	copy(elements, q.in)
+	q.in = q.in[:0]
+	return elements
+}
+
+func (q *BatchingQueue[T]) timeoutTicker() {
+	if q.timeout == 0 {
+		return
+	}
+	ticker := time.NewTicker(q.timeout)
+	for range ticker.C {
+		q.mu.Lock()
+		elements := q.dequeueAll()
+		q.mu.Unlock()
+		if len(elements) > 0 {
+			q.out <- elements
+		}
+	}
+}

util/batching_queue_test.go

@@ -0,0 +1,58 @@
+package util_test
+
+import (
+	"github.com/stretchr/testify/require"
+	"heckel.io/ntfy/util"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestBatchingQueue_InfTimeout(t *testing.T) {
+	q := util.NewBatchingQueue[int](25, 1*time.Hour)
+	batches, total := make([][]int, 0), 0
+	var mu sync.Mutex
+	go func() {
+		for batch := range q.Dequeue() {
+			mu.Lock()
+			batches = append(batches, batch)
+			total += len(batch)
+			mu.Unlock()
+		}
+	}()
+	for i := 0; i < 101; i++ {
+		go q.Enqueue(i)
+	}
+	time.Sleep(time.Second)
+	mu.Lock()
+	require.Equal(t, 100, total) // One is missing, stuck in the last batch!
+	require.Equal(t, 4, len(batches))
+	mu.Unlock()
+}
+
+func TestBatchingQueue_WithTimeout(t *testing.T) {
+	q := util.NewBatchingQueue[int](25, 100*time.Millisecond)
+	batches, total := make([][]int, 0), 0
+	var mu sync.Mutex
+	go func() {
+		for batch := range q.Dequeue() {
+			mu.Lock()
+			batches = append(batches, batch)
+			total += len(batch)
+			mu.Unlock()
+		}
+	}()
+	for i := 0; i < 101; i++ {
+		go func(i int) {
+			time.Sleep(time.Duration(rand.Intn(700)) * time.Millisecond)
+			q.Enqueue(i)
+		}(i)
+	}
+	time.Sleep(time.Second)
+	mu.Lock()
+	require.Equal(t, 101, total)
+	require.True(t, len(batches) > 4) // 101/25
+	require.True(t, len(batches) < 21)
+	mu.Unlock()
+}