From c45a28e6af94e571de3aa6833d072e051f69418c Mon Sep 17 00:00:00 2001 From: Philipp Heckel Date: Fri, 7 Jan 2022 14:49:28 +0100 Subject: [PATCH] Attachments limits; working visitor limit --- cmd/serve.go | 47 +++++++++++---- docs/publish.md | 21 +++++-- server/cache.go | 1 + server/cache_mem.go | 14 +++++ server/cache_sqlite.go | 53 ++++++++++------ server/config.go | 134 ++++++++++++++++++++--------------------- server/file_cache.go | 88 +++++++++++++++++++++++++++ server/message.go | 12 ++-- server/server.go | 103 +++++++++---------------------- 9 files changed, 287 insertions(+), 186 deletions(-) create mode 100644 server/file_cache.go diff --git a/cmd/serve.go b/cmd/serve.go index 6540e7c8..3380f433 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -21,7 +21,8 @@ var flagsServe = []cli.Flag{ altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-duration", Aliases: []string{"b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: server.DefaultCacheDuration, Usage: "buffer messages for this time to allow `since` requests"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-cache-dir", EnvVars: []string{"NTFY_ATTACHMENT_CACHE_DIR"}, Usage: "cache directory for attached files"}), - altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_SIZE_LIMIT"}, DefaultText: "15M", Usage: "attachment size limit (e.g. 10k, 2M)"}), + altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-total-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT"}, DefaultText: "1G", Usage: "limit of the on-disk attachment cache"}), + altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-file-size-limit", Aliases: []string{"Y"}, EnvVars: []string{"NTFY_ATTACHMENT_FILE_SIZE_LIMIT"}, DefaultText: "15M", Usage: "per-file attachment size limit (e.g. 300k, 2M, 100M)"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "keepalive-interval", Aliases: []string{"k"}, EnvVars: []string{"NTFY_KEEPALIVE_INTERVAL"}, Value: server.DefaultKeepaliveInterval, Usage: "interval of keepalive messages"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "manager-interval", Aliases: []string{"m"}, EnvVars: []string{"NTFY_MANAGER_INTERVAL"}, Value: server.DefaultManagerInterval, Usage: "interval of for message pruning and stats printing"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-sender-addr", EnvVars: []string{"NTFY_SMTP_SENDER_ADDR"}, Usage: "SMTP server address (host:port) for outgoing emails"}), @@ -33,6 +34,7 @@ var flagsServe = []cli.Flag{ altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-server-addr-prefix", EnvVars: []string{"NTFY_SMTP_SERVER_ADDR_PREFIX"}, Usage: "SMTP email address prefix for topics to prevent spam (e.g. 'ntfy-')"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "global-topic-limit", Aliases: []string{"T"}, EnvVars: []string{"NTFY_GLOBAL_TOPIC_LIMIT"}, Value: server.DefaultTotalTopicLimit, Usage: "total number of topics allowed"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-subscription-limit", EnvVars: []string{"NTFY_VISITOR_SUBSCRIPTION_LIMIT"}, Value: server.DefaultVisitorSubscriptionLimit, Usage: "number of subscriptions per visitor"}), + altsrc.NewStringFlag(&cli.StringFlag{Name: "visitor-attachment-total-size-limit", EnvVars: []string{"NTFY_VISITOR_ATTACHMENT_TOTAL_SIZE_LIMIT"}, Value: "50M", Usage: "total storage limit used for attachments per visitor"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-request-limit-burst", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_BURST"}, Value: server.DefaultVisitorRequestLimitBurst, Usage: "initial limit of requests per visitor"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "visitor-request-limit-replenish", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_REPLENISH"}, Value: server.DefaultVisitorRequestLimitReplenish, Usage: "interval at which burst limit is replenished (one per x)"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-email-limit-burst", EnvVars: []string{"NTFY_VISITOR_EMAIL_LIMIT_BURST"}, Value: server.DefaultVisitorEmailLimitBurst, Usage: "initial limit of e-mails per visitor"}), @@ -72,7 +74,8 @@ func execServe(c *cli.Context) error { cacheFile := c.String("cache-file") cacheDuration := c.Duration("cache-duration") attachmentCacheDir := c.String("attachment-cache-dir") - attachmentSizeLimitStr := c.String("attachment-size-limit") + attachmentTotalSizeLimitStr := c.String("attachment-total-size-limit") + attachmentFileSizeLimitStr := c.String("attachment-file-size-limit") keepaliveInterval := c.Duration("keepalive-interval") managerInterval := c.Duration("manager-interval") smtpSenderAddr := c.String("smtp-sender-addr") @@ -82,8 +85,9 @@ func execServe(c *cli.Context) error { smtpServerListen := c.String("smtp-server-listen") smtpServerDomain := c.String("smtp-server-domain") smtpServerAddrPrefix := c.String("smtp-server-addr-prefix") - globalTopicLimit := c.Int("global-topic-limit") + totalTopicLimit := c.Int("global-topic-limit") visitorSubscriptionLimit := c.Int("visitor-subscription-limit") + visitorAttachmentTotalSizeLimitStr := c.String("visitor-attachment-total-size-limit") visitorRequestLimitBurst := c.Int("visitor-request-limit-burst") visitorRequestLimitReplenish := c.Duration("visitor-request-limit-replenish") visitorEmailLimitBurst := c.Int("visitor-email-limit-burst") @@ -111,14 +115,18 @@ func execServe(c *cli.Context) error { return errors.New("if smtp-server-listen is set, smtp-server-domain must also be set") } - // Convert - attachmentSizeLimit := server.DefaultAttachmentSizeLimit - if attachmentSizeLimitStr != "" { - var err error - attachmentSizeLimit, err = util.ParseSize(attachmentSizeLimitStr) - if err != nil { - return err - } + // Convert sizes to bytes + attachmentTotalSizeLimit, err := parseSize(attachmentTotalSizeLimitStr, server.DefaultAttachmentTotalSizeLimit) + if err != nil { + return err + } + attachmentFileSizeLimit, err := parseSize(attachmentFileSizeLimitStr, server.DefaultAttachmentFileSizeLimit) + if err != nil { + return err + } + visitorAttachmentTotalSizeLimit, err := parseSize(visitorAttachmentTotalSizeLimitStr, server.DefaultVisitorAttachmentTotalSizeLimit) + if err != nil { + return err } // Run server @@ -132,7 +140,8 @@ func execServe(c *cli.Context) error { conf.CacheFile = cacheFile conf.CacheDuration = cacheDuration conf.AttachmentCacheDir = attachmentCacheDir - conf.AttachmentSizeLimit = attachmentSizeLimit + conf.AttachmentTotalSizeLimit = attachmentTotalSizeLimit + conf.AttachmentFileSizeLimit = attachmentFileSizeLimit conf.KeepaliveInterval = keepaliveInterval conf.ManagerInterval = managerInterval conf.SMTPSenderAddr = smtpSenderAddr @@ -142,8 +151,9 @@ func execServe(c *cli.Context) error { conf.SMTPServerListen = smtpServerListen conf.SMTPServerDomain = smtpServerDomain conf.SMTPServerAddrPrefix = smtpServerAddrPrefix - conf.TotalTopicLimit = globalTopicLimit + conf.TotalTopicLimit = totalTopicLimit conf.VisitorSubscriptionLimit = visitorSubscriptionLimit + conf.VisitorAttachmentTotalSizeLimit = visitorAttachmentTotalSizeLimit conf.VisitorRequestLimitBurst = visitorRequestLimitBurst conf.VisitorRequestLimitReplenish = visitorRequestLimitReplenish conf.VisitorEmailLimitBurst = visitorEmailLimitBurst @@ -159,3 +169,14 @@ func execServe(c *cli.Context) error { log.Printf("Exiting.") return nil } + +func parseSize(s string, defaultValue int64) (v int64, err error) { + if s == "" { + return defaultValue, nil + } + v, err = util.ParseSize(s) + if err != nil { + return 0, err + } + return v, nil +} diff --git a/docs/publish.md b/docs/publish.md index f5e4f5b4..fc5b804a 100644 --- a/docs/publish.md +++ b/docs/publish.md @@ -661,22 +661,31 @@ Here's an example that will open Reddit when the notification is clicked: ## Send files + URLs ``` +- Uploaded attachment +- External attachment +- Preview without attachment + + +# Send attachment curl -T image.jpg ntfy.sh/howdy +# Send attachment with custom message and filename curl \ -T flower.jpg \ -H "Message: Here's a flower for you" \ -H "Filename: flower.jpg" \ ntfy.sh/howdy +# Send attachment from another URL, with custom preview and message curl \ - -T files.zip \ + -H "Attachment: https://example.com/files.zip" \ + -H "Preview: https://example.com/filespreview.jpg" \ + "ntfy.sh/howdy?m=Important+documents+attached" + +# Send normal message with external image +curl \ + -H "Image: https://example.com/someimage.jpg" \ "ntfy.sh/howdy?m=Important+documents+attached" - -curl \ - -d "A link for you" \ - -H "Link: https://unifiedpush.org" \ - "ntfy.sh/howdy" ``` ## E-mail notifications diff --git a/server/cache.go b/server/cache.go index 64d517d0..7532ff7f 100644 --- a/server/cache.go +++ b/server/cache.go @@ -20,4 +20,5 @@ type cache interface { Topics() (map[string]*topic, error) Prune(olderThan time.Time) error MarkPublished(m *message) error + AttachmentsSize(owner string) (int64, error) } diff --git a/server/cache_mem.go b/server/cache_mem.go index 31c7bb97..91bcb38c 100644 --- a/server/cache_mem.go +++ b/server/cache_mem.go @@ -125,6 +125,20 @@ func (c *memCache) Prune(olderThan time.Time) error { return nil } +func (c *memCache) AttachmentsSize(owner string) (int64, error) { + c.mu.Lock() + defer c.mu.Unlock() + var size int64 + for topic := range c.messages { + for _, m := range c.messages[topic] { + if m.Attachment != nil && m.Attachment.Owner == owner { + size += m.Attachment.Size + } + } + } + return size, nil +} + func (c *memCache) pruneTopic(topic string, olderThan time.Time) { messages := make([]*message, 0) for _, m := range c.messages[topic] { diff --git a/server/cache_sqlite.go b/server/cache_sqlite.go index 99c4df66..4a52f281 100644 --- a/server/cache_sqlite.go +++ b/server/cache_sqlite.go @@ -27,32 +27,32 @@ const ( attachment_type TEXT NOT NULL, attachment_size INT NOT NULL, attachment_expires INT NOT NULL, - attachment_preview_url TEXT NOT NULL, attachment_url TEXT NOT NULL, + attachment_owner TEXT NOT NULL, published INT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic); COMMIT; ` insertMessageQuery = ` - INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url, published) + INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner, published) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` pruneMessagesQuery = `DELETE FROM messages WHERE time < ? AND published = 1` selectMessagesSinceTimeQuery = ` - SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url + SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner FROM messages WHERE topic = ? AND time >= ? AND published = 1 ORDER BY time ASC ` selectMessagesSinceTimeIncludeScheduledQuery = ` - SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url + SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner FROM messages WHERE topic = ? AND time >= ? ORDER BY time ASC ` selectMessagesDueQuery = ` - SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url + SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner FROM messages WHERE time <= ? AND published = 0 ` @@ -60,6 +60,7 @@ const ( selectMessagesCountQuery = `SELECT COUNT(*) FROM messages` selectMessageCountForTopicQuery = `SELECT COUNT(*) FROM messages WHERE topic = ?` selectTopicsQuery = `SELECT topic FROM messages GROUP BY topic` + selectAttachmentsSizeQuery = `SELECT IFNULL(SUM(attachment_size), 0) FROM messages WHERE attachment_owner = ?` ) // Schema management queries @@ -97,7 +98,7 @@ const ( ALTER TABLE messages ADD COLUMN attachment_type TEXT NOT NULL DEFAULT(''); ALTER TABLE messages ADD COLUMN attachment_size INT NOT NULL DEFAULT('0'); ALTER TABLE messages ADD COLUMN attachment_expires INT NOT NULL DEFAULT('0'); - ALTER TABLE messages ADD COLUMN attachment_preview_url TEXT NOT NULL DEFAULT(''); + ALTER TABLE messages ADD COLUMN attachment_owner TEXT NOT NULL DEFAULT(''); ALTER TABLE messages ADD COLUMN attachment_url TEXT NOT NULL DEFAULT(''); COMMIT; ` @@ -128,15 +129,15 @@ func (c *sqliteCache) AddMessage(m *message) error { } published := m.Time <= time.Now().Unix() tags := strings.Join(m.Tags, ",") - var attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string + var attachmentName, attachmentType, attachmentURL, attachmentOwner string var attachmentSize, attachmentExpires int64 if m.Attachment != nil { attachmentName = m.Attachment.Name attachmentType = m.Attachment.Type attachmentSize = m.Attachment.Size attachmentExpires = m.Attachment.Expires - attachmentPreviewURL = m.Attachment.PreviewURL attachmentURL = m.Attachment.URL + attachmentOwner = m.Attachment.Owner } _, err := c.db.Exec( insertMessageQuery, @@ -152,8 +153,8 @@ func (c *sqliteCache) AddMessage(m *message) error { attachmentType, attachmentSize, attachmentExpires, - attachmentPreviewURL, attachmentURL, + attachmentOwner, published, ) return err @@ -232,14 +233,32 @@ func (c *sqliteCache) Prune(olderThan time.Time) error { return err } +func (c *sqliteCache) AttachmentsSize(owner string) (int64, error) { + rows, err := c.db.Query(selectAttachmentsSizeQuery, owner) + if err != nil { + return 0, err + } + defer rows.Close() + var size int64 + if !rows.Next() { + return 0, errors.New("no rows found") + } + if err := rows.Scan(&size); err != nil { + return 0, err + } else if err := rows.Err(); err != nil { + return 0, err + } + return size, nil +} + func readMessages(rows *sql.Rows) ([]*message, error) { defer rows.Close() messages := make([]*message, 0) for rows.Next() { var timestamp, attachmentSize, attachmentExpires int64 var priority int - var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string - if err := rows.Scan(&id, ×tamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentPreviewURL, &attachmentURL); err != nil { + var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentURL, attachmentOwner string + if err := rows.Scan(&id, ×tamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentOwner, &attachmentURL); err != nil { return nil, err } var tags []string @@ -249,12 +268,12 @@ func readMessages(rows *sql.Rows) ([]*message, error) { var att *attachment if attachmentName != "" && attachmentURL != "" { att = &attachment{ - Name: attachmentName, - Type: attachmentType, - Size: attachmentSize, - Expires: attachmentExpires, - PreviewURL: attachmentPreviewURL, - URL: attachmentURL, + Name: attachmentName, + Type: attachmentType, + Size: attachmentSize, + Expires: attachmentExpires, + URL: attachmentURL, + Owner: attachmentOwner, } } messages = append(messages, &message{ diff --git a/server/config.go b/server/config.go index 6a57a4b1..76c38e3b 100644 --- a/server/config.go +++ b/server/config.go @@ -13,9 +13,9 @@ const ( DefaultAtSenderInterval = 10 * time.Second DefaultMinDelay = 10 * time.Second DefaultMaxDelay = 3 * 24 * time.Hour - DefaultMessageLimit = 4096 // Bytes - DefaultAttachmentSizeLimit = int64(15 * 1024 * 1024) - DefaultAttachmentSizePreviewMax = 20 * 1024 * 1024 // Bytes + DefaultMessageLimit = 4096 // Bytes + DefaultAttachmentTotalSizeLimit = int64(1024 * 1024 * 1024) // 1 GB + DefaultAttachmentFileSizeLimit = int64(15 * 1024 * 1024) // 15 MB DefaultAttachmentExpiryDuration = 3 * time.Hour DefaultFirebaseKeepaliveInterval = 3 * time.Hour // Not too frequently to save battery ) @@ -33,80 +33,78 @@ const ( DefaultVisitorRequestLimitReplenish = 10 * time.Second DefaultVisitorEmailLimitBurst = 16 DefaultVisitorEmailLimitReplenish = time.Hour - DefaultVisitorAttachmentBytesLimitBurst = 50 * 1024 * 1024 + DefaultVisitorAttachmentTotalSizeLimit = 50 * 1024 * 1024 DefaultVisitorAttachmentBytesLimitReplenish = time.Hour ) // Config is the main config struct for the application. Use New to instantiate a default config struct. type Config struct { - BaseURL string - ListenHTTP string - ListenHTTPS string - KeyFile string - CertFile string - FirebaseKeyFile string - CacheFile string - CacheDuration time.Duration - AttachmentCacheDir string - AttachmentSizeLimit int64 - AttachmentSizePreviewMax int64 - AttachmentExpiryDuration time.Duration - KeepaliveInterval time.Duration - ManagerInterval time.Duration - AtSenderInterval time.Duration - FirebaseKeepaliveInterval time.Duration - SMTPSenderAddr string - SMTPSenderUser string - SMTPSenderPass string - SMTPSenderFrom string - SMTPServerListen string - SMTPServerDomain string - SMTPServerAddrPrefix string - MessageLimit int - MinDelay time.Duration - MaxDelay time.Duration - TotalTopicLimit int - TotalAttachmentSizeLimit int64 - VisitorSubscriptionLimit int - VisitorRequestLimitBurst int - VisitorRequestLimitReplenish time.Duration - VisitorEmailLimitBurst int - VisitorEmailLimitReplenish time.Duration - VisitorAttachmentBytesLimitBurst int64 - VisitorAttachmentBytesLimitReplenish time.Duration - BehindProxy bool + BaseURL string + ListenHTTP string + ListenHTTPS string + KeyFile string + CertFile string + FirebaseKeyFile string + CacheFile string + CacheDuration time.Duration + AttachmentCacheDir string + AttachmentTotalSizeLimit int64 + AttachmentFileSizeLimit int64 + AttachmentExpiryDuration time.Duration + KeepaliveInterval time.Duration + ManagerInterval time.Duration + AtSenderInterval time.Duration + FirebaseKeepaliveInterval time.Duration + SMTPSenderAddr string + SMTPSenderUser string + SMTPSenderPass string + SMTPSenderFrom string + SMTPServerListen string + SMTPServerDomain string + SMTPServerAddrPrefix string + MessageLimit int + MinDelay time.Duration + MaxDelay time.Duration + TotalTopicLimit int + TotalAttachmentSizeLimit int64 + VisitorSubscriptionLimit int + VisitorAttachmentTotalSizeLimit int64 + VisitorRequestLimitBurst int + VisitorRequestLimitReplenish time.Duration + VisitorEmailLimitBurst int + VisitorEmailLimitReplenish time.Duration + BehindProxy bool } // NewConfig instantiates a default new server config func NewConfig() *Config { return &Config{ - BaseURL: "", - ListenHTTP: DefaultListenHTTP, - ListenHTTPS: "", - KeyFile: "", - CertFile: "", - FirebaseKeyFile: "", - CacheFile: "", - CacheDuration: DefaultCacheDuration, - AttachmentCacheDir: "", - AttachmentSizeLimit: DefaultAttachmentSizeLimit, - AttachmentSizePreviewMax: DefaultAttachmentSizePreviewMax, - AttachmentExpiryDuration: DefaultAttachmentExpiryDuration, - KeepaliveInterval: DefaultKeepaliveInterval, - ManagerInterval: DefaultManagerInterval, - MessageLimit: DefaultMessageLimit, - MinDelay: DefaultMinDelay, - MaxDelay: DefaultMaxDelay, - AtSenderInterval: DefaultAtSenderInterval, - FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval, - TotalTopicLimit: DefaultTotalTopicLimit, - VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit, - VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst, - VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish, - VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst, - VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish, - VisitorAttachmentBytesLimitBurst: DefaultVisitorAttachmentBytesLimitBurst, - VisitorAttachmentBytesLimitReplenish: DefaultVisitorAttachmentBytesLimitReplenish, - BehindProxy: false, + BaseURL: "", + ListenHTTP: DefaultListenHTTP, + ListenHTTPS: "", + KeyFile: "", + CertFile: "", + FirebaseKeyFile: "", + CacheFile: "", + CacheDuration: DefaultCacheDuration, + AttachmentCacheDir: "", + AttachmentTotalSizeLimit: DefaultAttachmentTotalSizeLimit, + AttachmentFileSizeLimit: DefaultAttachmentFileSizeLimit, + AttachmentExpiryDuration: DefaultAttachmentExpiryDuration, + KeepaliveInterval: DefaultKeepaliveInterval, + ManagerInterval: DefaultManagerInterval, + MessageLimit: DefaultMessageLimit, + MinDelay: DefaultMinDelay, + MaxDelay: DefaultMaxDelay, + AtSenderInterval: DefaultAtSenderInterval, + FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval, + TotalTopicLimit: DefaultTotalTopicLimit, + VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit, + VisitorAttachmentTotalSizeLimit: DefaultVisitorAttachmentTotalSizeLimit, + VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst, + VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish, + VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst, + VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish, + BehindProxy: false, } } diff --git a/server/file_cache.go b/server/file_cache.go new file mode 100644 index 00000000..e23718c6 --- /dev/null +++ b/server/file_cache.go @@ -0,0 +1,88 @@ +package server + +import ( + "errors" + "heckel.io/ntfy/util" + "io" + "log" + "os" + "path/filepath" + "regexp" + "sync" +) + +var ( + fileIDRegex = regexp.MustCompile(`^[-_A-Za-z0-9]+$`) + errInvalidFileID = errors.New("invalid file ID") +) + +type fileCache struct { + dir string + totalSizeCurrent int64 + totalSizeLimit int64 + fileSizeLimit int64 + mu sync.Mutex +} + +func newFileCache(dir string, totalSizeLimit int64, fileSizeLimit int64) (*fileCache, error) { + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var size int64 + for _, e := range entries { + info, err := e.Info() + if err != nil { + return nil, err + } + size += info.Size() + } + return &fileCache{ + dir: dir, + totalSizeCurrent: size, + totalSizeLimit: totalSizeLimit, + fileSizeLimit: fileSizeLimit, + }, nil +} + +func (c *fileCache) Write(id string, in io.Reader, limiters ...*util.Limiter) (int64, error) { + if !fileIDRegex.MatchString(id) { + return 0, errInvalidFileID + } + file := filepath.Join(c.dir, id) + f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return 0, err + } + defer f.Close() + log.Printf("remaining total: %d", c.remainingTotalSize()) + limiters = append(limiters, util.NewLimiter(c.remainingTotalSize()), util.NewLimiter(c.fileSizeLimit)) + limitWriter := util.NewLimitWriter(f, limiters...) + size, err := io.Copy(limitWriter, in) + if err != nil { + os.Remove(file) + return 0, err + } + if err := f.Close(); err != nil { + os.Remove(file) + return 0, err + } + c.mu.Lock() + c.totalSizeCurrent += size + c.mu.Unlock() + return size, nil + +} + +func (c *fileCache) remainingTotalSize() int64 { + c.mu.Lock() + defer c.mu.Unlock() + remaining := c.totalSizeLimit - c.totalSizeCurrent + if remaining < 0 { + return 0 + } + return remaining +} diff --git a/server/message.go b/server/message.go index b627bb39..27695f14 100644 --- a/server/message.go +++ b/server/message.go @@ -31,12 +31,12 @@ type message struct { } type attachment struct { - Name string `json:"name"` - Type string `json:"type,omitempty"` - Size int64 `json:"size,omitempty"` - Expires int64 `json:"expires,omitempty"` - PreviewURL string `json:"preview_url,omitempty"` - URL string `json:"url"` + Name string `json:"name"` + Type string `json:"type,omitempty"` + Size int64 `json:"size,omitempty"` + Expires int64 `json:"expires,omitempty"` + URL string `json:"url"` + Owner string `json:"-"` // IP address of uploader, used for rate limiting } // messageEncoder is a function that knows how to encode a message diff --git a/server/server.go b/server/server.go index f0bbf42d..afd7d38d 100644 --- a/server/server.go +++ b/server/server.go @@ -9,7 +9,6 @@ import ( firebase "firebase.google.com/go" "firebase.google.com/go/messaging" "fmt" - "github.com/disintegration/imaging" "github.com/emersion/go-smtp" "google.golang.org/api/option" "heckel.io/ntfy/util" @@ -45,6 +44,7 @@ type Server struct { mailer mailer messages int64 cache cache + fileCache *fileCache closeChan chan bool mu sync.Mutex } @@ -101,8 +101,7 @@ var ( staticRegex = regexp.MustCompile(`^/static/.+`) docsRegex = regexp.MustCompile(`^/docs(|/.*)$`) fileRegex = regexp.MustCompile(`^/file/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`) - previewRegex = regexp.MustCompile(`^/preview/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`) - disallowedTopics = []string{"docs", "static", "file", "preview"} + disallowedTopics = []string{"docs", "static", "file"} templateFnMap = template.FuncMap{ "durationToHuman": util.DurationToHuman, @@ -124,7 +123,6 @@ var ( docsStaticCached = &util.CachingEmbedFS{ModTime: time.Now(), FS: docsStaticFs} errHTTPNotFound = &errHTTP{40401, http.StatusNotFound, "page not found", ""} - errHTTPNotFoundTooLarge = &errHTTP{40402, http.StatusNotFound, "page not found: preview not available, file too large", ""} errHTTPTooManyRequestsLimitRequests = &errHTTP{42901, http.StatusTooManyRequests, "limit reached: too many requests, please be nice", "https://ntfy.sh/docs/publish/#limitations"} errHTTPTooManyRequestsLimitEmails = &errHTTP{42902, http.StatusTooManyRequests, "limit reached: too many emails, please be nice", "https://ntfy.sh/docs/publish/#limitations"} errHTTPTooManyRequestsLimitSubscriptions = &errHTTP{42903, http.StatusTooManyRequests, "limit reached: too many active subscriptions, please be nice", "https://ntfy.sh/docs/publish/#limitations"} @@ -174,18 +172,21 @@ func New(conf *Config) (*Server, error) { if err != nil { return nil, err } + var fileCache *fileCache if conf.AttachmentCacheDir != "" { - if err := os.MkdirAll(conf.AttachmentCacheDir, 0700); err != nil { + fileCache, err = newFileCache(conf.AttachmentCacheDir, conf.AttachmentTotalSizeLimit, conf.AttachmentFileSizeLimit) + if err != nil { return nil, err } } return &Server{ - config: conf, - cache: cache, - firebase: firebaseSubscriber, - mailer: mailer, - topics: topics, - visitors: make(map[string]*visitor), + config: conf, + cache: cache, + fileCache: fileCache, + firebase: firebaseSubscriber, + mailer: mailer, + topics: topics, + visitors: make(map[string]*visitor), }, nil } @@ -234,7 +235,6 @@ func createFirebaseSubscriber(conf *Config) (subscriber, error) { data["attachment_type"] = m.Attachment.Type data["attachment_size"] = fmt.Sprintf("%d", m.Attachment.Size) data["attachment_expires"] = fmt.Sprintf("%d", m.Attachment.Expires) - data["attachment_preview_url"] = m.Attachment.PreviewURL data["attachment_url"] = m.Attachment.URL } } @@ -355,8 +355,6 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request) error { return s.handleDocs(w, r) } else if r.Method == http.MethodGet && fileRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" { return s.withRateLimit(w, r, s.handleFile) - } else if r.Method == http.MethodGet && previewRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" { - return s.withRateLimit(w, r, s.handlePreview) } else if r.Method == http.MethodOptions { return s.handleOptions(w, r) } else if r.Method == http.MethodGet && topicPathRegex.MatchString(r.URL.Path) { @@ -436,39 +434,6 @@ func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, _ *visitor) return err } -func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request, _ *visitor) error { - if s.config.AttachmentCacheDir == "" { - return errHTTPInternalError - } - matches := previewRegex.FindStringSubmatch(r.URL.Path) - if len(matches) != 2 { - return errHTTPInternalErrorInvalidFilePath - } - messageID := matches[1] - file := filepath.Join(s.config.AttachmentCacheDir, messageID) - stat, err := os.Stat(file) - if err != nil { - return errHTTPNotFound - } - if stat.Size() > s.config.AttachmentSizePreviewMax { - return errHTTPNotFoundTooLarge - } - img, err := imaging.Open(file) - if err != nil { - return err - } - var width, height int - if width >= height { - width = 200 - height = int(float32(img.Bounds().Dy()) / float32(img.Bounds().Dx()) * float32(width)) - } else { - height = 200 - width = int(float32(img.Bounds().Dx()) / float32(img.Bounds().Dy()) * float32(height)) - } - preview := imaging.Resize(img, width, height, imaging.Lanczos) - return imaging.Encode(w, preview, imaging.JPEG, imaging.JPEGQuality(80)) -} - func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error { t, err := s.topicFromPath(r.URL.Path) if err != nil { @@ -482,7 +447,7 @@ func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visito filename := readParam(r, "x-filename", "filename", "file", "f") if filename == "" && !body.LimitReached && utf8.Valid(body.PeakedBytes) { m.Message = strings.TrimSpace(string(body.PeakedBytes)) - } else if s.config.AttachmentCacheDir != "" { + } else if s.fileCache != nil { if err := s.writeAttachment(r, v, m, body); err != nil { return err } @@ -601,48 +566,34 @@ func readParam(r *http.Request, names ...string) string { } func (s *Server) writeAttachment(r *http.Request, v *visitor, m *message, body *util.PeakedReadCloser) error { - if s.config.AttachmentCacheDir == "" { - return errHTTPBadRequestInvalidMessage - } contentType := http.DetectContentType(body.PeakedBytes) ext := util.ExtensionByType(contentType) fileURL := fmt.Sprintf("%s/file/%s%s", s.config.BaseURL, m.ID, ext) - previewURL := "" - if strings.HasPrefix(contentType, "image/") { - previewURL = fmt.Sprintf("%s/preview/%s%s", s.config.BaseURL, m.ID, ext) - } filename := readParam(r, "x-filename", "filename", "file", "f") if filename == "" { filename = fmt.Sprintf("attachment%s", ext) } - file := filepath.Join(s.config.AttachmentCacheDir, m.ID) - f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + // TODO do not allowed delayed delivery for attachments + visitorAttachmentsSize, err := s.cache.AttachmentsSize(v.ip) if err != nil { return err } - defer f.Close() - maxSizeLimiter := util.NewLimiter(s.config.AttachmentSizeLimit) //FIXME visitor limit - limitWriter := util.NewLimitWriter(f, maxSizeLimiter) - size, err := io.Copy(limitWriter, body) - if err != nil { - os.Remove(file) - if err == util.ErrLimitReached { - return errHTTPBadRequestMessageTooLarge - } - return err - } - if err := f.Close(); err != nil { - os.Remove(file) + remainingVisitorAttachmentSize := s.config.VisitorAttachmentTotalSizeLimit - visitorAttachmentsSize + log.Printf("remaining visitor: %d", remainingVisitorAttachmentSize) + size, err := s.fileCache.Write(m.ID, body, util.NewLimiter(remainingVisitorAttachmentSize)) + if err == util.ErrLimitReached { + return errHTTPBadRequestMessageTooLarge + } else if err != nil { return err } m.Message = fmt.Sprintf("You received a file: %s", filename) // May be overwritten later m.Attachment = &attachment{ - Name: filename, - Type: contentType, - Size: size, - Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(), - PreviewURL: previewURL, - URL: fileURL, + Name: filename, + Type: contentType, + Size: size, + Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(), + URL: fileURL, + Owner: v.ip, // Important for attachment rate limiting } return nil }