2021-11-02 19:08:21 +01:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
2022-02-27 20:21:34 +01:00
|
|
|
"database/sql"
|
2022-04-19 15:14:32 +02:00
|
|
|
"encoding/json"
|
2021-12-07 17:45:15 +01:00
|
|
|
"errors"
|
2022-02-27 20:21:34 +01:00
|
|
|
"fmt"
|
2022-10-05 22:42:07 +02:00
|
|
|
"net/netip"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2021-11-02 19:08:21 +01:00
|
|
|
_ "github.com/mattn/go-sqlite3" // SQLite driver
|
2022-06-02 17:59:22 +02:00
|
|
|
"heckel.io/ntfy/log"
|
2022-02-27 20:21:34 +01:00
|
|
|
"heckel.io/ntfy/util"
|
2021-11-02 19:08:21 +01:00
|
|
|
)
|
|
|
|
|
2021-12-07 17:45:15 +01:00
|
|
|
var (
|
|
|
|
errUnexpectedMessageType = errors.New("unexpected message type")
|
|
|
|
)
|
|
|
|
|
2022-02-27 20:21:34 +01:00
|
|
|
// Messages cache
|
|
|
|
const (
|
|
|
|
createMessagesTableQuery = `
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS messages (
|
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
mid TEXT NOT NULL,
|
|
|
|
time INT NOT NULL,
|
|
|
|
topic TEXT NOT NULL,
|
|
|
|
message TEXT NOT NULL,
|
|
|
|
title TEXT NOT NULL,
|
|
|
|
priority INT NOT NULL,
|
|
|
|
tags TEXT NOT NULL,
|
|
|
|
click TEXT NOT NULL,
|
2022-09-11 22:31:39 +02:00
|
|
|
icon TEXT NOT NULL,
|
2022-04-19 15:14:32 +02:00
|
|
|
actions TEXT NOT NULL,
|
2022-02-27 20:21:34 +01:00
|
|
|
attachment_name TEXT NOT NULL,
|
|
|
|
attachment_type TEXT NOT NULL,
|
|
|
|
attachment_size INT NOT NULL,
|
|
|
|
attachment_expires INT NOT NULL,
|
|
|
|
attachment_url TEXT NOT NULL,
|
2022-06-01 03:39:19 +02:00
|
|
|
sender TEXT NOT NULL,
|
2022-12-28 04:14:14 +01:00
|
|
|
user TEXT NOT NULL,
|
2022-02-27 20:21:34 +01:00
|
|
|
encoding TEXT NOT NULL,
|
2022-09-11 22:31:39 +02:00
|
|
|
published INT NOT NULL
|
2022-02-27 20:21:34 +01:00
|
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_mid ON messages (mid);
|
2022-11-16 16:28:20 +01:00
|
|
|
CREATE INDEX IF NOT EXISTS idx_time ON messages (time);
|
2022-02-27 20:21:34 +01:00
|
|
|
CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic);
|
|
|
|
COMMIT;
|
|
|
|
`
|
|
|
|
insertMessageQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
INSERT INTO messages (mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding, published)
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
2022-02-27 20:21:34 +01:00
|
|
|
`
|
|
|
|
pruneMessagesQuery = `DELETE FROM messages WHERE time < ? AND published = 1`
|
2022-06-20 18:11:52 +02:00
|
|
|
selectRowIDFromMessageID = `SELECT id FROM messages WHERE mid = ?` // Do not include topic, see #336 and TestServer_PollSinceID_MultipleTopics
|
2022-02-27 20:21:34 +01:00
|
|
|
selectMessagesSinceTimeQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
SELECT mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding
|
2022-02-27 20:21:34 +01:00
|
|
|
FROM messages
|
|
|
|
WHERE topic = ? AND time >= ? AND published = 1
|
|
|
|
ORDER BY time, id
|
|
|
|
`
|
|
|
|
selectMessagesSinceTimeIncludeScheduledQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
SELECT mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding
|
2022-02-27 20:21:34 +01:00
|
|
|
FROM messages
|
|
|
|
WHERE topic = ? AND time >= ?
|
|
|
|
ORDER BY time, id
|
|
|
|
`
|
|
|
|
selectMessagesSinceIDQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
SELECT mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding
|
2022-02-27 20:21:34 +01:00
|
|
|
FROM messages
|
|
|
|
WHERE topic = ? AND id > ? AND published = 1
|
|
|
|
ORDER BY time, id
|
|
|
|
`
|
|
|
|
selectMessagesSinceIDIncludeScheduledQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
SELECT mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding
|
2022-02-27 20:21:34 +01:00
|
|
|
FROM messages
|
|
|
|
WHERE topic = ? AND (id > ? OR published = 0)
|
|
|
|
ORDER BY time, id
|
|
|
|
`
|
|
|
|
selectMessagesDueQuery = `
|
2022-12-20 03:42:36 +01:00
|
|
|
SELECT mid, time, topic, message, title, priority, tags, click, icon, actions, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, sender, user, encoding
|
2022-02-27 20:21:34 +01:00
|
|
|
FROM messages
|
|
|
|
WHERE time <= ? AND published = 0
|
|
|
|
ORDER BY time, id
|
|
|
|
`
|
2022-12-20 03:42:36 +01:00
|
|
|
updateMessagePublishedQuery = `UPDATE messages SET published = 1 WHERE mid = ?`
|
|
|
|
selectMessagesCountQuery = `SELECT COUNT(*) FROM messages`
|
|
|
|
selectMessageCountPerTopicQuery = `SELECT topic, COUNT(*) FROM messages GROUP BY topic`
|
|
|
|
selectTopicsQuery = `SELECT topic FROM messages GROUP BY topic`
|
|
|
|
selectAttachmentsSizeBySenderQuery = `SELECT IFNULL(SUM(attachment_size), 0) FROM messages WHERE sender = ? AND attachment_expires >= ?`
|
|
|
|
selectAttachmentsSizeByUserQuery = `SELECT IFNULL(SUM(attachment_size), 0) FROM messages WHERE user = ? AND attachment_expires >= ?`
|
2022-02-27 20:21:34 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// Schema management queries
|
|
|
|
const (
|
2022-12-28 04:14:14 +01:00
|
|
|
currentSchemaVersion = 10
|
2022-02-27 20:21:34 +01:00
|
|
|
createSchemaVersionTableQuery = `
|
|
|
|
CREATE TABLE IF NOT EXISTS schemaVersion (
|
|
|
|
id INT PRIMARY KEY,
|
|
|
|
version INT NOT NULL
|
|
|
|
);
|
|
|
|
`
|
|
|
|
insertSchemaVersion = `INSERT INTO schemaVersion VALUES (1, ?)`
|
|
|
|
updateSchemaVersion = `UPDATE schemaVersion SET version = ? WHERE id = 1`
|
|
|
|
selectSchemaVersionQuery = `SELECT version FROM schemaVersion WHERE id = 1`
|
|
|
|
|
|
|
|
// 0 -> 1
|
|
|
|
migrate0To1AlterMessagesTableQuery = `
|
|
|
|
BEGIN;
|
|
|
|
ALTER TABLE messages ADD COLUMN title TEXT NOT NULL DEFAULT('');
|
|
|
|
ALTER TABLE messages ADD COLUMN priority INT NOT NULL DEFAULT(0);
|
|
|
|
ALTER TABLE messages ADD COLUMN tags TEXT NOT NULL DEFAULT('');
|
|
|
|
COMMIT;
|
|
|
|
`
|
|
|
|
|
|
|
|
// 1 -> 2
|
|
|
|
migrate1To2AlterMessagesTableQuery = `
|
|
|
|
ALTER TABLE messages ADD COLUMN published INT NOT NULL DEFAULT(1);
|
|
|
|
`
|
|
|
|
|
|
|
|
// 2 -> 3
|
|
|
|
migrate2To3AlterMessagesTableQuery = `
|
|
|
|
BEGIN;
|
|
|
|
ALTER TABLE messages ADD COLUMN click TEXT NOT NULL DEFAULT('');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_name TEXT NOT NULL DEFAULT('');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_type TEXT NOT NULL DEFAULT('');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_size INT NOT NULL DEFAULT('0');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_expires INT NOT NULL DEFAULT('0');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_owner TEXT NOT NULL DEFAULT('');
|
|
|
|
ALTER TABLE messages ADD COLUMN attachment_url TEXT NOT NULL DEFAULT('');
|
|
|
|
COMMIT;
|
|
|
|
`
|
|
|
|
// 3 -> 4
|
|
|
|
migrate3To4AlterMessagesTableQuery = `
|
|
|
|
ALTER TABLE messages ADD COLUMN encoding TEXT NOT NULL DEFAULT('');
|
|
|
|
`
|
|
|
|
|
|
|
|
// 4 -> 5
|
|
|
|
migrate4To5AlterMessagesTableQuery = `
|
|
|
|
BEGIN;
|
|
|
|
CREATE TABLE IF NOT EXISTS messages_new (
|
|
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
|
|
mid TEXT NOT NULL,
|
|
|
|
time INT NOT NULL,
|
|
|
|
topic TEXT NOT NULL,
|
|
|
|
message TEXT NOT NULL,
|
|
|
|
title TEXT NOT NULL,
|
|
|
|
priority INT NOT NULL,
|
|
|
|
tags TEXT NOT NULL,
|
|
|
|
click TEXT NOT NULL,
|
|
|
|
attachment_name TEXT NOT NULL,
|
|
|
|
attachment_type TEXT NOT NULL,
|
|
|
|
attachment_size INT NOT NULL,
|
|
|
|
attachment_expires INT NOT NULL,
|
|
|
|
attachment_url TEXT NOT NULL,
|
|
|
|
attachment_owner TEXT NOT NULL,
|
|
|
|
encoding TEXT NOT NULL,
|
|
|
|
published INT NOT NULL
|
|
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_mid ON messages_new (mid);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_topic ON messages_new (topic);
|
|
|
|
INSERT
|
|
|
|
INTO messages_new (
|
|
|
|
mid, time, topic, message, title, priority, tags, click, attachment_name, attachment_type,
|
|
|
|
attachment_size, attachment_expires, attachment_url, attachment_owner, encoding, published)
|
|
|
|
SELECT
|
|
|
|
id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type,
|
|
|
|
attachment_size, attachment_expires, attachment_url, attachment_owner, encoding, published
|
|
|
|
FROM messages;
|
|
|
|
DROP TABLE messages;
|
|
|
|
ALTER TABLE messages_new RENAME TO messages;
|
|
|
|
COMMIT;
|
|
|
|
`
|
2022-04-21 15:58:28 +02:00
|
|
|
|
|
|
|
// 5 -> 6
|
|
|
|
migrate5To6AlterMessagesTableQuery = `
|
|
|
|
ALTER TABLE messages ADD COLUMN actions TEXT NOT NULL DEFAULT('');
|
|
|
|
`
|
2022-06-01 03:39:19 +02:00
|
|
|
|
|
|
|
// 6 -> 7
|
|
|
|
migrate6To7AlterMessagesTableQuery = `
|
|
|
|
ALTER TABLE messages RENAME COLUMN attachment_owner TO sender;
|
|
|
|
`
|
2022-07-16 21:31:03 +02:00
|
|
|
|
|
|
|
// 7 -> 8
|
|
|
|
migrate7To8AlterMessagesTableQuery = `
|
2022-07-17 23:40:24 +02:00
|
|
|
ALTER TABLE messages ADD COLUMN icon TEXT NOT NULL DEFAULT('');
|
2022-07-16 21:31:03 +02:00
|
|
|
`
|
2022-11-16 16:28:20 +01:00
|
|
|
|
|
|
|
// 8 -> 9
|
|
|
|
migrate8To9AlterMessagesTableQuery = `
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_time ON messages (time);
|
|
|
|
`
|
2022-12-28 04:14:14 +01:00
|
|
|
|
|
|
|
// 9 -> 10
|
|
|
|
migrate9To10AlterMessagesTableQuery = `
|
|
|
|
ALTER TABLE messages ADD COLUMN user TEXT NOT NULL DEFAULT('');
|
|
|
|
`
|
2022-02-27 20:21:34 +01:00
|
|
|
)
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
type messageCache struct {
|
2022-11-15 20:24:56 +01:00
|
|
|
db *sql.DB
|
|
|
|
queue *util.BatchingQueue[*message]
|
|
|
|
nop bool
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
// newSqliteCache creates a SQLite file-backed cache
|
2022-11-16 16:28:20 +01:00
|
|
|
func newSqliteCache(filename, startupQueries string, batchSize int, batchTimeout time.Duration, nop bool) (*messageCache, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
db, err := sql.Open("sqlite3", filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-06-23 17:02:45 +02:00
|
|
|
if err := setupCacheDB(db, startupQueries); err != nil {
|
2022-02-27 20:21:34 +01:00
|
|
|
return nil, err
|
|
|
|
}
|
2022-11-16 16:28:20 +01:00
|
|
|
var queue *util.BatchingQueue[*message]
|
|
|
|
if batchSize > 0 || batchTimeout > 0 {
|
|
|
|
queue = util.NewBatchingQueue[*message](batchSize, batchTimeout)
|
|
|
|
}
|
2022-11-15 20:24:56 +01:00
|
|
|
cache := &messageCache{
|
|
|
|
db: db,
|
|
|
|
queue: queue,
|
|
|
|
nop: nop,
|
|
|
|
}
|
2022-11-16 16:28:20 +01:00
|
|
|
go cache.processMessageBatches()
|
2022-11-15 20:24:56 +01:00
|
|
|
return cache, nil
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// newMemCache creates an in-memory cache
|
2022-02-27 20:47:28 +01:00
|
|
|
func newMemCache() (*messageCache, error) {
|
2022-11-16 16:28:20 +01:00
|
|
|
return newSqliteCache(createMemoryFilename(), "", 0, 0, false)
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// newNopCache creates an in-memory cache that discards all messages;
|
|
|
|
// it is always empty and can be used if caching is entirely disabled
|
2022-02-27 20:47:28 +01:00
|
|
|
func newNopCache() (*messageCache, error) {
|
2022-11-16 16:28:20 +01:00
|
|
|
return newSqliteCache(createMemoryFilename(), "", 0, 0, true)
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// createMemoryFilename creates a unique memory filename to use for the SQLite backend.
|
|
|
|
// From mattn/go-sqlite3: "Each connection to ":memory:" opens a brand new in-memory
|
|
|
|
// sql database, so if the stdlib's sql engine happens to open another connection and
|
|
|
|
// you've only specified ":memory:", that connection will see a brand new database.
|
|
|
|
// A workaround is to use "file::memory:?cache=shared" (or "file:foobar?mode=memory&cache=shared").
|
|
|
|
// Every connection to this string will point to the same in-memory database."
|
|
|
|
func createMemoryFilename() string {
|
|
|
|
return fmt.Sprintf("file:%s?mode=memory&cache=shared", util.RandomString(10))
|
|
|
|
}
|
|
|
|
|
2022-11-16 16:28:20 +01:00
|
|
|
// AddMessage stores a message to the message cache synchronously, or queues it to be stored at a later date asyncronously.
|
|
|
|
// The message is queued only if "batchSize" or "batchTimeout" are passed to the constructor.
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) AddMessage(m *message) error {
|
2022-11-16 16:28:20 +01:00
|
|
|
if c.queue != nil {
|
|
|
|
c.queue.Enqueue(m)
|
|
|
|
return nil
|
|
|
|
}
|
2022-06-23 02:17:47 +02:00
|
|
|
return c.addMessages([]*message{m})
|
|
|
|
}
|
|
|
|
|
2022-11-16 16:28:20 +01:00
|
|
|
// addMessages synchronously stores a match of messages. If the database is locked, the transaction waits until
|
|
|
|
// SQLite's busy_timeout is exceeded before erroring out.
|
2022-06-23 02:17:47 +02:00
|
|
|
func (c *messageCache) addMessages(ms []*message) error {
|
2022-02-27 20:21:34 +01:00
|
|
|
if c.nop {
|
|
|
|
return nil
|
|
|
|
}
|
2022-12-12 03:39:02 +01:00
|
|
|
if len(ms) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
2022-11-16 16:28:20 +01:00
|
|
|
start := time.Now()
|
2022-06-23 02:17:47 +02:00
|
|
|
tx, err := c.db.Begin()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
2022-06-23 02:17:47 +02:00
|
|
|
defer tx.Rollback()
|
2022-12-12 15:55:17 +01:00
|
|
|
stmt, err := tx.Prepare(insertMessageQuery)
|
2022-12-12 03:39:02 +01:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-12-12 15:55:17 +01:00
|
|
|
defer stmt.Close()
|
2022-06-23 02:17:47 +02:00
|
|
|
for _, m := range ms {
|
|
|
|
if m.Event != messageEvent {
|
|
|
|
return errUnexpectedMessageType
|
|
|
|
}
|
|
|
|
published := m.Time <= time.Now().Unix()
|
|
|
|
tags := strings.Join(m.Tags, ",")
|
|
|
|
var attachmentName, attachmentType, attachmentURL string
|
|
|
|
var attachmentSize, attachmentExpires int64
|
|
|
|
if m.Attachment != nil {
|
|
|
|
attachmentName = m.Attachment.Name
|
|
|
|
attachmentType = m.Attachment.Type
|
|
|
|
attachmentSize = m.Attachment.Size
|
|
|
|
attachmentExpires = m.Attachment.Expires
|
|
|
|
attachmentURL = m.Attachment.URL
|
|
|
|
}
|
|
|
|
var actionsStr string
|
|
|
|
if len(m.Actions) > 0 {
|
|
|
|
actionsBytes, err := json.Marshal(m.Actions)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
actionsStr = string(actionsBytes)
|
|
|
|
}
|
2022-11-18 02:47:27 +01:00
|
|
|
var sender string
|
|
|
|
if m.Sender.IsValid() {
|
|
|
|
sender = m.Sender.String()
|
|
|
|
}
|
2022-12-12 15:55:17 +01:00
|
|
|
_, err := stmt.Exec(
|
2022-06-23 02:17:47 +02:00
|
|
|
m.ID,
|
|
|
|
m.Time,
|
|
|
|
m.Topic,
|
|
|
|
m.Message,
|
|
|
|
m.Title,
|
|
|
|
m.Priority,
|
|
|
|
tags,
|
|
|
|
m.Click,
|
2022-09-11 22:31:39 +02:00
|
|
|
m.Icon,
|
2022-06-23 02:17:47 +02:00
|
|
|
actionsStr,
|
|
|
|
attachmentName,
|
|
|
|
attachmentType,
|
|
|
|
attachmentSize,
|
|
|
|
attachmentExpires,
|
|
|
|
attachmentURL,
|
2022-11-18 02:47:27 +01:00
|
|
|
sender,
|
2022-12-20 03:42:36 +01:00
|
|
|
m.User,
|
2022-06-23 02:17:47 +02:00
|
|
|
m.Encoding,
|
|
|
|
published,
|
|
|
|
)
|
2022-04-19 15:14:32 +02:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2022-11-16 16:28:20 +01:00
|
|
|
if err := tx.Commit(); err != nil {
|
2022-11-16 16:33:12 +01:00
|
|
|
log.Error("Cache: Writing %d message(s) failed (took %v)", len(ms), time.Since(start))
|
2022-11-16 16:28:20 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
log.Debug("Cache: Wrote %d message(s) in %v", len(ms), time.Since(start))
|
|
|
|
return nil
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) Messages(topic string, since sinceMarker, scheduled bool) ([]*message, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
if since.IsNone() {
|
|
|
|
return make([]*message, 0), nil
|
|
|
|
} else if since.IsID() {
|
|
|
|
return c.messagesSinceID(topic, since, scheduled)
|
|
|
|
}
|
|
|
|
return c.messagesSinceTime(topic, since, scheduled)
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) messagesSinceTime(topic string, since sinceMarker, scheduled bool) ([]*message, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
var rows *sql.Rows
|
|
|
|
var err error
|
|
|
|
if scheduled {
|
|
|
|
rows, err = c.db.Query(selectMessagesSinceTimeIncludeScheduledQuery, topic, since.Time().Unix())
|
|
|
|
} else {
|
|
|
|
rows, err = c.db.Query(selectMessagesSinceTimeQuery, topic, since.Time().Unix())
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return readMessages(rows)
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) messagesSinceID(topic string, since sinceMarker, scheduled bool) ([]*message, error) {
|
2022-06-20 18:11:52 +02:00
|
|
|
idrows, err := c.db.Query(selectRowIDFromMessageID, since.ID())
|
2022-02-27 20:21:34 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer idrows.Close()
|
|
|
|
if !idrows.Next() {
|
|
|
|
return c.messagesSinceTime(topic, sinceAllMessages, scheduled)
|
|
|
|
}
|
|
|
|
var rowID int64
|
|
|
|
if err := idrows.Scan(&rowID); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
idrows.Close()
|
|
|
|
var rows *sql.Rows
|
|
|
|
if scheduled {
|
|
|
|
rows, err = c.db.Query(selectMessagesSinceIDIncludeScheduledQuery, topic, rowID)
|
|
|
|
} else {
|
|
|
|
rows, err = c.db.Query(selectMessagesSinceIDQuery, topic, rowID)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return readMessages(rows)
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) MessagesDue() ([]*message, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
rows, err := c.db.Query(selectMessagesDueQuery, time.Now().Unix())
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return readMessages(rows)
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) MarkPublished(m *message) error {
|
2022-02-27 20:21:34 +01:00
|
|
|
_, err := c.db.Exec(updateMessagePublishedQuery, m.ID)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-06-22 01:07:27 +02:00
|
|
|
func (c *messageCache) MessageCounts() (map[string]int, error) {
|
|
|
|
rows, err := c.db.Query(selectMessageCountPerTopicQuery)
|
2022-02-27 20:21:34 +01:00
|
|
|
if err != nil {
|
2022-06-22 01:07:27 +02:00
|
|
|
return nil, err
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
defer rows.Close()
|
2022-06-22 01:07:27 +02:00
|
|
|
var topic string
|
2022-02-27 20:21:34 +01:00
|
|
|
var count int
|
2022-06-22 01:07:27 +02:00
|
|
|
counts := make(map[string]int)
|
|
|
|
for rows.Next() {
|
|
|
|
if err := rows.Scan(&topic, &count); err != nil {
|
|
|
|
return nil, err
|
|
|
|
} else if err := rows.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
counts[topic] = count
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
2022-06-22 01:07:27 +02:00
|
|
|
return counts, nil
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) Topics() (map[string]*topic, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
rows, err := c.db.Query(selectTopicsQuery)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer rows.Close()
|
|
|
|
topics := make(map[string]*topic)
|
|
|
|
for rows.Next() {
|
|
|
|
var id string
|
|
|
|
if err := rows.Scan(&id); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
topics[id] = newTopic(id)
|
|
|
|
}
|
|
|
|
if err := rows.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return topics, nil
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:47:28 +01:00
|
|
|
func (c *messageCache) Prune(olderThan time.Time) error {
|
2022-11-16 16:28:20 +01:00
|
|
|
start := time.Now()
|
|
|
|
if _, err := c.db.Exec(pruneMessagesQuery, olderThan.Unix()); err != nil {
|
|
|
|
log.Warn("Cache: Pruning failed (after %v): %s", time.Since(start), err.Error())
|
|
|
|
}
|
|
|
|
log.Debug("Cache: Pruning successful (took %v)", time.Since(start))
|
|
|
|
return nil
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
|
2022-12-20 03:42:36 +01:00
|
|
|
func (c *messageCache) AttachmentBytesUsedBySender(sender string) (int64, error) {
|
|
|
|
rows, err := c.db.Query(selectAttachmentsSizeBySenderQuery, sender, time.Now().Unix())
|
2022-02-27 20:21:34 +01:00
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
2022-12-20 03:42:36 +01:00
|
|
|
return c.readAttachmentBytesUsed(rows)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *messageCache) AttachmentBytesUsedByUser(user string) (int64, error) {
|
|
|
|
rows, err := c.db.Query(selectAttachmentsSizeByUserQuery, user, time.Now().Unix())
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
return c.readAttachmentBytesUsed(rows)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *messageCache) readAttachmentBytesUsed(rows *sql.Rows) (int64, error) {
|
2022-02-27 20:21:34 +01:00
|
|
|
defer rows.Close()
|
|
|
|
var size int64
|
|
|
|
if !rows.Next() {
|
|
|
|
return 0, errors.New("no rows found")
|
|
|
|
}
|
|
|
|
if err := rows.Scan(&size); err != nil {
|
|
|
|
return 0, err
|
|
|
|
} else if err := rows.Err(); err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
return size, nil
|
|
|
|
}
|
|
|
|
|
2022-11-16 16:28:20 +01:00
|
|
|
func (c *messageCache) processMessageBatches() {
|
|
|
|
if c.queue == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
for messages := range c.queue.Dequeue() {
|
|
|
|
if err := c.addMessages(messages); err != nil {
|
|
|
|
log.Error("Cache: %s", err.Error())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-27 20:21:34 +01:00
|
|
|
func readMessages(rows *sql.Rows) ([]*message, error) {
|
|
|
|
defer rows.Close()
|
|
|
|
messages := make([]*message, 0)
|
|
|
|
for rows.Next() {
|
2022-07-17 23:40:24 +02:00
|
|
|
var timestamp, attachmentSize, attachmentExpires int64
|
2022-02-27 20:21:34 +01:00
|
|
|
var priority int
|
2022-12-20 03:42:36 +01:00
|
|
|
var id, topic, msg, title, tagsStr, click, icon, actionsStr, attachmentName, attachmentType, attachmentURL, sender, user, encoding string
|
2022-02-27 20:21:34 +01:00
|
|
|
err := rows.Scan(
|
|
|
|
&id,
|
|
|
|
×tamp,
|
|
|
|
&topic,
|
|
|
|
&msg,
|
|
|
|
&title,
|
|
|
|
&priority,
|
|
|
|
&tagsStr,
|
|
|
|
&click,
|
2022-09-11 22:31:39 +02:00
|
|
|
&icon,
|
2022-04-19 15:14:32 +02:00
|
|
|
&actionsStr,
|
2022-02-27 20:21:34 +01:00
|
|
|
&attachmentName,
|
|
|
|
&attachmentType,
|
|
|
|
&attachmentSize,
|
|
|
|
&attachmentExpires,
|
|
|
|
&attachmentURL,
|
2022-06-01 03:39:19 +02:00
|
|
|
&sender,
|
2022-12-20 03:42:36 +01:00
|
|
|
&user,
|
2022-02-27 20:21:34 +01:00
|
|
|
&encoding,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var tags []string
|
|
|
|
if tagsStr != "" {
|
|
|
|
tags = strings.Split(tagsStr, ",")
|
|
|
|
}
|
2022-04-19 15:14:32 +02:00
|
|
|
var actions []*action
|
|
|
|
if actionsStr != "" {
|
|
|
|
if err := json.Unmarshal([]byte(actionsStr), &actions); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2022-10-07 23:16:20 +02:00
|
|
|
senderIP, err := netip.ParseAddr(sender)
|
|
|
|
if err != nil {
|
2022-11-18 02:47:27 +01:00
|
|
|
senderIP = netip.Addr{} // if no IP stored in database, return invalid address
|
2022-10-07 23:16:20 +02:00
|
|
|
}
|
2022-02-27 20:21:34 +01:00
|
|
|
var att *attachment
|
|
|
|
if attachmentName != "" && attachmentURL != "" {
|
|
|
|
att = &attachment{
|
|
|
|
Name: attachmentName,
|
|
|
|
Type: attachmentType,
|
|
|
|
Size: attachmentSize,
|
|
|
|
Expires: attachmentExpires,
|
|
|
|
URL: attachmentURL,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
messages = append(messages, &message{
|
|
|
|
ID: id,
|
|
|
|
Time: timestamp,
|
|
|
|
Event: messageEvent,
|
|
|
|
Topic: topic,
|
|
|
|
Message: msg,
|
|
|
|
Title: title,
|
|
|
|
Priority: priority,
|
|
|
|
Tags: tags,
|
|
|
|
Click: click,
|
2022-07-17 23:40:24 +02:00
|
|
|
Icon: icon,
|
2022-04-19 15:14:32 +02:00
|
|
|
Actions: actions,
|
2022-02-27 20:21:34 +01:00
|
|
|
Attachment: att,
|
2022-10-07 23:16:20 +02:00
|
|
|
Sender: senderIP, // Must parse assuming database must be correct
|
2022-12-20 03:42:36 +01:00
|
|
|
User: user,
|
2022-02-27 20:21:34 +01:00
|
|
|
Encoding: encoding,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
if err := rows.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return messages, nil
|
|
|
|
}
|
|
|
|
|
2022-06-23 17:02:45 +02:00
|
|
|
func setupCacheDB(db *sql.DB, startupQueries string) error {
|
|
|
|
// Run startup queries
|
|
|
|
if startupQueries != "" {
|
|
|
|
if _, err := db.Exec(startupQueries); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-06-23 02:17:47 +02:00
|
|
|
}
|
|
|
|
|
2022-02-27 20:21:34 +01:00
|
|
|
// If 'messages' table does not exist, this must be a new database
|
|
|
|
rowsMC, err := db.Query(selectMessagesCountQuery)
|
|
|
|
if err != nil {
|
|
|
|
return setupNewCacheDB(db)
|
|
|
|
}
|
|
|
|
rowsMC.Close()
|
|
|
|
|
|
|
|
// If 'messages' table exists, check 'schemaVersion' table
|
|
|
|
schemaVersion := 0
|
|
|
|
rowsSV, err := db.Query(selectSchemaVersionQuery)
|
|
|
|
if err == nil {
|
|
|
|
defer rowsSV.Close()
|
|
|
|
if !rowsSV.Next() {
|
|
|
|
return errors.New("cannot determine schema version: cache file may be corrupt")
|
|
|
|
}
|
|
|
|
if err := rowsSV.Scan(&schemaVersion); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
rowsSV.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Do migrations
|
|
|
|
if schemaVersion == currentSchemaVersion {
|
|
|
|
return nil
|
|
|
|
} else if schemaVersion == 0 {
|
|
|
|
return migrateFrom0(db)
|
|
|
|
} else if schemaVersion == 1 {
|
|
|
|
return migrateFrom1(db)
|
|
|
|
} else if schemaVersion == 2 {
|
|
|
|
return migrateFrom2(db)
|
|
|
|
} else if schemaVersion == 3 {
|
|
|
|
return migrateFrom3(db)
|
|
|
|
} else if schemaVersion == 4 {
|
|
|
|
return migrateFrom4(db)
|
2022-04-21 15:58:28 +02:00
|
|
|
} else if schemaVersion == 5 {
|
|
|
|
return migrateFrom5(db)
|
2022-06-01 03:39:19 +02:00
|
|
|
} else if schemaVersion == 6 {
|
|
|
|
return migrateFrom6(db)
|
2022-07-16 21:31:03 +02:00
|
|
|
} else if schemaVersion == 7 {
|
|
|
|
return migrateFrom7(db)
|
2022-11-16 16:28:20 +01:00
|
|
|
} else if schemaVersion == 8 {
|
|
|
|
return migrateFrom8(db)
|
2022-12-28 04:14:14 +01:00
|
|
|
} else if schemaVersion == 9 {
|
|
|
|
return migrateFrom9(db)
|
2022-02-27 20:21:34 +01:00
|
|
|
}
|
|
|
|
return fmt.Errorf("unexpected schema version found: %d", schemaVersion)
|
|
|
|
}
|
|
|
|
|
|
|
|
func setupNewCacheDB(db *sql.DB) error {
|
|
|
|
if _, err := db.Exec(createMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(createSchemaVersionTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(insertSchemaVersion, currentSchemaVersion); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom0(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 0 to 1")
|
2022-02-27 20:21:34 +01:00
|
|
|
if _, err := db.Exec(migrate0To1AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(createSchemaVersionTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(insertSchemaVersion, 1); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return migrateFrom1(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom1(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 1 to 2")
|
2022-02-27 20:21:34 +01:00
|
|
|
if _, err := db.Exec(migrate1To2AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 2); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return migrateFrom2(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom2(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 2 to 3")
|
2022-02-27 20:21:34 +01:00
|
|
|
if _, err := db.Exec(migrate2To3AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 3); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return migrateFrom3(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom3(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 3 to 4")
|
2022-02-27 20:21:34 +01:00
|
|
|
if _, err := db.Exec(migrate3To4AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 4); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return migrateFrom4(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom4(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 4 to 5")
|
2022-02-27 20:21:34 +01:00
|
|
|
if _, err := db.Exec(migrate4To5AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 5); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-04-21 15:58:28 +02:00
|
|
|
return migrateFrom5(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom5(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 5 to 6")
|
2022-04-21 15:58:28 +02:00
|
|
|
if _, err := db.Exec(migrate5To6AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 6); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-06-01 03:39:19 +02:00
|
|
|
return migrateFrom6(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom6(db *sql.DB) error {
|
2022-06-02 17:59:22 +02:00
|
|
|
log.Info("Migrating cache database schema: from 6 to 7")
|
2022-06-01 03:39:19 +02:00
|
|
|
if _, err := db.Exec(migrate6To7AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 7); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-07-16 21:31:03 +02:00
|
|
|
return migrateFrom7(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom7(db *sql.DB) error {
|
|
|
|
log.Info("Migrating cache database schema: from 7 to 8")
|
|
|
|
if _, err := db.Exec(migrate7To8AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 8); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-11-16 16:28:20 +01:00
|
|
|
return migrateFrom8(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom8(db *sql.DB) error {
|
|
|
|
log.Info("Migrating cache database schema: from 8 to 9")
|
|
|
|
if _, err := db.Exec(migrate8To9AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 9); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-12-28 04:14:14 +01:00
|
|
|
return migrateFrom9(db)
|
|
|
|
}
|
|
|
|
|
|
|
|
func migrateFrom9(db *sql.DB) error {
|
|
|
|
log.Info("Migrating cache database schema: from 9 to 10")
|
|
|
|
if _, err := db.Exec(migrate9To10AlterMessagesTableQuery); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if _, err := db.Exec(updateSchemaVersion, 10); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-02-27 20:21:34 +01:00
|
|
|
return nil // Update this when a new version is added
|
2021-11-02 19:08:21 +01:00
|
|
|
}
|