perf(store): index attachments by memo (#6536)

Avoid repeated attachment table scans in memo lists and attachment filters.

Validate existing index definitions and allow MySQL startup retries after DDL commits. Keep unbounded PostgreSQL MIME types out of B-tree keys, document coordinated upgrades, and cover migrations and filters with regression tests.

Co-authored-by: b84330808 <b84330808@gmail.com>
This commit is contained in:
Wei-Lun Li
2026-11-15 03:44:16 +00:00
committed by GitHub
parent 3011bec691
commit 8599fe8829
14 changed files with 370 additions and 7 deletions
+15
View File
@@ -250,3 +250,18 @@ The `FieldKindExists` kind makes further relation-backed flags one schema entry
`title`, which the payload already stores, is still not a filter field. A scalar JSON string field would allow `title.contains(...)`
and a `has_title` presence flag; it is a content fact and belongs in the payload where it already is.
## Amendments
### 2026-11-15: Attachment lookup indexes
The deferred lookup index is added in schema 26.11.1 after measurements showed
that SQLite attachment filters repeatedly scan blob overflow pages. SQLite and
MySQL use `(memo_id, type)` to cover both existence predicates. PostgreSQL uses
`memo_id` alone because its unbounded MIME text can exceed the B-tree entry limit.
Index creation runs in the existing startup migration. Upgrades require stopping
all instances and other writers, migrating once, then restarting, as described in
[the migration procedure](../../store/migration/README.md#coordinating-an-upgrade).
The migrations reject conflicting index definitions; MySQL also accepts a
completed index when retrying after an implicit DDL commit.
@@ -0,0 +1,53 @@
package test
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/require"
apipb "github.com/usememos/memos/proto/gen/api"
"github.com/usememos/memos/store"
)
func TestListMemosAttachmentFilters(t *testing.T) {
ctx := context.Background()
ts := NewTestService(t)
defer ts.Cleanup()
owner, err := ts.CreateRegularUser(ctx, "attachment-filters")
require.NoError(t, err)
ownerCtx := ts.CreateUserContext(ctx, owner.ID)
for i, mime := range []string{"", "application/pdf", "image/png"} {
memo, err := ts.Store.CreateMemo(ctx, &store.Memo{
UID: fmt.Sprintf("filter-%d", i), CreatorID: owner.ID,
Content: "no inline image", Visibility: store.Private,
})
require.NoError(t, err)
if mime != "" {
_, err = ts.Store.CreateAttachment(ctx, &store.Attachment{
UID: fmt.Sprintf("filter-att-%d", i), CreatorID: owner.ID,
MemoID: &memo.ID, Type: mime,
})
require.NoError(t, err)
}
}
for _, tc := range []struct {
filter string
names []string
}{
{"has_attachments", []string{"memos/filter-1", "memos/filter-2"}},
{"has_image", []string{"memos/filter-2"}},
{"has_attachments && !has_image", []string{"memos/filter-1"}},
} {
t.Run(tc.filter, func(t *testing.T) {
resp, err := ts.Service.ListMemos(ownerCtx, &apipb.ListMemosRequest{Filter: tc.filter, PageSize: 50})
require.NoError(t, err)
names := make([]string, 0, len(resp.Memos))
for _, memo := range resp.Memos {
names = append(names, memo.Name)
}
require.ElementsMatch(t, tc.names, names)
})
}
}
@@ -6,6 +6,8 @@ import (
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/internal/version"
apipb "github.com/usememos/memos/proto/gen/api"
@@ -248,6 +250,30 @@ func BenchmarkListMemos(b *testing.B) {
}
}
})
// The attachment filters check attachment.memo_id once per candidate memo.
for _, filter := range []string{"has_attachments", "has_image"} {
b.Run("authenticated_"+filter, func(b *testing.B) {
req := &apipb.ListMemosRequest{PageSize: benchmarkPageSize, Filter: filter}
resp, err := bench.Service.ListMemos(bench.authenticatedCtx, req)
require.NoError(b, err)
require.Len(b, resp.Memos, benchmarkPageSize)
for i, memo := range resp.Memos {
require.Equal(b, fmt.Sprintf("memos/memo-%06d", (benchmarkTopLevelMemoCount-1)/3*3-i*3), memo.Name)
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
resp, err := bench.Service.ListMemos(bench.authenticatedCtx, req)
if err != nil {
b.Fatalf("ListMemos failed: %v", err)
}
if len(resp.Memos) == 0 {
b.Fatalf("expected memos in %s benchmark response", filter)
}
}
})
}
}
func BenchmarkListMemoCommentsPreview(b *testing.B) {
@@ -273,3 +299,48 @@ func BenchmarkListMemoCommentsPreview(b *testing.B) {
}
}
}
// BenchmarkListMemosLargeAttachments exercises SQLite overflow pages with real
// database blobs while only the attachment lookup index is needed by filters.
func BenchmarkListMemosLargeAttachments(b *testing.B) {
ctx := context.Background()
ts := newTestingServiceForTB(b)
owner, err := ts.CreateHostUser(ctx, "blob-bench")
require.NoError(b, err)
ownerCtx := ts.CreateUserContext(ctx, owner.ID)
blob := make([]byte, 256<<10)
const memoCount = 2000
for i := range memoCount {
memo, err := ts.Store.CreateMemo(ctx, &store.Memo{
UID: fmt.Sprintf("blob-%06d", i), CreatorID: owner.ID,
Content: "attachment benchmark", Visibility: store.Private,
})
require.NoError(b, err)
if i%5 == 0 {
_, err = ts.Store.CreateAttachment(ctx, &store.Attachment{
UID: fmt.Sprintf("blob-att-%06d", i), CreatorID: owner.ID, MemoID: &memo.ID,
Type: "image/png", Blob: blob, Size: int64(len(blob)),
})
require.NoError(b, err)
}
}
for _, filter := range []string{"has_attachments", "has_image"} {
b.Run(filter, func(b *testing.B) {
req := &apipb.ListMemosRequest{PageSize: benchmarkPageSize, Filter: filter}
resp, err := ts.Service.ListMemos(ownerCtx, req)
require.NoError(b, err)
require.Len(b, resp.Memos, benchmarkPageSize)
for i, memo := range resp.Memos {
require.Equal(b, fmt.Sprintf("memos/blob-%06d", (memoCount-1)/5*5-i*5), memo.Name)
}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
_, err := ts.Service.ListMemos(ownerCtx, req)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+17
View File
@@ -27,3 +27,20 @@ without schema changes keeps the existing schema version. Calendar schemas
sort after the legacy 0.x baseline. Migration directories keep months zero-padded
(`01` through `12`), while stored schema markers omit the padding. This lets older binaries' SemVer comparisons
recognize a calendar schema as newer and reject a downgrade.
## Coordinating an upgrade
Back up the database, stop every Memos instance and other database writers, then
start one instance of the new version to run migrations. Start the remaining
instances only after that instance has completed migration successfully. Do not
run different Memos versions against the same database during an upgrade.
Migrations run at startup before the HTTP server starts. Index builds can take
time on large attachment tables and block concurrent writes; SQLite also allows
only one writer. The attachment lookup index in schema 26.11.1 uses this
maintenance window rather than an online build. PostgreSQL builds it inside the
migration transaction, so failure rolls back both the index and schema marker.
MySQL commits DDL implicitly; this migration accepts an already completed index
on retry, but rejects a conflicting index definition. If a migration reports a
conflict, inspect the named index before retrying; do not advance the schema
marker manually.
@@ -0,0 +1,16 @@
-- MySQL commits DDL before the schema marker. Accept a completed index on retry,
-- but let CREATE fail for a conflicting definition instead of advancing the marker.
SET @memos_attachment_index_sql = IF(
(SELECT COUNT(*) = 2
AND SUM(SEQ_IN_INDEX = 1 AND COLUMN_NAME = 'memo_id') = 1
AND SUM(SEQ_IN_INDEX = 2 AND COLUMN_NAME = 'type') = 1
AND SUM(SUB_PART IS NOT NULL OR NON_UNIQUE <> 1 OR INDEX_TYPE <> 'BTREE' OR IS_VISIBLE <> 'YES') = 0
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attachment'
AND INDEX_NAME = 'idx_attachment_memo_id_type'),
'SELECT 1',
'CREATE INDEX `idx_attachment_memo_id_type` ON `attachment`(`memo_id`, `type`)'
);
PREPARE memos_attachment_index FROM @memos_attachment_index_sql;
EXECUTE memos_attachment_index;
DEALLOCATE PREPARE memos_attachment_index;
+1
View File
@@ -97,6 +97,7 @@ CREATE TABLE `attachment` (
);
CREATE INDEX `idx_attachment_creator_id` ON `attachment`(`creator_id`);
CREATE INDEX `idx_attachment_memo_id_type` ON `attachment`(`memo_id`, `type`);
-- idp
CREATE TABLE `idp` (
@@ -0,0 +1,20 @@
-- Index the bounded lookup key; attachment.type is unbounded text on PostgreSQL.
DO $$
BEGIN
IF to_regclass('idx_attachment_memo_id') IS NULL THEN
CREATE INDEX idx_attachment_memo_id ON attachment(memo_id);
ELSIF NOT EXISTS (
SELECT 1 FROM pg_index i
JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_am am ON am.oid = c.relam
WHERE i.indexrelid = to_regclass('idx_attachment_memo_id')
AND i.indrelid = 'attachment'::regclass
AND i.indisvalid AND i.indisready AND NOT i.indisunique
AND i.indpred IS NULL AND i.indexprs IS NULL
AND i.indnatts = 1 AND i.indnkeyatts = 1
AND pg_get_indexdef(i.indexrelid, 1, true) = 'memo_id'
AND am.amname = 'btree'
) THEN
RAISE EXCEPTION 'conflicting definition for idx_attachment_memo_id';
END IF;
END $$;
+1
View File
@@ -97,6 +97,7 @@ CREATE TABLE attachment (
);
CREATE INDEX idx_attachment_creator_id ON attachment(creator_id);
CREATE INDEX idx_attachment_memo_id ON attachment(memo_id);
-- idp
CREATE TABLE idp (
@@ -0,0 +1,17 @@
CREATE INDEX IF NOT EXISTS idx_attachment_memo_id_type ON attachment(memo_id, type);
-- IF NOT EXISTS alone would silently accept an operator-created conflicting index.
CREATE TEMP TABLE attachment_index_check (
valid INTEGER CONSTRAINT idx_attachment_memo_id_type_definition CHECK (valid = 1)
);
INSERT INTO attachment_index_check
SELECT EXISTS (
SELECT 1 FROM pragma_index_list('attachment')
WHERE name = 'idx_attachment_memo_id_type' AND "unique" = 0 AND partial = 0
) AND (
SELECT COUNT(*) = 2
AND SUM(seqno = 0 AND name = 'memo_id') = 1
AND SUM(seqno = 1 AND name = 'type') = 1
FROM pragma_index_info('idx_attachment_memo_id_type')
);
DROP TABLE attachment_index_check;
+1
View File
@@ -98,6 +98,7 @@ CREATE TABLE attachment (
);
CREATE INDEX idx_attachment_creator_id ON attachment(creator_id);
CREATE INDEX idx_attachment_memo_id_type ON attachment(memo_id, type);
-- idp
CREATE TABLE idp (
@@ -0,0 +1,112 @@
package test
import (
"context"
"crypto/sha256"
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
func attachmentLookupIndex(driver string) string {
if driver == "postgres" {
return "idx_attachment_memo_id"
}
return "idx_attachment_memo_id_type"
}
// TestMigrationAttachmentIndex covers a new index, a retry after DDL committed,
// and a conflicting operator-created index without losing attachment data.
func TestMigrationAttachmentIndex(t *testing.T) {
for _, state := range []string{"absent", "present", "conflict"} {
t.Run(state, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
ts := NewTestingStore(ctx, t)
t.Cleanup(func() { require.NoError(t, ts.Close()) })
driver := getDriverFromEnv()
index := attachmentLookupIndex(driver)
user, err := createTestingHostUser(ctx, ts)
require.NoError(t, err)
memo, err := ts.CreateMemo(ctx, &store.Memo{UID: "index-upgrade", CreatorID: user.ID, Content: "preserved", Visibility: store.Private})
require.NoError(t, err)
attachment, err := ts.CreateAttachment(ctx, &store.Attachment{
UID: "index-attachment", CreatorID: user.ID, MemoID: &memo.ID,
Type: "image/png", Blob: []byte("preserved blob"), Size: 14, StorageType: storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED,
})
require.NoError(t, err)
if state != "present" {
query := "DROP INDEX " + index
if driver == "mysql" {
query += " ON attachment"
}
_, err = ts.GetDriver().GetDB().ExecContext(ctx, query)
require.NoError(t, err)
}
if state == "conflict" {
_, err = ts.GetDriver().GetDB().ExecContext(ctx, "CREATE INDEX "+index+" ON attachment(creator_id)")
require.NoError(t, err)
}
setSchemaVersion(ctx, t, ts, "26.10.1")
err = ts.Migrate(ctx)
if state == "conflict" {
require.ErrorContains(t, err, index)
} else {
require.NoError(t, err)
requireLookupIndexes(ctx, t, ts, driver)
require.NoError(t, ts.Migrate(ctx), "a completed upgrade can restart")
}
setting, err := ts.GetUncachedInstanceSetting(ctx, &store.FindInstanceSetting{Name: "BASIC"})
require.NoError(t, err)
expected := "26.11.1"
if state == "conflict" {
expected = "26.10.1"
}
require.Equal(t, expected, setting.GetBasicSetting().SchemaVersion)
stored, err := ts.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID, GetBlob: true})
require.NoError(t, err)
require.Equal(t, []byte("preserved blob"), stored.Blob)
require.Equal(t, attachment.MemoID, stored.MemoID)
})
}
}
// TestMigrationAttachmentLongType preserves PostgreSQL's unbounded MIME field
// on both fresh installations and upgrades, without a B-tree entry-size failure.
func TestMigrationAttachmentLongType(t *testing.T) {
if getDriverFromEnv() != "postgres" {
t.Skip("PostgreSQL stores the attachment type as unbounded text")
}
var mime strings.Builder
mime.WriteString("application/")
for i := range 256 {
fmt.Fprintf(&mime, "%x", sha256.Sum256([]byte(fmt.Sprint(i))))
}
for _, upgrade := range []bool{false, true} {
t.Run(fmt.Sprintf("upgrade=%t", upgrade), func(t *testing.T) {
ctx := context.Background()
ts := NewTestingStore(ctx, t)
t.Cleanup(func() { require.NoError(t, ts.Close()) })
user, err := createTestingHostUser(ctx, ts)
require.NoError(t, err)
if upgrade {
_, err := ts.GetDriver().GetDB().ExecContext(ctx, "DROP INDEX "+attachmentLookupIndex("postgres"))
require.NoError(t, err)
setSchemaVersion(ctx, t, ts, "26.10.1")
}
attachment, err := ts.CreateAttachment(ctx, &store.Attachment{UID: "long-type", CreatorID: user.ID, Type: mime.String()})
require.NoError(t, err)
require.NoError(t, ts.Migrate(ctx))
requireLookupIndexes(ctx, t, ts, "postgres")
stored, err := ts.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID})
require.NoError(t, err)
require.Equal(t, mime.String(), stored.Type)
})
}
}
+1 -1
View File
@@ -45,7 +45,7 @@ func TestMigrationRejectsUnsupportedSchema(t *testing.T) {
{name: "zero", schema: "0.0.0", message: "First upgrade to v0.25.3"},
{name: "missing-basic", missingBasic: true, message: "First upgrade to v0.25.3"},
{name: "invalid-month", schema: "26.13.1", message: "invalid database schema version"},
{name: "calver-newer", schema: "26.10.2", message: "cannot downgrade schema version"},
{name: "calver-newer", schema: "26.11.2", message: "cannot downgrade schema version"},
{name: "invalid", schema: "unknown", message: "invalid database schema version"},
{name: "incomplete", schema: "0.31", message: "invalid database schema version"},
{name: "prerelease", schema: "0.31.8-rc.1", message: "invalid database schema version"},
+44 -5
View File
@@ -47,7 +47,7 @@ func TestFreshInstall(t *testing.T) {
// Verify migration completed successfully
currentSchemaVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
require.Equal(t, "26.10.1", currentSchemaVersion, "fresh install should record the latest migration")
require.Equal(t, "26.11.1", currentSchemaVersion, "fresh install should record the latest migration")
// Verify we can read instance settings (basic sanity check)
instanceSetting, err := ts.GetInstanceBasicSetting(ctx)
@@ -82,12 +82,14 @@ func TestFreshInstall(t *testing.T) {
requireQueryError(ctx, t, ts.GetDriver().GetDB(), "SELECT parent_memo_id, root_memo_id FROM memo LIMIT 0", "fresh memo schema must not contain canonical-root columns")
requireQueryError(ctx, t, ts.GetDriver().GetDB(), "SELECT row_status FROM space LIMIT 0", "fresh Space schema has no archived state")
requireCreatorIndexes(ctx, t, ts, driver)
requireLookupIndexes(ctx, t, ts, driver)
}
// requireCreatorIndexes checks the indexes that usage limit checks rely on to
// count a user's memos and sum a user's attachment sizes.
func requireCreatorIndexes(ctx context.Context, t *testing.T, ts *store.Store, driver string) {
// requireLookupIndexes checks the indexes that usage limit checks rely on to
// count a user's memos and sum a user's attachment sizes, and the index that
// memo lists and the has_attachments and has_image filters use to find a
// memo's attachments without scanning the attachment table.
func requireLookupIndexes(ctx context.Context, t *testing.T, ts *store.Store, driver string) {
t.Helper()
for _, indexName := range []string{"idx_memo_creator_id", "idx_attachment_creator_id"} {
var query string
@@ -105,6 +107,43 @@ func requireCreatorIndexes(ctx context.Context, t *testing.T, ts *store.Store, d
require.NoError(t, ts.GetDriver().GetDB().QueryRowContext(ctx, query, indexName).Scan(&count))
require.Equal(t, 1, count, "index %s must exist", indexName)
}
index := attachmentLookupIndex(driver)
var query string
columns := []string{"memo_id", "type"}
switch driver {
case "sqlite":
query = `SELECT info.name FROM pragma_index_list('attachment') AS list
JOIN pragma_index_info(list.name) AS info
WHERE list.name = ? AND list."unique" = 0 AND list.partial = 0 ORDER BY info.seqno`
case "mysql":
query = `SELECT COLUMN_NAME FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attachment' AND INDEX_NAME = ?
AND NON_UNIQUE = 1 AND SUB_PART IS NULL AND INDEX_TYPE = 'BTREE' AND IS_VISIBLE = 'YES'
ORDER BY SEQ_IN_INDEX`
case "postgres":
columns = []string{"memo_id"}
query = `SELECT pg_get_indexdef(i.indexrelid, k, true)
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_am am ON am.oid = c.relam
CROSS JOIN LATERAL generate_series(1, i.indnatts) AS k
WHERE i.indexrelid = to_regclass($1) AND i.indrelid = 'attachment'::regclass
AND i.indisvalid AND i.indisready AND NOT i.indisunique
AND i.indpred IS NULL AND i.indexprs IS NULL AND am.amname = 'btree' ORDER BY k`
default:
t.Fatalf("unsupported driver: %s", driver)
}
rows, err := ts.GetDriver().GetDB().QueryContext(ctx, query, index)
require.NoError(t, err)
defer rows.Close()
var actual []string
for rows.Next() {
var column string
require.NoError(t, rows.Scan(&column))
actual = append(actual, column)
}
require.NoError(t, rows.Err())
require.Equal(t, columns, actual, "index %s must have the expected ordered attachment columns", index)
}
// TestMigrationReRun verifies that re-running the migration on an already
+1 -1
View File
@@ -126,7 +126,7 @@ func requireUpgradePreservesData(ctx context.Context, t *testing.T, previous *st
preservedAccess, err := candidate.GetInstanceAccessSetting(ctx)
require.NoError(t, err)
require.True(t, proto.Equal(access.GetAccessSetting(), preservedAccess), "explicit access mode must not be reset")
requireCreatorIndexes(ctx, t, candidate, candidateProfile.Driver)
requireLookupIndexes(ctx, t, candidate, candidateProfile.Driver)
}
driver, err := storedb.NewDBDriver(candidateProfile)