diff --git a/core/memoarchive/expansion_test.go b/core/memoarchive/expansion_test.go new file mode 100644 index 000000000..3e745edcf --- /dev/null +++ b/core/memoarchive/expansion_test.go @@ -0,0 +1,31 @@ +package memoarchive + +import ( + "archive/zip" + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +// A container whose entries declare far more bytes than it holds is refused +// before any entry is read. +func TestReadRejectsDecompressionBomb(t *testing.T) { + var buf bytes.Buffer + writer := zip.NewWriter(&buf) + entry, err := writer.Create(AttachmentsDir + "bomb/zeros.bin") + require.NoError(t, err) + zeros := make([]byte, 1<<20) + for range 80 { + _, err = entry.Write(zeros) + require.NoError(t, err) + } + require.NoError(t, writer.Close()) + require.Less(t, buf.Len(), 1<<20, "80 MiB of zeros deflates to well under a mebibyte") + + _, err = Read(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.Error(t, err) + require.Contains(t, err.Error(), "uncompressed bytes") + _ = io.EOF +} diff --git a/core/memoarchive/reader.go b/core/memoarchive/reader.go index ae9217c71..3923efdf7 100644 --- a/core/memoarchive/reader.go +++ b/core/memoarchive/reader.go @@ -26,6 +26,12 @@ const ( // maxContentBytes bounds one memo content file. Instances enforce their // own, smaller content length limit on import. maxContentBytes = 64 << 20 + // maxExpansionRatio bounds the total declared uncompressed size relative + // to the container, with maxExpansionSlack of headroom for tiny archives. + // Memo text deflates around ten to one and media hardly at all, so a + // container that claims to expand further is a decompression bomb. + maxExpansionRatio = 64 + maxExpansionSlack = 64 << 20 ) // Archive is a validated, opened Memo Archive. Memos are ordered so that a @@ -50,11 +56,17 @@ func Read(r io.ReaderAt, size int64) (*Archive, error) { return nil, errors.Errorf("archive has more than %d entries", maxEntries) } archive := &Archive{entries: make(map[string]*zip.File, len(reader.File))} + var totalCompressed, totalUncompressed uint64 for _, file := range reader.File { if strings.HasSuffix(file.Name, "/") && file.UncompressedSize64 == 0 { // Directory entries carry no data; readers must not depend on them. continue } + totalCompressed += file.CompressedSize64 + totalUncompressed += file.UncompressedSize64 + if totalUncompressed > maxExpansionRatio*totalCompressed+maxExpansionSlack { + return nil, errors.Errorf("archive declares %d uncompressed bytes for %d compressed bytes", totalUncompressed, totalCompressed) + } if err := ValidateEntryName(file.Name); err != nil { return nil, err } diff --git a/filter/engine.go b/filter/engine.go index e477b0c79..74889b0ac 100644 --- a/filter/engine.go +++ b/filter/engine.go @@ -21,8 +21,14 @@ type Engine struct { } // NewEngine builds a new Engine for the provided schema. +// maxFilterExpressionSize bounds a filter expression in code points. Every +// term compiles to its own SQL predicate, so the expression length is what +// bounds the query the database has to run. +const maxFilterExpressionSize = 8 << 10 + func NewEngine(schema Schema) (*Engine, error) { - env, err := cel.NewEnv(schema.EnvOptions...) + options := append([]cel.EnvOption{cel.ParserExpressionSizeLimit(maxFilterExpressionSize)}, schema.EnvOptions...) + env, err := cel.NewEnv(options...) if err != nil { return nil, errors.Wrap(err, "failed to create CEL environment") } diff --git a/filter/engine_test.go b/filter/engine_test.go index 5678de330..c4c773b7b 100644 --- a/filter/engine_test.go +++ b/filter/engine_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "testing" "github.com/stretchr/testify/require" @@ -688,3 +689,20 @@ func TestRenderRegexLiteralFallbackOnD1(t *testing.T) { require.ErrorContains(t, err, "not supported on Cloudflare D1", expression) } } + +func TestCompileRejectsOversizedExpression(t *testing.T) { + t.Parallel() + engine, err := NewEngine(NewSchema()) + require.NoError(t, err) + + var builder strings.Builder + builder.WriteString(`content.contains("a")`) + for builder.Len() < maxFilterExpressionSize+64 { + builder.WriteString(` || content.contains("a")`) + } + _, err = engine.Compile(context.Background(), builder.String()) + require.Error(t, err) + + _, err = engine.Compile(context.Background(), `content.contains("a") || content.contains("b")`) + require.NoError(t, err) +} diff --git a/go.mod b/go.mod index 53bdde078..6a02f71aa 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/testcontainers/testcontainers-go/modules/mysql v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/yuin/goldmark v1.8.5 - golang.org/x/crypto v0.55.0 + golang.org/x/crypto v0.56.0 golang.org/x/mod v0.40.0 golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 @@ -43,7 +43,7 @@ require ( google.golang.org/genproto v0.0.0-20260810153831-ec0a7760b754 google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 - google.golang.org/grpc v1.83.0 + google.golang.org/grpc v1.83.1 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 modernc.org/libc v1.75.3 diff --git a/go.sum b/go.sum index f1a940e63..cd134531e 100644 --- a/go.sum +++ b/go.sum @@ -295,8 +295,8 @@ go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d/go.mod h1:92Uoe3 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -337,8 +337,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/imagelimit/imagelimit.go b/internal/imagelimit/imagelimit.go new file mode 100644 index 000000000..e0e5c0cfe --- /dev/null +++ b/internal/imagelimit/imagelimit.go @@ -0,0 +1,49 @@ +// Package imagelimit bounds the decoded size of untrusted images before a +// full decode allocates memory for them. A few kilobytes of compressed data +// can declare billions of pixels; reading only the header keeps the check +// cheap and lets callers refuse the image before paying for it. +package imagelimit + +import ( + "image" + "io" + + // Register the standard formats so DecodeConfig can read their headers. + _ "image/gif" + _ "image/jpeg" + _ "image/png" + + "github.com/pkg/errors" +) + +// MaxPixels is the largest decoded pixel count accepted for any image the +// server decodes itself: EXIF stripping on upload and thumbnail generation +// on read. 50 megapixels decodes to about 200 MiB of RGBA. +const MaxPixels = 50_000_000 + +// ErrTooLarge reports an image whose declared dimensions exceed MaxPixels. +var ErrTooLarge = errors.Errorf("image dimensions exceed maximum of %d pixels", MaxPixels) + +// CheckReader reads the image header from r and reports whether the declared +// dimensions fit within MaxPixels. Formats that the image registry cannot +// size are allowed through so the full decoder decides; the caller must then +// be prepared for that decode to fail. The reader is consumed; rewind it +// before decoding. +func CheckReader(r io.Reader) error { + config, _, err := image.DecodeConfig(r) + if err != nil { + return nil //nolint:nilerr // unknown formats defer to the full decoder + } + return CheckDimensions(config.Width, config.Height) +} + +// CheckDimensions reports whether width×height fits within MaxPixels. +func CheckDimensions(width, height int) error { + if width <= 0 || height <= 0 { + return errors.New("invalid image dimensions") + } + if width > MaxPixels/height { + return ErrTooLarge + } + return nil +} diff --git a/internal/imagelimit/imagelimit_test.go b/internal/imagelimit/imagelimit_test.go new file mode 100644 index 000000000..9f38c3433 --- /dev/null +++ b/internal/imagelimit/imagelimit_test.go @@ -0,0 +1,50 @@ +package imagelimit + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/png" + "testing" + + "github.com/stretchr/testify/require" +) + +// oversizedPNG returns a syntactically valid PNG header that declares +// width×height pixels without carrying the pixel data for them. +func oversizedPNG(width, height uint32) []byte { + var buf bytes.Buffer + buf.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:4], width) + binary.BigEndian.PutUint32(ihdr[4:8], height) + ihdr[8] = 8 // bit depth + ihdr[9] = 6 // RGBA + chunk := append([]byte("IHDR"), ihdr...) + binary.Write(&buf, binary.BigEndian, uint32(len(ihdr))) + buf.Write(chunk) + binary.Write(&buf, binary.BigEndian, crc32.ChecksumIEEE(chunk)) + return buf.Bytes() +} + +func TestCheckReader(t *testing.T) { + var small bytes.Buffer + require.NoError(t, png.Encode(&small, image.NewRGBA(image.Rect(0, 0, 4, 4)))) + require.NoError(t, CheckReader(bytes.NewReader(small.Bytes()))) + + err := CheckReader(bytes.NewReader(oversizedPNG(30_000, 30_000))) + require.ErrorIs(t, err, ErrTooLarge) + + // A format the registry cannot size defers to the full decoder. + require.NoError(t, CheckReader(bytes.NewReader([]byte("not an image")))) +} + +func TestCheckDimensions(t *testing.T) { + require.NoError(t, CheckDimensions(8000, 6000)) + require.ErrorIs(t, CheckDimensions(10_000, 10_000), ErrTooLarge) + require.Error(t, CheckDimensions(0, 10)) + require.Error(t, CheckDimensions(10, -1)) + _ = color.Black +} diff --git a/internal/linkmeta/image.go b/internal/linkmeta/image.go deleted file mode 100644 index b569045e4..000000000 --- a/internal/linkmeta/image.go +++ /dev/null @@ -1,45 +0,0 @@ -package linkmeta - -import ( - "errors" - "io" - "net/http" - "net/url" - "strings" -) - -type Image struct { - Blob []byte - Mediatype string -} - -func GetImage(urlStr string) (*Image, error) { - if _, err := url.Parse(urlStr); err != nil { - return nil, err - } - - response, err := http.Get(urlStr) - if err != nil { - return nil, err - } - defer response.Body.Close() - - mediatype, err := getMediatype(response) - if err != nil { - return nil, err - } - if !strings.HasPrefix(mediatype, "image/") { - return nil, errors.New("wrong image mediatype") - } - - bodyBytes, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - image := &Image{ - Blob: bodyBytes, - Mediatype: mediatype, - } - return image, nil -} diff --git a/server/api/v1/attachment_exif_sniff_test.go b/server/api/v1/attachment_exif_sniff_test.go new file mode 100644 index 000000000..4962c8ea0 --- /dev/null +++ b/server/api/v1/attachment_exif_sniff_test.go @@ -0,0 +1,81 @@ +package v1 + +import ( + "bytes" + "context" + "encoding/binary" + "image" + "image/color" + "image/jpeg" + "testing" + + "github.com/stretchr/testify/require" + + v1pb "github.com/usememos/memos/proto/gen/api/v1" + "github.com/usememos/memos/store" +) + +const exifTestMarker = "MEMOS-GPS-MARKER-0123456789" + +// jpegWithExifMarker returns a decodable JPEG whose APP1 segment carries a +// recognizable marker, standing in for privacy-sensitive EXIF metadata. +func jpegWithExifMarker(t *testing.T) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, 16, 16)) + for y := range 16 { + for x := range 16 { + img.Set(x, y, color.RGBA{R: 200, G: 30, B: 30, A: 255}) + } + } + var encoded bytes.Buffer + require.NoError(t, jpeg.Encode(&encoded, img, &jpeg.Options{Quality: 90})) + raw := encoded.Bytes() + require.Equal(t, []byte{0xFF, 0xD8}, raw[:2]) + + payload := append([]byte("Exif\x00\x00"), []byte(exifTestMarker)...) + var out bytes.Buffer + out.Write(raw[:2]) + out.Write([]byte{0xFF, 0xE1}) + _ = binary.Write(&out, binary.BigEndian, uint16(len(payload)+2)) + out.Write(payload) + out.Write(raw[2:]) + return out.Bytes() +} + +func TestShouldStripExifContentSniffsMislabeledJPEG(t *testing.T) { + t.Parallel() + content := jpegWithExifMarker(t) + + for _, declared := range []string{"application/octet-stream", "image/png", "text/plain"} { + strip, err := shouldStripExifContent(bytes.NewReader(content), declared) + require.NoError(t, err) + require.True(t, strip, "declared %s", declared) + } + strip, err := shouldStripExifContent(bytes.NewReader([]byte("plain text file")), "application/octet-stream") + require.NoError(t, err) + require.False(t, strip) +} + +// A JPEG uploaded under a non-image declared type is still stripped of its +// metadata; the declared type alone must not decide the privacy policy. +func TestCreateAttachmentStripsExifFromMislabeledJPEG(t *testing.T) { + svc := newIntegrationService(t) + ctx := context.Background() + user := createSpaceTestUser(ctx, t, svc, "exif-mislabel", store.RoleUser) + content := jpegWithExifMarker(t) + require.Contains(t, string(content), exifTestMarker) + + created, err := svc.CreateAttachment(userCtx(ctx, user.ID), &v1pb.CreateAttachmentRequest{Attachment: &v1pb.Attachment{ + Filename: "photo.bin", Type: "application/octet-stream", Content: content, + }}) + require.NoError(t, err) + + uid := created.Name[len(AttachmentNamePrefix):] + stored, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) + require.NoError(t, err) + blob, err := svc.GetAttachmentBlob(ctx, stored) + require.NoError(t, err) + require.NotContains(t, string(blob), exifTestMarker) + _, err = jpeg.Decode(bytes.NewReader(blob)) + require.NoError(t, err) +} diff --git a/server/api/v1/attachment_service.go b/server/api/v1/attachment_service.go index bd247d050..a8e1850e8 100644 --- a/server/api/v1/attachment_service.go +++ b/server/api/v1/attachment_service.go @@ -33,7 +33,6 @@ const ( // Quality 95 maintains visual quality while ensuring metadata is removed. defaultJPEGQuality = 95 maxBatchDeleteAttachments = 100 - maxImagePixels = 50_000_000 ) // exifCapableImageTypes defines image formats that may contain EXIF metadata. @@ -228,7 +227,13 @@ func (s *APIV1Service) processAndSaveAttachment(ctx context.Context, create *sto content := io.ReadSeeker(source) // Strip EXIF metadata from images for privacy protection. Motion photo // containers are kept intact because re-encoding would drop the video. - if shouldStripExif(create.Type) && !isAndroidMotionContainer(create.Payload.GetMotionMedia()) { + // The decision uses the sniffed bytes as well as the declared type, so a + // client cannot skip stripping by labeling a JPEG as something else. + needsExifStrip, err := shouldStripExifContent(source, create.Type) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to inspect attachment content: %v", err) + } + if needsExifStrip && !isAndroidMotionContainer(create.Payload.GetMotionMedia()) { release, err := s.acquireImageProcessingSlot(ctx) if err != nil { return nil, status.Errorf(codes.ResourceExhausted, "too many image processing requests") @@ -298,7 +303,7 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAt if request.PageToken != "" { // Simple implementation: page token is the offset as string // In production, you might want to use encrypted tokens - if parsed, err := fmt.Sscanf(request.PageToken, "%d", &offset); err != nil || parsed != 1 { + if parsed, err := fmt.Sscanf(request.PageToken, "%d", &offset); err != nil || parsed != 1 || offset < 0 { return nil, status.Errorf(codes.InvalidArgument, "invalid page token") } } diff --git a/server/api/v1/attachment_service_image.go b/server/api/v1/attachment_service_image.go index dc15965f0..39a3ffbb7 100644 --- a/server/api/v1/attachment_service_image.go +++ b/server/api/v1/attachment_service_image.go @@ -3,14 +3,15 @@ package v1 import ( "bytes" "context" - "image" "io" + "net/http" "github.com/disintegration/imaging" "github.com/pkg/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/usememos/memos/internal/imagelimit" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" ) @@ -45,6 +46,28 @@ func shouldStripExif(mimeType string) bool { return exifCapableImageTypes[mimeType] } +// shouldStripExifContent reports whether the content must be stripped of +// EXIF metadata, judging by the declared type and by sniffing the leading +// bytes. The source is rewound afterwards. +func shouldStripExifContent(source io.ReadSeeker, declaredType string) (bool, error) { + if shouldStripExif(declaredType) { + return true, nil + } + if _, err := source.Seek(0, io.SeekStart); err != nil { + return false, errors.Wrap(err, "failed to rewind attachment content") + } + header := make([]byte, 512) + n, err := io.ReadFull(source, header) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { + return false, errors.Wrap(err, "failed to read attachment header") + } + if _, err := source.Seek(0, io.SeekStart); err != nil { + return false, errors.Wrap(err, "failed to rewind attachment content") + } + sniffed, ok := normalizeMimeType(http.DetectContentType(header[:n])) + return ok && shouldStripExif(sniffed), nil +} + func (s *APIV1Service) acquireImageProcessingSlot(ctx context.Context) (func(), error) { if s.imageProcessingSemaphore == nil { return func() {}, nil @@ -58,19 +81,7 @@ func (s *APIV1Service) acquireImageProcessingSlot(ctx context.Context) (func(), } func validateImageReaderPixelCount(reader io.Reader) error { - config, _, err := image.DecodeConfig(reader) - if err != nil { - // Some formats supported by imaging do not expose dimensions through - // the standard image registry. Let the full decoder handle those. - return nil //nolint:nilerr - } - if config.Width <= 0 || config.Height <= 0 { - return errors.New("invalid image dimensions") - } - if config.Width > maxImagePixels/config.Height { - return errors.Errorf("image dimensions exceed maximum of %d pixels", maxImagePixels) - } - return nil + return imagelimit.CheckReader(reader) } // stripImageExif removes EXIF metadata from image files by decoding and re-encoding them. @@ -79,7 +90,7 @@ func validateImageReaderPixelCount(reader io.Reader) error { // The function preserves the correct image orientation by applying EXIF orientation tags // during decoding before stripping all metadata. Images are re-encoded with high quality // to minimize visual degradation. The re-encoded output is returned in memory; its size -// is bounded by maxImagePixels, which the decoder already has to hold. +// is bounded by imagelimit.MaxPixels, which the decoder already has to hold. // // Supported formats: // - JPEG/JPG: Re-encoded as JPEG with quality 95 diff --git a/server/api/v1/attachment_service_storage.go b/server/api/v1/attachment_service_storage.go index 49d446a75..06e2663a7 100644 --- a/server/api/v1/attachment_service_storage.go +++ b/server/api/v1/attachment_service_storage.go @@ -139,8 +139,15 @@ func saveAttachmentContent( if !strings.Contains(filepathTemplate, "{filename}") { filepathTemplate = filepath.Join(filepathTemplate, "{filename}") } - filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename) - key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, content) + objectKey := replaceFilenameWithPathTemplate(filepathTemplate, create.Filename) + if !strings.Contains(filepathTemplate, "{uuid}") { + // PutObject overwrites silently, and two users can pick the same + // filename in the same second, so a template without a random + // component gets the attachment UID appended to keep keys unique. + ext := filepath.Ext(objectKey) + objectKey = strings.TrimSuffix(objectKey, ext) + "_" + create.UID + ext + } + key, err := driver.UploadObject(ctx, objectKey, create.Type, content) if err != nil { return errors.Wrap(err, "failed to upload via storage driver") } diff --git a/server/api/v1/auth_service_session.go b/server/api/v1/auth_service_session.go index d92bb1c1e..55bb0b3df 100644 --- a/server/api/v1/auth_service_session.go +++ b/server/api/v1/auth_service_session.go @@ -182,6 +182,28 @@ func (s *APIV1Service) RefreshToken(ctx context.Context, _ *v1pb.RefreshTokenReq }, nil } +// currentRefreshTokenID returns the token ID of the refresh cookie on the +// request, or "" when the request carries no valid refresh cookie. +func currentRefreshTokenID(ctx context.Context, secret string) string { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "" + } + cookies := md.Get("cookie") + if len(cookies) == 0 { + return "" + } + refreshToken := auth.ExtractRefreshTokenFromCookie(cookies[0]) + if refreshToken == "" { + return "" + } + claims, err := auth.ParseRefreshToken(refreshToken, []byte(secret)) + if err != nil { + return "" + } + return claims.TokenID +} + func (s *APIV1Service) clearAuthCookies(ctx context.Context) error { // Clear refresh token cookie refreshCookie := s.buildRefreshTokenCookie(ctx, "", time.Time{}) diff --git a/server/api/v1/memo_archive_import.go b/server/api/v1/memo_archive_import.go index 1184a5f44..12227451f 100644 --- a/server/api/v1/memo_archive_import.go +++ b/server/api/v1/memo_archive_import.go @@ -18,6 +18,7 @@ import ( "github.com/usememos/memos/core/memoarchive" "github.com/usememos/memos/core/memopayload" + "github.com/usememos/memos/internal/ratelimit" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store" @@ -167,7 +168,11 @@ func (s *APIV1Service) ImportMemoArchive(ctx context.Context, user *store.User, if err := importer.importMemo(record); err != nil { importer.report.Failed++ importer.report.Failures = append(importer.report.Failures, &v1pb.MemoImportIssue{Memo: record.UID, Message: err.Error()}) + continue } + // An imported memo counts against the same write budget as one + // created through CreateMemo, so an archive cannot bypass it. + s.charge(ratelimit.ScopeWriteUser, userKey(user.ID), 1) } for _, written := range importer.written { if err := ctx.Err(); err != nil { @@ -429,40 +434,58 @@ func (i *memoArchiveImporter) importAttachments(record *memoarchive.Memo, curren for _, attachment := range current { bound[attachment.UID] = struct{}{} } + toBind := make([]*v1pb.Attachment, 0, len(record.Attachments)) + // created holds only attachments this run stored; a pre-existing + // attachment that is merely re-bound is never discarded on failure. created := make([]*v1pb.Attachment, 0, len(record.Attachments)) for index := range record.Attachments { entry := &record.Attachments[index] if _, ok := bound[entry.UID]; ok { continue } - name, err := i.importAttachment(entry) + name, stored, err := i.importAttachment(entry) if err != nil { i.discardAttachments(created) return nil, errors.Wrapf(err, "attachment %s", entry.UID) } - created = append(created, &v1pb.Attachment{Name: name}) + attachment := &v1pb.Attachment{Name: name} + toBind = append(toBind, attachment) + if stored { + created = append(created, attachment) + // Stored bytes count against the same upload budget as an upload. + i.service.charge(ratelimit.ScopeUploadUser, userKey(i.user.ID), 1) + } } - return created, nil + return toBind, nil } // importAttachment stores one attachment entry and returns the resource name -// to bind. -func (i *memoArchiveImporter) importAttachment(entry *memoarchive.Attachment) (string, error) { +// to bind, and whether this run stored a new attachment for it. +func (i *memoArchiveImporter) importAttachment(entry *memoarchive.Attachment) (string, bool, error) { mimeType, ok := normalizeMimeType(entry.Type) if !ok { - return "", errors.Errorf("invalid media type %q", entry.Type) + return "", false, errors.Errorf("invalid media type %q", entry.Type) } if !validateFilename(entry.Filename) { - return "", errors.New("filename contains invalid characters") + return "", false, errors.New("filename contains invalid characters") } uid, err := i.resolveAttachmentUID(entry) if err != nil { - return "", err + return "", false, err } if uid == "" { // An identical unlinked attachment already belongs to the user. - return AttachmentNamePrefix + entry.UID, nil + return AttachmentNamePrefix + entry.UID, false, nil } + name, err := i.storeAttachment(entry, uid, mimeType) + if err != nil { + return "", false, err + } + return name, true, nil +} + +// storeAttachment writes a new attachment for the entry under uid. +func (i *memoArchiveImporter) storeAttachment(entry *memoarchive.Attachment, uid, mimeType string) (string, error) { if entry.ExternalLink != "" { attachment, err := i.service.Store.CreateAttachment(i.ctx, &store.Attachment{ UID: uid, diff --git a/server/api/v1/ratelimit.go b/server/api/v1/ratelimit.go index a430660fd..0ecfb24cc 100644 --- a/server/api/v1/ratelimit.go +++ b/server/api/v1/ratelimit.go @@ -2,6 +2,8 @@ package v1 import ( "context" + "crypto/sha256" + "encoding/hex" "log/slog" "math" "net/http" @@ -14,6 +16,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/usememos/memos/internal/clientip" + "github.com/usememos/memos/internal/identifier" "github.com/usememos/memos/internal/ratelimit" "github.com/usememos/memos/server/auth" ) @@ -122,6 +125,18 @@ func userKey(userID int32) string { return strconv.Itoa(int(userID)) } +// accountKey is the limiter key for a submitted account name. Names longer +// than any valid username are replaced by a digest, so the limiter stores a +// bounded key for an attacker-chosen string while still counting each +// distinct submission separately. +func accountKey(username string) string { + if len(username) <= identifier.MaxUsernameLength { + return username + } + sum := sha256.Sum256([]byte(username)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + // charge records cost units against scope and key. func (s *APIV1Service) charge(scope ratelimit.Scope, key string, cost int) { if s.RateLimiter == nil || key == "" { @@ -198,7 +213,7 @@ type signInAttempt struct { // reserveSignIn consumes the sign-in budgets for an attempt. It returns the // rate-limit error when either budget is spent; a refusal charges nothing. func (s *APIV1Service) reserveSignIn(clientIP, username string) (*signInAttempt, error) { - attempt := &signInAttempt{service: s, clientIP: clientIP, username: username} + attempt := &signInAttempt{service: s, clientIP: clientIP, username: accountKey(username)} if err := s.throttleAndCharge(ratelimit.ScopeSignInIP, clientIP, 1); err != nil { return nil, err } diff --git a/server/api/v1/ratelimit_test.go b/server/api/v1/ratelimit_test.go index aeeba750d..691fe3f3d 100644 --- a/server/api/v1/ratelimit_test.go +++ b/server/api/v1/ratelimit_test.go @@ -1,6 +1,7 @@ package v1 import ( + "strings" "testing" "time" @@ -10,6 +11,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/usememos/memos/internal/identifier" "github.com/usememos/memos/internal/ratelimit" ) @@ -85,3 +87,14 @@ func TestNewChallengeRequiredError(t *testing.T) { require.Equal(t, reasonChallengeRequired, info.Reason) require.Nil(t, rateLimitHTTPHeaders(st)) } + +func TestAccountKeyBoundsLength(t *testing.T) { + require.Equal(t, "alice", accountKey("alice")) + long := strings.Repeat("a", identifier.MaxUsernameLength+1) + key := accountKey(long) + require.True(t, strings.HasPrefix(key, "sha256:")) + require.Len(t, key, len("sha256:")+64) + // Distinct submissions stay distinct; the same submission maps to one key. + require.NotEqual(t, key, accountKey(long+"b")) + require.Equal(t, key, accountKey(long)) +} diff --git a/server/api/v1/request_body_limit_test.go b/server/api/v1/request_body_limit_test.go new file mode 100644 index 000000000..baaf61eba --- /dev/null +++ b/server/api/v1/request_body_limit_test.go @@ -0,0 +1,26 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Only procedures that carry file bytes inline keep the large body cap; every +// other request is bounded by the general cap so buffered bodies cannot be +// used to exhaust memory. +func TestRequestBodyLimitPerProcedure(t *testing.T) { + require.EqualValues(t, uploadRequestLimit, requestBodyLimit(attachmentUploadProcedure)) + require.EqualValues(t, uploadRequestLimit, requestBodyLimit(importMemosProcedure)) + require.EqualValues(t, MaxAPIRequestBytes, requestBodyLimit("/memos.api.v1.AttachmentService/CreateAttachment")) + require.EqualValues(t, MaxAPIRequestBytes, requestBodyLimit("/memos.api.v1.AIService/Transcribe")) + for _, procedure := range []string{ + "/memos.api.v1.AuthService/SignIn", + "/memos.api.v1.UserService/CreateUser", + "/memos.api.v1.MemoService/CreateMemo", + "/memos.api.v1.MemoService/ListMemos", + } { + require.EqualValues(t, maxGeneralRequestBytes, requestBodyLimit(procedure), procedure) + } + require.Less(t, int64(maxGeneralRequestBytes), int64(MaxAPIRequestBytes)) +} diff --git a/server/api/v1/test/attachment_service_s3_test.go b/server/api/v1/test/attachment_service_s3_test.go index a33017aab..109481670 100644 --- a/server/api/v1/test/attachment_service_s3_test.go +++ b/server/api/v1/test/attachment_service_s3_test.go @@ -255,3 +255,42 @@ func upsertS3StorageSetting(ctx context.Context, t testing.TB, ts *TestService, }) require.NoError(t, err) } + +// Two uploads with the same filename under a path template that carries no +// random component must not overwrite each other's object. +func TestS3ObjectKeysStayUniqueWithoutUUIDTemplate(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + fake := fakes3.New(t, "flat") + ctx := context.Background() + storage := fakeStorage("flat-s3", "Flat", fake.Config("flat")) + _, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ + Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{ + FilepathTemplate: "files/{filename}", UploadSizeLimitMb: 30, Storages: []*storepb.Storage{storage}, DefaultStorageId: storage.Id, + }}, + }) + require.NoError(t, err) + + keys := map[string]bool{} + for i, username := range []string{"first-user", "second-user"} { + user, err := ts.CreateRegularUser(ctx, username) + require.NoError(t, err) + userCtx := ts.CreateUserContext(ctx, user.ID) + content := []byte("content of user " + username) + created, err := ts.Service.CreateAttachment(userCtx, &v1pb.CreateAttachmentRequest{Attachment: &v1pb.Attachment{ + Filename: "notes.txt", Type: "text/plain", Content: content, + }}) + require.NoError(t, err) + uid, err := apiv1.ExtractAttachmentUIDFromName(created.Name) + require.NoError(t, err) + row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) + require.NoError(t, err) + key := row.Payload.GetS3Object().GetKey() + require.False(t, keys[key], "upload %d reused key %q", i, key) + keys[key] = true + stored, err := fake.GetObject("flat", key) + require.NoError(t, err) + require.Equal(t, content, stored) + } +} diff --git a/server/api/v1/test/attachment_service_test.go b/server/api/v1/test/attachment_service_test.go index a0300b844..ab43adf43 100644 --- a/server/api/v1/test/attachment_service_test.go +++ b/server/api/v1/test/attachment_service_test.go @@ -960,3 +960,13 @@ func TestDeleteMotionMediaGroupChecksBeyondDefaultAttachmentPage(t *testing.T) { _, err = ts.Service.DeleteAttachment(userCtx, &v1pb.DeleteAttachmentRequest{Name: "attachments/" + video.UID}) require.Equal(t, codes.FailedPrecondition, status.Code(err)) } + +func TestListAttachmentsRejectsNegativePageToken(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + ctx := context.Background() + user, err := ts.CreateRegularUser(ctx, "pager") + require.NoError(t, err) + _, err = ts.Service.ListAttachments(ts.CreateUserContext(ctx, user.ID), &v1pb.ListAttachmentsRequest{PageToken: "-5"}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} diff --git a/server/api/v1/test/memo_archive_test.go b/server/api/v1/test/memo_archive_test.go index c6630199e..78ebb9a0a 100644 --- a/server/api/v1/test/memo_archive_test.go +++ b/server/api/v1/test/memo_archive_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "strconv" "testing" "time" @@ -15,7 +16,9 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/usememos/memos/core/memoarchive" + "github.com/usememos/memos/internal/ratelimit" v1pb "github.com/usememos/memos/proto/gen/api/v1" + storepb "github.com/usememos/memos/proto/gen/store" apiv1 "github.com/usememos/memos/server/api/v1" "github.com/usememos/memos/store" ) @@ -527,3 +530,116 @@ func TestImportMemos(t *testing.T) { require.Equal(t, codes.NotFound, status.Code(err)) }) } + +// buildImportArchive writes a one-memo archive whose attachments are given +// as (uid, filename, bytes). Sizes and digests are computed from the bytes. +func buildImportArchive(t *testing.T, username string, attachments []struct { + uid, filename string + content []byte +}) []byte { + t.Helper() + var buf bytes.Buffer + writer := memoarchive.NewWriter(&buf, mustTime(t, "2026-04-01T00:00:00Z")) + require.NoError(t, writer.WriteManifest(&memoarchive.Manifest{ + Format: memoarchive.Format, FormatVersion: memoarchive.FormatVersion, + Generator: memoarchive.Generator{Name: "test", Version: "1"}, + ExportTime: "2026-04-01T00:00:00Z", + Scope: memoarchive.Scope{Kind: memoarchive.ScopeKindUser, User: &memoarchive.ScopeUser{Username: username}}, + })) + record := &memoarchive.Memo{ + UID: "importedmemo1", Creator: username, CreateTime: "2026-04-01T00:00:00Z", UpdateTime: "2026-04-01T00:00:00Z", + State: "NORMAL", Visibility: "PRIVATE", ContentPath: memoarchive.ContentPath("importedmemo1"), + } + for _, attachment := range attachments { + path := memoarchive.AttachmentPath(attachment.uid, attachment.filename) + digest, size, err := writer.WriteAttachment(path, bytes.NewReader(attachment.content)) + require.NoError(t, err) + record.Attachments = append(record.Attachments, memoarchive.Attachment{ + UID: attachment.uid, Filename: attachment.filename, Type: "application/octet-stream", Size: size, SHA256: digest, Path: path, + CreateTime: "2026-04-01T00:00:00Z", + }) + } + require.NoError(t, writer.WriteMemo(record, []byte("imported memo"))) + require.NoError(t, writer.Close()) + return buf.Bytes() +} + +// A memo that fails to import must not take a pre-existing attachment of the +// user with it: only attachments the run stored are discarded. +func TestImportMemosFailureKeepsPreexistingAttachment(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + t.Cleanup(ts.Service.CloseUploads) + ctx := context.Background() + _, err := ts.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ + Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 1}}, + }) + require.NoError(t, err) + user, err := ts.CreateRegularUser(ctx, "keeper") + require.NoError(t, err) + userCtx := ts.CreateUserContext(ctx, user.ID) + + existingContent := []byte("bytes the user already uploaded") + existing, err := ts.Service.CreateAttachment(userCtx, &v1pb.CreateAttachmentRequest{ + AttachmentId: "keepme0001", + Attachment: &v1pb.Attachment{Filename: "keep.bin", Type: "application/octet-stream", Content: existingContent}, + }) + require.NoError(t, err) + + archive := buildImportArchive(t, "keeper", []struct { + uid, filename string + content []byte + }{ + {"keepme0001", "keep.bin", existingContent}, + {"toolarge01", "big.bin", bytes.Repeat([]byte("x"), 2<<20)}, + }) + response, err := ts.Service.ImportMemos(userCtx, &v1pb.ImportMemosRequest{ + Name: userName(user), + Upload: &v1pb.ImportMemosRequest_Spec{Spec: &v1pb.ImportMemosSpec{TotalSize: int64(len(archive))}}, + Data: archive, + FinishWrite: true, + }) + require.NoError(t, err) + report := response.GetReport() + require.NotNil(t, report) + require.Equal(t, int32(1), report.Failed) + require.Zero(t, report.Created) + + kept, err := ts.Service.GetAttachment(userCtx, &v1pb.GetAttachmentRequest{Name: existing.Name}) + require.NoError(t, err, "the pre-existing attachment survives the failed import") + require.Equal(t, existing.Name, kept.Name) +} + +// Imported memos and stored attachments count against the same per-user +// write and upload budgets as the regular create paths. +func TestImportMemosChargesWriteAndUploadBudgets(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + t.Cleanup(ts.Service.CloseUploads) + limiter := ratelimit.NewMemoryLimiter(ratelimit.DefaultPolicy()) + ts.Service.RateLimiter = limiter + ctx := context.Background() + user, err := ts.CreateRegularUser(ctx, "charged") + require.NoError(t, err) + userCtx := ts.CreateUserContext(ctx, user.ID) + key := strconv.Itoa(int(user.ID)) + writeBefore := limiter.Allowed(ratelimit.ScopeWriteUser, key, 1).Remaining + uploadBefore := limiter.Allowed(ratelimit.ScopeUploadUser, key, 1).Remaining + + archive := buildImportArchive(t, "charged", []struct { + uid, filename string + content []byte + }{{"newfile001", "new.bin", []byte("fresh bytes")}}) + response, err := ts.Service.ImportMemos(userCtx, &v1pb.ImportMemosRequest{ + Name: userName(user), + Upload: &v1pb.ImportMemosRequest_Spec{Spec: &v1pb.ImportMemosSpec{TotalSize: int64(len(archive))}}, + Data: archive, + FinishWrite: true, + }) + require.NoError(t, err) + require.Equal(t, int32(1), response.GetReport().Created) + + require.Equal(t, writeBefore-1, limiter.Allowed(ratelimit.ScopeWriteUser, key, 1).Remaining) + require.Equal(t, uploadBefore-1, limiter.Allowed(ratelimit.ScopeUploadUser, key, 1).Remaining) +} diff --git a/server/api/v1/test/user_service_password_test.go b/server/api/v1/test/user_service_password_test.go new file mode 100644 index 000000000..3ac4d7e07 --- /dev/null +++ b/server/api/v1/test/user_service_password_test.go @@ -0,0 +1,110 @@ +package test + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/fieldmaskpb" + "google.golang.org/protobuf/types/known/timestamppb" + + v1pb "github.com/usememos/memos/proto/gen/api/v1" + storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/server/auth" + "github.com/usememos/memos/store" +) + +func addSession(ctx context.Context, t *testing.T, ts *TestService, userID int32, tokenID string) { + t.Helper() + require.NoError(t, ts.Store.AddUserRefreshToken(ctx, userID, &storepb.RefreshTokensUserSetting_RefreshToken{ + TokenId: tokenID, CreatedAt: timestamppb.Now(), + })) +} + +func sessionIDs(ctx context.Context, t *testing.T, ts *TestService, userID int32) []string { + t.Helper() + tokens, err := ts.Store.GetUserRefreshTokens(ctx, userID) + require.NoError(t, err) + ids := make([]string, 0, len(tokens)) + for _, token := range tokens { + ids = append(ids, token.TokenId) + } + return ids +} + +func changePassword(ctx context.Context, ts *TestService, user *store.User, password string) error { + _, err := ts.Service.UpdateUser(ctx, &v1pb.UpdateUserRequest{ + User: &v1pb.User{Name: "users/" + user.Username, Password: password}, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"password"}}, + }) + return err +} + +// Changing a password ends every other session of the account; the browser +// session that made the change keeps working. +func TestUpdateUserPasswordRevokesOtherSessions(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + ctx := context.Background() + user, err := ts.CreateRegularUser(ctx, "rotator") + require.NoError(t, err) + addSession(ctx, t, ts, user.ID, "current-browser") + addSession(ctx, t, ts, user.ID, "stolen-session") + addSession(ctx, t, ts, user.ID, "old-phone") + + refreshToken, _, err := auth.GenerateRefreshToken(user.ID, "current-browser", []byte(ts.Service.Secret)) + require.NoError(t, err) + userCtx := metadata.NewIncomingContext(ts.CreateUserContext(ctx, user.ID), metadata.Pairs("cookie", auth.RefreshTokenCookieName+"="+refreshToken)) + + require.NoError(t, changePassword(userCtx, ts, user, "new-password")) + require.Equal(t, []string{"current-browser"}, sessionIDs(ctx, t, ts, user.ID)) + + // Without a refresh cookie (an API token caller) every session is revoked. + addSession(ctx, t, ts, user.ID, "another") + require.NoError(t, changePassword(ts.CreateUserContext(ctx, user.ID), ts, user, "newer-password")) + require.Empty(t, sessionIDs(ctx, t, ts, user.ID)) +} + +// An administrator resetting another account's password revokes all of that +// account's sessions, never the administrator's own. +func TestAdminPasswordResetRevokesTargetSessions(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + ctx := context.Background() + admin, err := ts.CreateHostUser(ctx, "admin") + require.NoError(t, err) + target, err := ts.CreateRegularUser(ctx, "target") + require.NoError(t, err) + addSession(ctx, t, ts, admin.ID, "admin-browser") + addSession(ctx, t, ts, target.ID, "target-browser") + + adminToken, _, err := auth.GenerateRefreshToken(admin.ID, "admin-browser", []byte(ts.Service.Secret)) + require.NoError(t, err) + adminCtx := metadata.NewIncomingContext(ts.CreateUserContext(ctx, admin.ID), metadata.Pairs("cookie", auth.RefreshTokenCookieName+"="+adminToken)) + + require.NoError(t, changePassword(adminCtx, ts, target, "reset-by-admin")) + require.Empty(t, sessionIDs(ctx, t, ts, target.ID)) + require.Equal(t, []string{"admin-browser"}, sessionIDs(ctx, t, ts, admin.ID)) +} + +// The avatar size limit the web client applies is enforced by the server too. +func TestUpdateUserAvatarSizeBoundedServerSide(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + ctx := context.Background() + user, err := ts.CreateRegularUser(ctx, "avatar") + require.NoError(t, err) + userCtx := ts.CreateUserContext(ctx, user.ID) + + oversized := "data:image/png;base64," + strings.Repeat("A", 3<<20) + _, err = ts.Service.UpdateUser(userCtx, &v1pb.UpdateUserRequest{ + User: &v1pb.User{Name: "users/" + user.Username, AvatarUrl: oversized}, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"avatar_url"}}, + }) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.Contains(t, err.Error(), "avatar exceeds") +} diff --git a/server/api/v1/user_service.go b/server/api/v1/user_service.go index 5ae637415..2f1995875 100644 --- a/server/api/v1/user_service.go +++ b/server/api/v1/user_service.go @@ -24,6 +24,15 @@ import ( const maxBatchGetUsers = 100 +const ( + // maxAvatarImageBytes is the largest avatar image a user may set. It + // matches the limit the web client enforces before encoding the file. + maxAvatarImageBytes = 2 << 20 + // maxAvatarDataURIBytes bounds the stored data URI: the image bytes + // base64-encoded (4/3 expansion) plus the "data:;base64," prefix. + maxAvatarDataURIBytes = maxAvatarImageBytes/3*4 + 256 +) + func validatePassword(password string) error { if password == "" { return errors.New("password must not be empty") @@ -359,6 +368,7 @@ func (s *APIV1Service) UpdateUser(ctx context.Context, request *v1pb.UpdateUserR if err != nil { return nil, status.Errorf(codes.Internal, "failed to get instance general setting: %v", err) } + passwordChanged := false for _, field := range request.UpdateMask.Paths { switch field { case "username": @@ -383,6 +393,9 @@ func (s *APIV1Service) UpdateUser(ctx context.Context, request *v1pb.UpdateUserR case "avatar_url": // Validate avatar MIME type to prevent XSS during upload if request.User.AvatarUrl != "" { + if len(request.User.AvatarUrl) > maxAvatarDataURIBytes { + return nil, status.Errorf(codes.InvalidArgument, "avatar exceeds the maximum size of %d bytes", maxAvatarImageBytes) + } imageType, _, err := extractImageInfo(request.User.AvatarUrl) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "invalid avatar format: %v", err) @@ -419,6 +432,7 @@ func (s *APIV1Service) UpdateUser(ctx context.Context, request *v1pb.UpdateUserR } passwordHashStr := string(passwordHash) update.PasswordHash = &passwordHashStr + passwordChanged = true case "state": if currentUser.Role != store.RoleAdmin { return nil, status.Errorf(codes.PermissionDenied, "permission denied") @@ -437,6 +451,18 @@ func (s *APIV1Service) UpdateUser(ctx context.Context, request *v1pb.UpdateUserR } return nil, convertUserWriteError(err, "failed to update user") } + if passwordChanged { + // A new password ends every other session. The caller's own browser + // session survives when they changed their own password; an admin + // resetting another account revokes all of that account's sessions. + keepTokenID := "" + if currentUser.ID == user.ID { + keepTokenID = currentRefreshTokenID(ctx, s.Secret) + } + if err := s.Store.RemoveUserRefreshTokensExcept(ctx, user.ID, keepTokenID); err != nil { + slog.Warn("failed to revoke sessions after password change", "user_id", user.ID, "error", err) + } + } return convertUserFromStore(updatedUser, currentUser), nil } diff --git a/server/api/v1/user_service_memo_archive.go b/server/api/v1/user_service_memo_archive.go index b785617cd..2f36e8a7e 100644 --- a/server/api/v1/user_service_memo_archive.go +++ b/server/api/v1/user_service_memo_archive.go @@ -140,6 +140,11 @@ func (s *APIV1Service) finishMemoImport(ctx context.Context, user *store.User, u return status.Errorf(codes.InvalidArgument, "invalid memo archive: %v", err) } if request.ValidateOnly { + // A plan can be requested again and again on the same staged archive, + // each time re-parsing it in full, so it costs an archive unit too. + if err := s.throttleAndCharge(ratelimit.ScopeArchiveUser, userKey(user.ID), 1); err != nil { + return err + } plan, err := s.PlanMemoArchiveImport(ctx, user, archive) if err != nil { slog.Error("memo archive plan failed", slog.Int("user", int(user.ID)), slog.Any("error", err)) diff --git a/server/api/v1/v1.go b/server/api/v1/v1.go index e15d2eeb1..7c270e20c 100644 --- a/server/api/v1/v1.go +++ b/server/api/v1/v1.go @@ -29,13 +29,30 @@ import ( // it derives its own limit from this constant to keep the two gates in lockstep. const MaxAPIRequestBytes = 256 << 20 +// maxGeneralRequestBytes caps every procedure that does not carry file bytes. +// Request bodies are buffered in memory before they are parsed, so the cap is +// what bounds memory per in-flight request; it leaves ample room for the +// largest memo content and settings an instance can be configured to accept. +const maxGeneralRequestBytes = 16 << 20 + +// inlineContentProcedures carry a whole file in one request body and keep +// the large cap; the size of that content is checked again by the handler +// against the instance upload limit. +var inlineContentProcedures = map[string]struct{}{ + "/memos.api.v1.AttachmentService/CreateAttachment": {}, + "/memos.api.v1.AIService/Transcribe": {}, +} + // requestBodyLimit returns the request body cap for a procedure. Chunked // uploads carry at most one chunk per call, so they get a much lower cap. func requestBodyLimit(procedure string) int64 { if procedure == attachmentUploadProcedure || procedure == importMemosProcedure { return uploadRequestLimit } - return MaxAPIRequestBytes + if _, ok := inlineContentProcedures[procedure]; ok { + return MaxAPIRequestBytes + } + return maxGeneralRequestBytes } type APIV1Service struct { diff --git a/server/fileserver/fileserver.go b/server/fileserver/fileserver.go index 263ad634d..e7a945bf1 100644 --- a/server/fileserver/fileserver.go +++ b/server/fileserver/fileserver.go @@ -21,6 +21,7 @@ import ( "golang.org/x/sync/semaphore" "github.com/usememos/memos/core/access" + "github.com/usememos/memos/internal/imagelimit" "github.com/usememos/memos/internal/motionphoto" "github.com/usememos/memos/internal/profile" storepb "github.com/usememos/memos/proto/gen/store" @@ -47,6 +48,11 @@ const ( // maxConcurrentThumbnails limits concurrent thumbnail generation to prevent memory exhaustion. maxConcurrentThumbnails = 3 + // thumbnailFailedMarkerSuffix names the empty file left next to a + // thumbnail that could not be generated, so an image that fails to decode + // is not decoded again on every request. + thumbnailFailedMarkerSuffix = ".failed" + // cacheMaxAge is the max-age value for Cache-Control headers (1 hour). cacheMaxAge = "public, max-age=3600" @@ -89,6 +95,12 @@ var avatarAllowedTypes = map[string]bool{ var errUseOriginalForThumbnail = errors.New("serve original image instead of metadata-stripping thumbnail") +// errThumbnailUnsupported marks an image the server will never thumbnail: +// one whose declared size exceeds the decode bound or whose bytes do not +// decode. Only these failures are remembered; a storage read error is +// transient and must be retried. +var errThumbnailUnsupported = errors.New("image cannot be thumbnailed") + // dataURIRegex parses data URI format: data:image/png;base64,iVBORw0KGgo... var dataURIRegex = regexp.MustCompile(`^data:(?P[^;]+);base64,(?P.+)`) @@ -132,10 +144,10 @@ func (s *FileServerService) serveAttachmentFile(c *echo.Context) error { wantThumbnail := c.QueryParam("thumbnail") == "true" wantMotion := c.QueryParam("motion") == "true" - attachment, err := s.Store.GetAttachment(ctx, &store.FindAttachment{ - UID: &uid, - GetBlob: true, - }) + // Authorize on metadata alone. Database-stored bytes are loaded only + // after the read is permitted, so an unauthorized request never pulls a + // blob out of the database. + attachment, err := s.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "failed to get attachment").Wrap(err) } @@ -150,6 +162,15 @@ func (s *FileServerService) serveAttachmentFile(c *echo.Context) error { if readClass == access.MemoReadClassPublic { c.Response().Header().Set(echo.HeaderCacheControl, publicAttachmentCacheControl) } + if attachment.StorageType != storepb.AttachmentStorageType_LOCAL && attachment.StorageType != storepb.AttachmentStorageType_S3 { + attachment, err = s.Store.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID, GetBlob: true}) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "failed to get attachment").Wrap(err) + } + if attachment == nil { + return echo.NewHTTPError(http.StatusNotFound, "attachment not found") + } + } if wantMotion { return s.serveMotionClip(c, attachment) @@ -401,6 +422,9 @@ func (s *FileServerService) getOrGenerateThumbnail(ctx context.Context, attachme if blob, err := os.ReadFile(thumbnailPath); err == nil { return blob, nil } + if _, err := os.Stat(thumbnailPath + thumbnailFailedMarkerSuffix); err == nil { + return nil, errUseOriginalForThumbnail + } useOriginal, err := s.shouldUseOriginalForThumbnail(ctx, attachment) if err != nil { @@ -420,8 +444,21 @@ func (s *FileServerService) getOrGenerateThumbnail(ctx context.Context, attachme if blob, err := os.ReadFile(thumbnailPath); err == nil { return blob, nil } + if _, err := os.Stat(thumbnailPath + thumbnailFailedMarkerSuffix); err == nil { + return nil, errUseOriginalForThumbnail + } - return s.generateThumbnail(ctx, attachment, thumbnailPath) + blob, err := s.generateThumbnail(ctx, attachment, thumbnailPath) + if errors.Is(err, errThumbnailUnsupported) && ctx.Err() == nil { + // Remember the verdict so the next request serves the original + // without decoding again. A canceled request is not a verdict on the + // image. The marker is removed with the attachment's other derived + // caches. + if markerErr := os.WriteFile(thumbnailPath+thumbnailFailedMarkerSuffix, nil, 0644); markerErr != nil { + slog.Warn("failed to record thumbnail failure", "error", markerErr) + } + } + return blob, err } // getThumbnailPath returns the file path for a cached thumbnail. @@ -498,6 +535,14 @@ func hasThumbnailSensitiveMetadata(data []byte) bool { // generateThumbnail creates a new thumbnail and saves it to disk. func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *store.Attachment, thumbnailPath string) ([]byte, error) { + // Bound the decode before paying for it: the pixel count is read from + // the header, and the upload path only enforces it for the formats it + // re-encodes, so a stored image may still declare far more pixels than + // the server is willing to allocate. + if err := s.checkThumbnailSourceBounds(ctx, attachment); err != nil { + return nil, err + } + reader, err := s.getAttachmentReader(ctx, attachment) if err != nil { return nil, errors.Wrap(err, "failed to get attachment reader") @@ -506,7 +551,7 @@ func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *s img, err := imaging.Decode(reader, imaging.AutoOrientation(true)) if err != nil { - return nil, errors.Wrap(err, "failed to decode image") + return nil, errors.Wrapf(errThumbnailUnsupported, "failed to decode image: %v", err) } width, height := img.Bounds().Dx(), img.Bounds().Dy() @@ -525,6 +570,20 @@ func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *s return buf.Bytes(), nil } +// checkThumbnailSourceBounds reads the image header and refuses images whose +// declared dimensions exceed the decode budget. +func (s *FileServerService) checkThumbnailSourceBounds(ctx context.Context, attachment *store.Attachment) error { + reader, err := s.getAttachmentReader(ctx, attachment) + if err != nil { + return errors.Wrap(err, "failed to open image for size probe") + } + defer reader.Close() + if err := imagelimit.CheckReader(io.LimitReader(reader, thumbnailMetadataProbeSize)); err != nil { + return errors.Wrapf(errThumbnailUnsupported, "thumbnail source exceeds decode bounds: %v", err) + } + return nil +} + // calculateThumbnailDimensions calculates the target dimensions for a thumbnail. // The largest dimension is constrained to thumbnailMaxSize while maintaining aspect ratio. // Small images are not enlarged. diff --git a/server/fileserver/fileserver_test.go b/server/fileserver/fileserver_test.go index 0a04867ff..f960c5eb8 100644 --- a/server/fileserver/fileserver_test.go +++ b/server/fileserver/fileserver_test.go @@ -12,6 +12,8 @@ import ( "image/png" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -31,6 +33,7 @@ import ( apiv1service "github.com/usememos/memos/server/api/v1" "github.com/usememos/memos/server/auth" "github.com/usememos/memos/store" + "github.com/usememos/memos/store/db" teststore "github.com/usememos/memos/store/test" ) @@ -861,3 +864,164 @@ func newExpiredRefreshTokenCookie(ctx context.Context, t *testing.T, stores *sto require.NoError(t, err) return &http.Cookie{Name: auth.RefreshTokenCookieName, Value: refreshToken} } + +// oversizedPNGHeader returns a valid PNG header that declares width×height +// pixels without carrying pixel data, the shape of a decompression bomb. +func oversizedPNGHeader(width, height uint32) []byte { + var buf bytes.Buffer + buf.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:4], width) + binary.BigEndian.PutUint32(ihdr[4:8], height) + ihdr[8] = 8 + ihdr[9] = 2 + for _, chunk := range []struct { + kind string + data []byte + }{{"IHDR", ihdr}, {"IEND", nil}} { + _ = binary.Write(&buf, binary.BigEndian, uint32(len(chunk.data))) + buf.WriteString(chunk.kind) + buf.Write(chunk.data) + _ = binary.Write(&buf, binary.BigEndian, crc32.ChecksumIEEE(append([]byte(chunk.kind), chunk.data...))) + } + return buf.Bytes() +} + +func createPublicImageMemo(ctx context.Context, t *testing.T, svc *apiv1service.APIV1Service, username, filename, mimeType string, content []byte) *apiv1.Attachment { + t.Helper() + creator, err := svc.Store.CreateUser(ctx, &store.User{Username: username, Role: store.RoleUser, Email: username + "@example.com"}) + require.NoError(t, err) + creatorCtx := context.WithValue(ctx, auth.UserIDContextKey, creator.ID) + attachment, err := svc.CreateAttachment(creatorCtx, &apiv1.CreateAttachmentRequest{Attachment: &apiv1.Attachment{ + Filename: filename, Type: mimeType, Content: content, + }}) + require.NoError(t, err) + _, err = svc.CreateMemo(creatorCtx, &apiv1.CreateMemoRequest{Memo: &apiv1.Memo{ + Content: "image memo", Visibility: apiv1.Visibility_PUBLIC, Attachments: []*apiv1.Attachment{{Name: attachment.Name}}, + }}) + require.NoError(t, err) + return attachment +} + +// A small PNG that declares billions of pixels must not be decoded for a +// thumbnail; the original is served and the failure is remembered so later +// requests do not retry the decode. +func TestServeAttachmentFile_ThumbnailRefusesOversizedImage(t *testing.T) { + ctx := context.Background() + svc, fs, _, cleanup := newShareAttachmentTestServices(ctx, t) + defer cleanup() + + bomb := oversizedPNGHeader(100_000, 100_000) + attachment := createPublicImageMemo(ctx, t, svc, "bomb-owner", "bomb.png", "image/png", bomb) + + e := echo.New() + fs.RegisterRoutes(e) + for range 2 { + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/bomb.png?thumbnail=true", attachment.Name), nil)) + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "image/png", rec.Header().Get(echo.HeaderContentType)) + require.Equal(t, bomb, rec.Body.Bytes(), "original is served instead of a thumbnail") + } + uid := strings.TrimPrefix(attachment.Name, "attachments/") + require.NoFileExists(t, filepath.Join(fs.Profile.Data, thumbnailCacheFolder, uid+".v2.jpeg")) + require.FileExists(t, filepath.Join(fs.Profile.Data, thumbnailCacheFolder, uid+".v2.jpeg"+thumbnailFailedMarkerSuffix)) +} + +// newSharedProfileTestServices builds the API service, file server, and +// store on one profile, as in production, so derived caches written by the +// file server live in the directory the store cleans up. +func newSharedProfileTestServices(ctx context.Context, t *testing.T) (*apiv1service.APIV1Service, *FileServerService, func()) { + t.Helper() + dataDir := t.TempDir() + testProfile := &profile.Profile{ + Demo: true, + Version: "test-1.0.0", + InstanceURL: "http://localhost:8080", + Driver: "sqlite", + DSN: filepath.Join(dataDir, "memos_test.db"), + Data: dataDir, + } + dbDriver, err := db.NewDBDriver(testProfile) + require.NoError(t, err) + testStore := store.New(dbDriver, testProfile) + require.NoError(t, testStore.Migrate(ctx)) + setInstanceAccessMode(ctx, t, testStore, storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PUBLIC) + secret := "test-secret" + apiService := &apiv1service.APIV1Service{ + Secret: secret, + Profile: testProfile, + Store: testStore, + MarkdownService: markdown.NewService(markdown.WithTagExtension()), + SSEHub: apiv1service.NewSSEHub(), + } + return apiService, NewFileServerService(testProfile, testStore, secret), func() { testStore.Close() } +} + +// Deleting an attachment removes the cached thumbnail written for it. +func TestDeleteAttachmentRemovesCachedThumbnail(t *testing.T) { + ctx := context.Background() + svc, fs, cleanup := newSharedProfileTestServices(ctx, t) + defer cleanup() + + var pngBuf bytes.Buffer + require.NoError(t, png.Encode(&pngBuf, image.NewRGBA(image.Rect(0, 0, 8, 8)))) + attachment := createPublicImageMemo(ctx, t, svc, "thumb-owner", "small.png", "image/png", pngBuf.Bytes()) + owner, err := svc.Store.GetUser(ctx, &store.FindUser{Username: func() *string { s := "thumb-owner"; return &s }()}) + require.NoError(t, err) + + e := echo.New() + fs.RegisterRoutes(e) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/small.png?thumbnail=true", attachment.Name), nil)) + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "image/jpeg", rec.Header().Get(echo.HeaderContentType)) + uid := strings.TrimPrefix(attachment.Name, "attachments/") + thumbnailPath := filepath.Join(fs.Profile.Data, thumbnailCacheFolder, uid+".v2.jpeg") + require.FileExists(t, thumbnailPath) + + ownerCtx := context.WithValue(ctx, auth.UserIDContextKey, owner.ID) + _, err = svc.DeleteAttachment(ownerCtx, &apiv1.DeleteAttachmentRequest{Name: attachment.Name}) + require.NoError(t, err) + require.NoFileExists(t, thumbnailPath) +} + +// A storage read failure is transient and must not be remembered as a +// thumbnail verdict; only unsupported images get the failure marker. +func TestThumbnailFailureMarkerOnlyForUnsupportedImages(t *testing.T) { + ctx := context.Background() + svc, fs, cleanup := newSharedProfileTestServices(ctx, t) + defer cleanup() + _, err := svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ + Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{ + Storages: []*storepb.Storage{{Id: "local", Name: "Local", Type: storepb.StorageType_STORAGE_TYPE_LOCAL}}, + DefaultStorageId: "local", + }}, + }) + require.NoError(t, err) + + var pngBuf bytes.Buffer + require.NoError(t, png.Encode(&pngBuf, image.NewRGBA(image.Rect(0, 0, 8, 8)))) + attachment := createPublicImageMemo(ctx, t, svc, "transient-owner", "gone.png", "image/png", pngBuf.Bytes()) + uid := strings.TrimPrefix(attachment.Name, "attachments/") + stored, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) + require.NoError(t, err) + require.Equal(t, storepb.AttachmentStorageType_LOCAL, stored.StorageType) + localPath := fs.resolveLocalPath(stored.Reference) + require.NoError(t, os.Rename(localPath, localPath+".away")) + + e := echo.New() + fs.RegisterRoutes(e) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/gone.png?thumbnail=true", attachment.Name), nil)) + marker := filepath.Join(fs.Profile.Data, thumbnailCacheFolder, uid+".v2.jpeg"+thumbnailFailedMarkerSuffix) + require.NoFileExists(t, marker, "a read failure is not a verdict on the image") + + // Once the bytes are back, the thumbnail is generated normally. + require.NoError(t, os.Rename(localPath+".away", localPath)) + rec = httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/gone.png?thumbnail=true", attachment.Name), nil)) + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "image/jpeg", rec.Header().Get(echo.HeaderContentType)) +} diff --git a/server/server.go b/server/server.go index 4d71f8366..17b30e830 100644 --- a/server/server.go +++ b/server/server.go @@ -24,7 +24,17 @@ import ( "github.com/usememos/memos/store" ) -const shutdownTimeout = 10 * time.Second +const ( + shutdownTimeout = 10 * time.Second + + // readHeaderTimeout bounds how long a client may take to send request + // headers, so idle or slow connections cannot pin a worker forever. Bodies + // and responses are not bounded here: uploads and SSE streams are + // legitimately long, and the request context still ends on disconnect. + readHeaderTimeout = 15 * time.Second + // idleTimeout closes keep-alive connections that send nothing. + idleTimeout = 2 * time.Minute +) type Server struct { Secret string @@ -117,7 +127,11 @@ func (s *Server) Start() error { } // Start Echo server directly (no cmux needed - all traffic is HTTP). - s.httpServer = &http.Server{Handler: s.echoServer} + s.httpServer = &http.Server{ + Handler: s.echoServer, + ReadHeaderTimeout: readHeaderTimeout, + IdleTimeout: idleTimeout, + } go func() { if err := s.httpServer.Serve(listener); err != nil && err != http.ErrServerClosed { slog.Error("failed to start echo server", "error", err) diff --git a/store/attachment.go b/store/attachment.go index 2fa922c59..0d801ca6b 100644 --- a/store/attachment.go +++ b/store/attachment.go @@ -381,8 +381,14 @@ func AttachmentNeedsInstanceStorageSetting(attachment *Attachment) bool { return attachment.Payload.GetS3Object() != nil } +// deleteAttachmentDerivedCaches removes every derived asset the file server +// may have written for the attachment: the current thumbnail, its failure +// marker, the pre-v2 thumbnail name, and the extracted motion clip. The names +// must match server/fileserver. func (s *Store) deleteAttachmentDerivedCaches(attachment *Attachment) { for _, cachePath := range []string{ + filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".v2.jpeg"), + filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".v2.jpeg.failed"), filepath.Join(s.profile.Data, thumbnailCacheFolder, attachment.UID+".jpeg"), filepath.Join(s.profile.Data, motionCacheFolder, attachment.UID+".mp4"), } { diff --git a/store/db/sqlite/functions.go b/store/db/sqlite/functions.go index 4e87f927a..b85b342e4 100644 --- a/store/db/sqlite/functions.go +++ b/store/db/sqlite/functions.go @@ -8,6 +8,7 @@ import ( "errors" "regexp" "sync" + "sync/atomic" "golang.org/x/text/cases" msqlite "modernc.org/sqlite" @@ -50,8 +51,16 @@ var ( registerRegexpErr error // regexpCache memoizes compiled patterns; keys are pattern strings. regexpCache sync.Map + // regexpCacheSize counts cached patterns so a stream of distinct + // attacker-chosen patterns cannot grow the cache without bound. + regexpCacheSize atomic.Int64 ) +// maxRegexpCacheEntries caps the compiled-pattern cache. When it is full the +// cache is cleared rather than evicted entry by entry; recompiling a pattern +// is cheap next to letting the cache grow unbounded. +const maxRegexpCacheEntries = 256 + // ensureRegexpRegistered registers a Go-backed `regexp(pattern, value)` scalar // function so SQLite's `value REGEXP pattern` operator works (modernc.org/sqlite // has no built-in implementation). Patterns use Go's RE2 syntax. Registered once @@ -99,6 +108,12 @@ func compileRegexp(pattern string) (*regexp.Regexp, error) { if err != nil { return nil, err } - regexpCache.Store(pattern, re) + if regexpCacheSize.Load() >= maxRegexpCacheEntries { + regexpCache.Clear() + regexpCacheSize.Store(0) + } + if _, loaded := regexpCache.LoadOrStore(pattern, re); !loaded { + regexpCacheSize.Add(1) + } return re, nil } diff --git a/store/db/sqlite/functions_test.go b/store/db/sqlite/functions_test.go index 413708bea..87019eb5d 100644 --- a/store/db/sqlite/functions_test.go +++ b/store/db/sqlite/functions_test.go @@ -1,6 +1,7 @@ package sqlite import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -22,3 +23,16 @@ func TestRegexpFunctionMatches(t *testing.T) { _, err = compileRegexp(`(`) require.Error(t, err) } + +func TestRegexpCacheStaysBounded(t *testing.T) { + regexpCache.Clear() + regexpCacheSize.Store(0) + for i := range maxRegexpCacheEntries * 3 { + _, err := compileRegexp(fmt.Sprintf(`^pattern-%d$`, i)) + require.NoError(t, err) + require.LessOrEqual(t, regexpCacheSize.Load(), int64(maxRegexpCacheEntries)) + } + count := 0 + regexpCache.Range(func(_, _ any) bool { count++; return true }) + require.LessOrEqual(t, count, maxRegexpCacheEntries) +} diff --git a/store/test/user_setting_refresh_prune_test.go b/store/test/user_setting_refresh_prune_test.go new file mode 100644 index 000000000..842dd3efe --- /dev/null +++ b/store/test/user_setting_refresh_prune_test.go @@ -0,0 +1,58 @@ +package test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + + storepb "github.com/usememos/memos/proto/gen/store" +) + +func TestUserSettingRefreshTokensPruneExpiredOnAdd(t *testing.T) { + t.Parallel() + ctx := context.Background() + ts := NewTestingStore(ctx, t) + defer ts.Close() + user, err := createTestingHostUser(ctx, ts) + require.NoError(t, err) + + expired := &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: "expired", ExpiresAt: timestamppb.New(time.Now().Add(-time.Hour))} + live := &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: "live", ExpiresAt: timestamppb.New(time.Now().Add(time.Hour))} + require.NoError(t, ts.AddUserRefreshToken(ctx, user.ID, expired)) + require.NoError(t, ts.AddUserRefreshToken(ctx, user.ID, live)) + require.NoError(t, ts.AddUserRefreshToken(ctx, user.ID, &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: "newest"})) + + tokens, err := ts.GetUserRefreshTokens(ctx, user.ID) + require.NoError(t, err) + ids := []string{} + for _, token := range tokens { + ids = append(ids, token.TokenId) + } + require.Equal(t, []string{"live", "newest"}, ids) +} + +func TestUserSettingRemoveRefreshTokensExcept(t *testing.T) { + t.Parallel() + ctx := context.Background() + ts := NewTestingStore(ctx, t) + defer ts.Close() + user, err := createTestingHostUser(ctx, ts) + require.NoError(t, err) + for _, id := range []string{"a", "b", "c"} { + require.NoError(t, ts.AddUserRefreshToken(ctx, user.ID, &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: id})) + } + + require.NoError(t, ts.RemoveUserRefreshTokensExcept(ctx, user.ID, "b")) + tokens, err := ts.GetUserRefreshTokens(ctx, user.ID) + require.NoError(t, err) + require.Len(t, tokens, 1) + require.Equal(t, "b", tokens[0].TokenId) + + require.NoError(t, ts.RemoveUserRefreshTokensExcept(ctx, user.ID, "")) + tokens, err = ts.GetUserRefreshTokens(ctx, user.ID) + require.NoError(t, err) + require.Empty(t, tokens) +} diff --git a/store/user_setting.go b/store/user_setting.go index dd7031577..a75b16127 100644 --- a/store/user_setting.go +++ b/store/user_setting.go @@ -2,6 +2,7 @@ package store import ( "context" + "time" "github.com/pkg/errors" "google.golang.org/protobuf/encoding/protojson" @@ -170,7 +171,10 @@ func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *st return err } - tokens = append(tokens, token) + // Expired tokens can never authenticate again; drop them here so the + // list is bounded by the tokens issued within one lifetime rather than + // by every sign-in the account ever made. + tokens = append(pruneExpiredRefreshTokens(tokens, time.Now()), token) _, err = s.UpsertUserSetting(ctx, &storepb.UserSetting{ UserId: userID, @@ -184,6 +188,43 @@ func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *st return err } +func pruneExpiredRefreshTokens(tokens []*storepb.RefreshTokensUserSetting_RefreshToken, now time.Time) []*storepb.RefreshTokensUserSetting_RefreshToken { + live := make([]*storepb.RefreshTokensUserSetting_RefreshToken, 0, len(tokens)) + for _, token := range tokens { + if token == nil || (token.ExpiresAt != nil && token.ExpiresAt.AsTime().Before(now)) { + continue + } + live = append(live, token) + } + return live +} + +// RemoveUserRefreshTokensExcept revokes every refresh token of the user other +// than keepTokenID. An empty keepTokenID revokes all of them. +func (s *Store) RemoveUserRefreshTokensExcept(ctx context.Context, userID int32, keepTokenID string) error { + s.refreshTokenMu.Lock() + defer s.refreshTokenMu.Unlock() + + existingTokens, err := s.GetUserRefreshTokens(ctx, userID) + if err != nil { + return err + } + kept := make([]*storepb.RefreshTokensUserSetting_RefreshToken, 0, 1) + for _, token := range existingTokens { + if keepTokenID != "" && token.TokenId == keepTokenID { + kept = append(kept, token) + } + } + _, err = s.UpsertUserSetting(ctx, &storepb.UserSetting{ + UserId: userID, + Key: storepb.UserSetting_REFRESH_TOKENS, + Value: &storepb.UserSetting_RefreshTokens{ + RefreshTokens: &storepb.RefreshTokensUserSetting{RefreshTokens: kept}, + }, + }) + return err +} + // RemoveUserRefreshToken removes a refresh token from the user. func (s *Store) RemoveUserRefreshToken(ctx context.Context, userID int32, tokenID string) error { s.refreshTokenMu.Lock() diff --git a/web/src/components/MemoContent/LinkMetadataCard.tsx b/web/src/components/MemoContent/LinkMetadataCard.tsx index 36d2a329c..e63fe648c 100644 --- a/web/src/components/MemoContent/LinkMetadataCard.tsx +++ b/web/src/components/MemoContent/LinkMetadataCard.tsx @@ -59,6 +59,7 @@ const LinkMetadataCard = ({ url, fallback, enabled = true }: LinkMetadataCardPro className="h-full w-full object-cover transition-transform duration-200 group-hover:scale-[1.01]" loading="lazy" decoding="async" + referrerPolicy="no-referrer" onError={() => setImageFailed(true)} /> diff --git a/web/src/components/MemoContent/constants.ts b/web/src/components/MemoContent/constants.ts index b91efd8e1..faec6a24f 100644 --- a/web/src/components/MemoContent/constants.ts +++ b/web/src/components/MemoContent/constants.ts @@ -57,7 +57,9 @@ export const SANITIZE_SCHEMA = { "loading", ], }, - tagNames: [...(defaultSchema.tagNames || []), "iframe"], + // `picture`/`source` carry `srcSet`, which the https-only `src` rule below does not cover; + // drop them so every remote image goes through the `img` `src` rule. + tagNames: [...(defaultSchema.tagNames || []).filter((tag) => tag !== "picture" && tag !== "source"), "iframe"], protocols: { ...defaultSchema.protocols, src: ["https"], diff --git a/web/src/components/MemoContent/markdown/Image.tsx b/web/src/components/MemoContent/markdown/Image.tsx index ebcb58643..9e84e6e75 100644 --- a/web/src/components/MemoContent/markdown/Image.tsx +++ b/web/src/components/MemoContent/markdown/Image.tsx @@ -16,6 +16,7 @@ export const Image = ({ className, alt, node: _node, height, width, style, ...pr {...props} loading="lazy" decoding="async" + referrerPolicy="no-referrer" /> ); }; diff --git a/web/tests/memo-content-security.test.tsx b/web/tests/memo-content-security.test.tsx index 7fd059dcf..cc108165f 100644 --- a/web/tests/memo-content-security.test.tsx +++ b/web/tests/memo-content-security.test.tsx @@ -76,6 +76,17 @@ describe("trusted iframe providers", () => { expect(isTrustedIframeSrc("https://evil.example/embed/abc123")).toBe(false); }); + it("drops picture sources that would bypass the https-only image rule", () => { + const html = renderMemoContent( + '', + ); + + expect(html).not.toMatch(/ { const trusted = renderMemoContent(''); const untrusted = renderMemoContent('');