mirror of
https://github.com/usememos/memos.git
synced 2026-11-19 18:39:12 +01:00
265 lines
7.4 KiB
Go
265 lines
7.4 KiB
Go
package api
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// Limits shared by every chunked upload RPC.
|
|
const (
|
|
uploadChunkSize = 2 << 20
|
|
uploadRequestLimit = 4 << 20
|
|
uploadTTL = 30 * time.Minute
|
|
uploadMaxSessions = 1024
|
|
uploadMaxActive = 128
|
|
uploadMaxActivePerUser = 8
|
|
)
|
|
|
|
// uploadSession is one chunked upload staged in a temporary file. Upload
|
|
// state is private to the process. A session holds the last chunk's digest,
|
|
// never its contents, plus a kind-specific state T.
|
|
type uploadSession[T any] struct {
|
|
mu sync.Mutex
|
|
ownerID int32
|
|
path string
|
|
totalSize int64
|
|
committedSize int64
|
|
// expireTime is zeroed to revoke an upload before its TTL elapses.
|
|
expireTime time.Time
|
|
// lastDigest identifies the most recently accepted chunk so a client can
|
|
// safely resend it after losing the response.
|
|
lastDigest [sha256.Size]byte
|
|
complete bool
|
|
state T
|
|
}
|
|
|
|
// write appends one chunk at writeOffset. With no data and finishWrite
|
|
// false it only reports progress. finishWrite requires the committed size to
|
|
// equal the total size once the data is written.
|
|
func (u *uploadSession[T]) write(writeOffset int64, data []byte, finishWrite bool) error {
|
|
if len(data) == 0 {
|
|
if finishWrite && writeOffset != u.committedSize {
|
|
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
|
|
}
|
|
return nil
|
|
}
|
|
digest := sha256.Sum256(data)
|
|
if u.committedSize > 0 && writeOffset+int64(len(data)) == u.committedSize && digest == u.lastDigest {
|
|
return nil // A lost response can be retried without appending bytes twice.
|
|
}
|
|
if u.complete {
|
|
return status.Errorf(codes.FailedPrecondition, "upload is already complete")
|
|
}
|
|
if writeOffset != u.committedSize {
|
|
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
|
|
}
|
|
if int64(len(data)) > u.totalSize-u.committedSize {
|
|
return status.Errorf(codes.InvalidArgument, "data exceeds total_size")
|
|
}
|
|
file, err := os.OpenFile(u.path, os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return status.Errorf(codes.Internal, "failed to open upload file: %v", err)
|
|
}
|
|
defer file.Close()
|
|
n, err := file.WriteAt(data, u.committedSize)
|
|
if err == nil && n != len(data) {
|
|
err = io.ErrShortWrite
|
|
}
|
|
if err != nil {
|
|
return status.Errorf(codes.Internal, "failed to write upload file: %v", err)
|
|
}
|
|
if err := file.Sync(); err != nil {
|
|
return status.Errorf(codes.Internal, "failed to sync upload file: %v", err)
|
|
}
|
|
u.committedSize += int64(len(data))
|
|
u.lastDigest = digest
|
|
return nil
|
|
}
|
|
|
|
// uploadSessions tracks the sessions of one upload kind, sweeps expired ones
|
|
// and orphaned files under its temp prefix, and bounds how many are active.
|
|
type uploadSessions[T any] struct {
|
|
mu sync.Mutex
|
|
prefix string
|
|
entries map[string]*uploadSession[T]
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
closed bool
|
|
}
|
|
|
|
func (m *uploadSessions[T]) startLocked(dir, prefix string) {
|
|
if m.entries != nil {
|
|
return
|
|
}
|
|
m.prefix = prefix
|
|
m.entries = make(map[string]*uploadSession[T])
|
|
m.stop = make(chan struct{})
|
|
m.done = make(chan struct{})
|
|
go func() {
|
|
defer close(m.done)
|
|
ticker := time.NewTicker(time.Minute)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-m.stop:
|
|
return
|
|
case now := <-ticker.C:
|
|
m.mu.Lock()
|
|
m.sweepLocked(now, 0)
|
|
m.removeOrphansLocked(dir, now)
|
|
m.mu.Unlock()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// sweepLocked drops expired uploads and counts the unfinished ones, both in
|
|
// total and for ownerID. An upload locked by an in-flight request is
|
|
// unfinished by definition.
|
|
func (m *uploadSessions[T]) sweepLocked(now time.Time, ownerID int32) (owned, total int) {
|
|
count := func(upload *uploadSession[T]) {
|
|
total++
|
|
if upload.ownerID == ownerID {
|
|
owned++
|
|
}
|
|
}
|
|
for id, upload := range m.entries {
|
|
// Never hold the manager lock while waiting for a file write or finalize.
|
|
if !upload.mu.TryLock() {
|
|
count(upload)
|
|
continue
|
|
}
|
|
if !now.Before(upload.expireTime) {
|
|
os.Remove(upload.path)
|
|
delete(m.entries, id)
|
|
} else if !upload.complete {
|
|
count(upload)
|
|
}
|
|
upload.mu.Unlock()
|
|
}
|
|
return owned, total
|
|
}
|
|
|
|
func (m *uploadSessions[T]) removeOrphansLocked(dir string, now time.Time) {
|
|
tracked := make(map[string]bool, len(m.entries))
|
|
for _, upload := range m.entries {
|
|
tracked[upload.path] = true // path is immutable after insertion.
|
|
}
|
|
paths, _ := filepath.Glob(filepath.Join(dir, m.prefix+"*"))
|
|
for _, path := range paths {
|
|
if tracked[path] {
|
|
continue
|
|
}
|
|
if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && now.Sub(info.ModTime()) >= uploadTTL {
|
|
os.Remove(path)
|
|
}
|
|
}
|
|
}
|
|
|
|
// create registers a new session for ownerID with totalSize bytes to come,
|
|
// backed by a temp file under dir named with prefix, and returns its id.
|
|
func (m *uploadSessions[T]) create(dir, prefix string, ownerID int32, totalSize int64, state T) (string, *uploadSession[T], error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if m.closed {
|
|
return "", nil, status.Errorf(codes.Unavailable, "server is shutting down")
|
|
}
|
|
m.startLocked(dir, prefix)
|
|
owned, total := m.sweepLocked(time.Now(), ownerID)
|
|
if owned >= uploadMaxActivePerUser {
|
|
return "", nil, status.Errorf(codes.ResourceExhausted, "too many active uploads")
|
|
}
|
|
if len(m.entries) >= uploadMaxSessions || total >= uploadMaxActive {
|
|
return "", nil, status.Errorf(codes.ResourceExhausted, "too many uploads")
|
|
}
|
|
file, err := os.CreateTemp(dir, prefix+"*")
|
|
if err != nil {
|
|
return "", nil, status.Errorf(codes.Internal, "failed to create upload file: %v", err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
os.Remove(file.Name())
|
|
return "", nil, status.Errorf(codes.Internal, "failed to close upload file: %v", err)
|
|
}
|
|
upload := &uploadSession[T]{
|
|
ownerID: ownerID,
|
|
path: file.Name(),
|
|
totalSize: totalSize,
|
|
expireTime: time.Now().Add(uploadTTL),
|
|
state: state,
|
|
}
|
|
id := rand.Text()
|
|
m.entries[id] = upload
|
|
return id, upload, nil
|
|
}
|
|
|
|
func (m *uploadSessions[T]) get(id string, ownerID int32) (*uploadSession[T], error) {
|
|
m.mu.Lock()
|
|
upload := m.entries[id]
|
|
closed := m.closed
|
|
m.mu.Unlock()
|
|
if closed {
|
|
return nil, status.Errorf(codes.Unavailable, "server is shutting down")
|
|
}
|
|
if upload == nil || upload.ownerID != ownerID {
|
|
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
|
|
}
|
|
return upload, nil
|
|
}
|
|
|
|
// resume looks up and locks the session for a continuing call. The caller
|
|
// unlocks it.
|
|
func (m *uploadSessions[T]) resume(id string, ownerID int32) (*uploadSession[T], error) {
|
|
upload, err := m.get(id, ownerID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
upload.mu.Lock()
|
|
if !time.Now().Before(upload.expireTime) {
|
|
upload.mu.Unlock()
|
|
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
|
|
}
|
|
return upload, nil
|
|
}
|
|
|
|
// close stops expiration work and removes pending upload files.
|
|
func (m *uploadSessions[T]) close() {
|
|
m.mu.Lock()
|
|
if m.closed {
|
|
m.mu.Unlock()
|
|
return
|
|
}
|
|
m.closed = true
|
|
if m.stop != nil {
|
|
close(m.stop)
|
|
}
|
|
done := m.done
|
|
m.mu.Unlock()
|
|
if done != nil {
|
|
<-done
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
for id, upload := range m.entries {
|
|
upload.mu.Lock()
|
|
upload.expireTime = time.Time{}
|
|
os.Remove(upload.path)
|
|
upload.mu.Unlock()
|
|
delete(m.entries, id)
|
|
}
|
|
}
|
|
|
|
// CloseUploads stops every upload manager and removes pending upload files.
|
|
// Call it after draining HTTP requests during server shutdown.
|
|
func (s *APIService) CloseUploads() {
|
|
s.attachmentUploads.close()
|
|
s.memoImports.close()
|
|
}
|