package api import ( "crypto/rand" "crypto/sha256" "io" "os" "path/filepath" "sync" "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // Limits shared by every chunked upload RPC. const ( uploadChunkSize = 2 << 20 uploadRequestLimit = 4 << 20 uploadTTL = 30 * time.Minute uploadMaxSessions = 1024 uploadMaxActive = 128 uploadMaxActivePerUser = 8 ) // uploadSession is one chunked upload staged in a temporary file. Upload // state is private to the process. A session holds the last chunk's digest, // never its contents, plus a kind-specific state T. type uploadSession[T any] struct { mu sync.Mutex ownerID int32 path string totalSize int64 committedSize int64 // expireTime is zeroed to revoke an upload before its TTL elapses. expireTime time.Time // lastDigest identifies the most recently accepted chunk so a client can // safely resend it after losing the response. lastDigest [sha256.Size]byte complete bool state T } // write appends one chunk at writeOffset. With no data and finishWrite // false it only reports progress. finishWrite requires the committed size to // equal the total size once the data is written. func (u *uploadSession[T]) write(writeOffset int64, data []byte, finishWrite bool) error { if len(data) == 0 { if finishWrite && writeOffset != u.committedSize { return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size") } return nil } digest := sha256.Sum256(data) if u.committedSize > 0 && writeOffset+int64(len(data)) == u.committedSize && digest == u.lastDigest { return nil // A lost response can be retried without appending bytes twice. } if u.complete { return status.Errorf(codes.FailedPrecondition, "upload is already complete") } if writeOffset != u.committedSize { return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size") } if int64(len(data)) > u.totalSize-u.committedSize { return status.Errorf(codes.InvalidArgument, "data exceeds total_size") } file, err := os.OpenFile(u.path, os.O_WRONLY, 0600) if err != nil { return status.Errorf(codes.Internal, "failed to open upload file: %v", err) } defer file.Close() n, err := file.WriteAt(data, u.committedSize) if err == nil && n != len(data) { err = io.ErrShortWrite } if err != nil { return status.Errorf(codes.Internal, "failed to write upload file: %v", err) } if err := file.Sync(); err != nil { return status.Errorf(codes.Internal, "failed to sync upload file: %v", err) } u.committedSize += int64(len(data)) u.lastDigest = digest return nil } // uploadSessions tracks the sessions of one upload kind, sweeps expired ones // and orphaned files under its temp prefix, and bounds how many are active. type uploadSessions[T any] struct { mu sync.Mutex prefix string entries map[string]*uploadSession[T] stop chan struct{} done chan struct{} closed bool } func (m *uploadSessions[T]) startLocked(dir, prefix string) { if m.entries != nil { return } m.prefix = prefix m.entries = make(map[string]*uploadSession[T]) m.stop = make(chan struct{}) m.done = make(chan struct{}) go func() { defer close(m.done) ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { select { case <-m.stop: return case now := <-ticker.C: m.mu.Lock() m.sweepLocked(now, 0) m.removeOrphansLocked(dir, now) m.mu.Unlock() } } }() } // sweepLocked drops expired uploads and counts the unfinished ones, both in // total and for ownerID. An upload locked by an in-flight request is // unfinished by definition. func (m *uploadSessions[T]) sweepLocked(now time.Time, ownerID int32) (owned, total int) { count := func(upload *uploadSession[T]) { total++ if upload.ownerID == ownerID { owned++ } } for id, upload := range m.entries { // Never hold the manager lock while waiting for a file write or finalize. if !upload.mu.TryLock() { count(upload) continue } if !now.Before(upload.expireTime) { os.Remove(upload.path) delete(m.entries, id) } else if !upload.complete { count(upload) } upload.mu.Unlock() } return owned, total } func (m *uploadSessions[T]) removeOrphansLocked(dir string, now time.Time) { tracked := make(map[string]bool, len(m.entries)) for _, upload := range m.entries { tracked[upload.path] = true // path is immutable after insertion. } paths, _ := filepath.Glob(filepath.Join(dir, m.prefix+"*")) for _, path := range paths { if tracked[path] { continue } if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && now.Sub(info.ModTime()) >= uploadTTL { os.Remove(path) } } } // create registers a new session for ownerID with totalSize bytes to come, // backed by a temp file under dir named with prefix, and returns its id. func (m *uploadSessions[T]) create(dir, prefix string, ownerID int32, totalSize int64, state T) (string, *uploadSession[T], error) { m.mu.Lock() defer m.mu.Unlock() if m.closed { return "", nil, status.Errorf(codes.Unavailable, "server is shutting down") } m.startLocked(dir, prefix) owned, total := m.sweepLocked(time.Now(), ownerID) if owned >= uploadMaxActivePerUser { return "", nil, status.Errorf(codes.ResourceExhausted, "too many active uploads") } if len(m.entries) >= uploadMaxSessions || total >= uploadMaxActive { return "", nil, status.Errorf(codes.ResourceExhausted, "too many uploads") } file, err := os.CreateTemp(dir, prefix+"*") if err != nil { return "", nil, status.Errorf(codes.Internal, "failed to create upload file: %v", err) } if err := file.Close(); err != nil { os.Remove(file.Name()) return "", nil, status.Errorf(codes.Internal, "failed to close upload file: %v", err) } upload := &uploadSession[T]{ ownerID: ownerID, path: file.Name(), totalSize: totalSize, expireTime: time.Now().Add(uploadTTL), state: state, } id := rand.Text() m.entries[id] = upload return id, upload, nil } func (m *uploadSessions[T]) get(id string, ownerID int32) (*uploadSession[T], error) { m.mu.Lock() upload := m.entries[id] closed := m.closed m.mu.Unlock() if closed { return nil, status.Errorf(codes.Unavailable, "server is shutting down") } if upload == nil || upload.ownerID != ownerID { return nil, status.Errorf(codes.NotFound, "upload not found or expired") } return upload, nil } // resume looks up and locks the session for a continuing call. The caller // unlocks it. func (m *uploadSessions[T]) resume(id string, ownerID int32) (*uploadSession[T], error) { upload, err := m.get(id, ownerID) if err != nil { return nil, err } upload.mu.Lock() if !time.Now().Before(upload.expireTime) { upload.mu.Unlock() return nil, status.Errorf(codes.NotFound, "upload not found or expired") } return upload, nil } // close stops expiration work and removes pending upload files. func (m *uploadSessions[T]) close() { m.mu.Lock() if m.closed { m.mu.Unlock() return } m.closed = true if m.stop != nil { close(m.stop) } done := m.done m.mu.Unlock() if done != nil { <-done } m.mu.Lock() defer m.mu.Unlock() for id, upload := range m.entries { upload.mu.Lock() upload.expireTime = time.Time{} os.Remove(upload.path) upload.mu.Unlock() delete(m.entries, id) } } // CloseUploads stops every upload manager and removes pending upload files. // Call it after draining HTTP requests during server shutdown. func (s *APIService) CloseUploads() { s.attachmentUploads.close() s.memoImports.close() }