Files

345 lines
14 KiB
Go

package api
import (
"context"
"log/slog"
"net/http"
"strings"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/encoding/protojson"
"connectrpc.com/connect"
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
"github.com/pkg/errors"
"golang.org/x/sync/semaphore"
"github.com/usememos/memos/core/notification"
"github.com/usememos/memos/internal/httphandler"
"github.com/usememos/memos/internal/linkmeta"
"github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/internal/ratelimit"
"github.com/usememos/memos/markdown"
apipb "github.com/usememos/memos/proto/gen/api"
"github.com/usememos/memos/server/auth"
"github.com/usememos/memos/store"
)
// MaxAPIRequestBytes caps the size of a request body accepted by the API. The
// in-process MCP endpoint forwards every tool call through these same routes, so
// it derives its own limit from this constant to keep the two gates in lockstep.
const MaxAPIRequestBytes = 256 << 20
// maxGeneralRequestBytes caps every procedure that does not carry file bytes.
// Request bodies are buffered in memory before they are parsed, so the cap is
// what bounds memory per in-flight request; it leaves ample room for the
// largest memo content and settings an instance can be configured to accept.
const maxGeneralRequestBytes = 16 << 20
// inlineContentProcedures carry a whole file in one request body and keep
// the large cap; the size of that content is checked again by the handler
// against the user's upload limits.
var inlineContentProcedures = map[string]struct{}{
"/memos.api.AttachmentService/CreateAttachment": {},
"/memos.api.AIService/Transcribe": {},
}
// requestBodyLimit returns the request body cap for a procedure. Chunked
// uploads carry at most one chunk per call, so they get a much lower cap.
func requestBodyLimit(procedure string) int64 {
if procedure == attachmentUploadProcedure || procedure == importMemosProcedure {
return uploadRequestLimit
}
if _, ok := inlineContentProcedures[procedure]; ok {
return MaxAPIRequestBytes
}
return maxGeneralRequestBytes
}
type APIService struct {
apipb.UnimplementedInstanceServiceServer
apipb.UnimplementedAuthServiceServer
apipb.UnimplementedUserServiceServer
apipb.UnimplementedMemoServiceServer
apipb.UnimplementedSpaceServiceServer
apipb.UnimplementedAttachmentServiceServer
apipb.UnimplementedAIServiceServer
apipb.UnimplementedIdentityProviderServiceServer
Secret string
Profile *profile.Profile
Store *store.Store
MarkdownService markdown.Service
SSEHub *SSEHub
NotificationEmailSender notification.EmailSender
// RateLimiter bounds request rates; nil disables every limit.
RateLimiter ratelimit.Limiter
// Challenge verifies proof-of-humanity tokens on signup and password
// sign-in; nil means no challenge is configured.
Challenge ratelimit.Challenge
// SignupPolicy decides whether a self-service registration may proceed;
// nil allows every registration.
SignupPolicy ratelimit.SignupPolicy
// thumbnailSemaphore limits concurrent thumbnail generation to prevent memory exhaustion
thumbnailSemaphore *semaphore.Weighted
imageProcessingSemaphore *semaphore.Weighted
// instanceStatsCache memoizes GetInstanceStats results for instanceStatsCacheTTL.
instanceStatsCache instanceStatsCache
linkMetadataFetcher linkMetadataFetcher
attachmentUploads attachmentUploads
attachmentReplaces attachmentReplaceLocks
memoImports memoImports
}
// NewAPIService creates an API v1 service with its shared dependencies.
func NewAPIService(secret string, profile *profile.Profile, store *store.Store) *APIService {
markdownService := markdown.NewService(
markdown.WithTagExtension(),
markdown.WithMentionExtension(),
)
service := &APIService{
Secret: secret,
Profile: profile,
Store: store,
MarkdownService: markdownService,
SSEHub: NewSSEHub(),
NotificationEmailSender: nil,
RateLimiter: newProfileRateLimiter(profile),
thumbnailSemaphore: semaphore.NewWeighted(3), // Limit to 3 concurrent thumbnail generations
imageProcessingSemaphore: semaphore.NewWeighted(2),
}
service.linkMetadataFetcher = linkmeta.NewHTMLMetaFetcher()
return service
}
// newGatewayMarshaler mirrors grpc-gateway's default JSON marshaler with one
// change: EmitDefaultValues replaces EmitUnpopulated. Both keep proto3 scalar
// defaults ("" / 0 / false) and empty lists in the payload — the generated
// OpenAPI schema declares several of them required — but EmitUnpopulated also
// writes `null` for every unset message field (e.g. Attachment.motion_media on
// a plain image). No schema marks those fields nullable, so a client that
// validates responses against the spec rejects them; the MCP tools serve the
// same schema as their outputSchema, and strict MCP clients fail every call
// that returns an attachment. EmitDefaultValues omits unset message fields
// instead, which the schema already allows.
func newGatewayMarshaler() *runtime.HTTPBodyMarshaler {
return &runtime.HTTPBodyMarshaler{
Marshaler: &runtime.JSONPb{
EmitDefaultValues: true,
DiscardUnknown: true,
},
}
}
// NewHandler serves REST, Connect, and SSE with shared authentication policies.
func (s *APIService) NewHandler(ctx context.Context) (http.Handler, error) {
// Shared authorizer: one source of truth for authentication and anonymous-access
// policy, used by both the gRPC-Gateway middleware and the Connect interceptor.
authorizer := NewAuthorizer(s.Store, s.Secret).WithRateLimiter(s.RateLimiter)
// grpc-gateway does not hand the matched procedure to middleware:
// runtime.RPCMethod is only populated by the generated handler, which runs
// after middleware. Resolve the procedure from the proto HTTP bindings
// instead so the policy check actually runs on this transport.
routeResolver, err := newGatewayRouteResolver()
if err != nil {
return nil, errors.Wrap(err, "failed to build gateway route resolver")
}
gatewayAuthMiddleware := func(next runtime.HandlerFunc) runtime.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) {
// grpc-gateway does not pass through the Connect metadata interceptor.
// Apply the same no-store policy here so a memo that later loses access
// cannot remain readable from a browser or intermediary response cache.
setAPIResponseNoStoreHeaders(w.Header())
ctx := r.Context()
authHeader := r.Header.Get("Authorization")
result := authorizer.Authenticate(ctx, authHeader)
// An unresolved path yields an empty procedure, which CheckAccess
// treats as protected: authenticated callers pass and anonymous ones
// are refused. Failing closed keeps a routing gap from becoming an
// access-control gap.
procedure, _ := routeResolver.resolveRequest(r)
if err := authorizer.CheckAccess(ctx, procedure, result); err != nil {
writeGatewayAuthorizationError(w, err)
return
}
if err := authorizer.Throttle(ctx, procedure, result); err != nil {
writeGatewayStatusError(w, err)
return
}
// Apply the identity to the context (no-op for permitted anonymous requests).
if result != nil {
r = r.WithContext(auth.ApplyToContext(ctx, result))
}
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(procedure))
next(w, r, pathParams)
}
}
// Create gRPC-Gateway mux with auth middleware.
gwMux := runtime.NewServeMux(
runtime.WithMarshalerOption(runtime.MIMEWildcard, newGatewayMarshaler()),
runtime.WithMiddlewares(gatewayAuthMiddleware),
runtime.WithErrorHandler(gatewayErrorHandler),
runtime.WithIncomingHeaderMatcher(gatewayIncomingHeaderMatcher),
)
if err := apipb.RegisterInstanceServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterAuthServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterUserServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterMemoServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterSpaceServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterAttachmentServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterAIServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
if err := apipb.RegisterIdentityProviderServiceHandlerServer(ctx, gwMux, s); err != nil {
return nil, err
}
// Connect handlers for browser clients (replaces grpc-web).
logStacktraces := s.Profile.Demo
connectInterceptors := connect.WithInterceptors(
NewMetadataInterceptor(), // Convert HTTP headers to gRPC metadata first
NewLoggingInterceptor(logStacktraces),
NewRecoveryInterceptor(logStacktraces),
NewAuthInterceptor(authorizer),
)
connectMux := http.NewServeMux()
connectHandler := NewConnectServiceHandler(s)
connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors,
// Bound the decompressed message as well as the wire bytes below.
connect.WithConditionalHandlerOptions(func(spec connect.Spec) []connect.HandlerOption {
return []connect.HandlerOption{connect.WithReadMaxBytes(int(requestBodyLimit(spec.Procedure)))}
}))
// Prefixes within a path segment (memos.api.*) are dispatched explicitly;
// ServeMux wildcards match complete segments only.
sseMux := http.NewServeMux()
RegisterSSERoutes(sseMux, s.SSEHub, s.Store, s.Secret)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestPath := r.URL.Path
if r.URL.RawPath != "" {
requestPath = r.URL.RawPath
}
switch {
case requestPath == "/api/sse" && r.Method == http.MethodGet:
sseMux.ServeHTTP(w, r)
case strings.HasPrefix(requestPath, "/api/v1/"):
removedAPIVersionHandler(false).ServeHTTP(w, r)
case strings.HasPrefix(requestPath, "/memos.api.v1."):
removedAPIVersionHandler(true).ServeHTTP(w, r)
case strings.HasPrefix(requestPath, "/memos.api."):
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(r.URL.Path))
connectMux.ServeHTTP(w, r)
case strings.HasPrefix(requestPath, "/api/") || strings.HasPrefix(requestPath, "/file/"):
gwMux.ServeHTTP(w, r)
default:
httphandler.NotFound(w, r)
}
}), nil
}
// removedAPIMessage tells clients of the removed v1 API where it went.
const removedAPIMessage = "API v1 has been removed. Use /api/... for REST and /memos.api.<Service>/<Method> for Connect. " +
"See https://github.com/usememos/memos/blob/main/proto/api/CHANGELOG.md"
// removedAPIVersionHandler answers requests to the removed /api/v1 and
// memos.api.v1 routes with 410 Gone, in the error body shape each transport's
// clients decode. It exists for one release so old clients fail with a
// pointer to the migration; delete it in the release after.
func removedAPIVersionHandler(connectProtocol bool) http.Handler {
return httphandler.Func(func(w http.ResponseWriter, _ *http.Request) error {
if connectProtocol {
return httphandler.JSON(w, http.StatusGone, map[string]string{"code": connect.CodeUnimplemented.String(), "message": removedAPIMessage})
}
return httphandler.JSON(w, http.StatusGone, map[string]any{"code": int(codes.Unimplemented), "message": removedAPIMessage})
})
}
func setAPIResponseNoStoreHeaders(header http.Header) {
header.Set("Cache-Control", "no-cache, no-store, must-revalidate")
header.Set("Pragma", "no-cache")
header.Set("Expires", "0")
}
// newProfileRateLimiter builds the default limiter, or nil when the profile
// turns rate limiting off.
func newProfileRateLimiter(profile *profile.Profile) ratelimit.Limiter {
if !profile.RateLimit {
return nil
}
return ratelimit.NewMemoryLimiter(ratelimit.DefaultPolicy())
}
// gatewayIncomingHeaderMatcher forwards the challenge token to metadata on top
// of the gateway's default set.
func gatewayIncomingHeaderMatcher(key string) (string, bool) {
if http.CanonicalHeaderKey(key) == challengeTokenHeader {
return challengeTokenMetadataKey, true
}
return runtime.DefaultHeaderMatcher(key)
}
// gatewayErrorHandler adds the rate-limit header fields to a refusal before
// the default handler writes the status body.
func gatewayErrorHandler(ctx context.Context, mux *runtime.ServeMux, marshaler runtime.Marshaler, w http.ResponseWriter, r *http.Request, err error) {
if st, ok := status.FromError(err); ok {
for key, values := range rateLimitHTTPHeaders(st) {
for _, value := range values {
w.Header().Add(key, value)
}
}
}
runtime.DefaultHTTPErrorHandler(ctx, mux, marshaler, w, r, err)
}
// writeGatewayStatusError writes a gRPC status as the gateway's JSON error body,
// including its details and any rate-limit header fields.
func writeGatewayStatusError(w http.ResponseWriter, err error) {
st := status.Convert(err)
for key, values := range rateLimitHTTPHeaders(st) {
for _, value := range values {
w.Header().Add(key, value)
}
}
body, marshalErr := (protojson.MarshalOptions{EmitDefaultValues: true}).Marshal(st.Proto())
if marshalErr != nil {
http.Error(w, `{"code": 13, "message": "failed to encode error"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(runtime.HTTPStatusFromCode(st.Code()))
_, _ = w.Write(body)
}
func writeGatewayAuthorizationError(w http.ResponseWriter, err error) {
if errors.Is(err, ErrUnauthenticated) {
writeGatewayStatusError(w, status.Error(codes.Unauthenticated, "authentication required"))
return
}
slog.Error("failed to resolve API access policy", "error", err)
writeGatewayStatusError(w, status.Error(codes.Internal, "failed to resolve API access policy"))
}