diff --git a/server/mcp/catalog.go b/server/mcp/catalog.go index 25c77373c..91a805ef0 100644 --- a/server/mcp/catalog.go +++ b/server/mcp/catalog.go @@ -102,9 +102,10 @@ var requestBodySchemaOverrides = map[string]requestBodySchemaOverride{ required: []string{"relations"}, omittedProperties: []string{"name"}, }, + // The memo is the parent of its reactions; the path binds it. "MemoService_UpsertMemoReaction": { required: []string{"reaction"}, - omittedProperties: []string{"name"}, + omittedProperties: []string{"parent"}, }, // UpdateSpace requires an update mask, which the REST gateway infers from // the fields present in the body. The space name comes from the path. @@ -165,6 +166,16 @@ func validateOperationOverrides(registry map[string]*openAPIOperation) error { } } } + // An omitted property must exist in the body, or the override is stale + // after a binding rename and silently leaves a path-bound field exposed. + for operationID, override := range requestBodySchemaOverrides { + properties := schemaProperties(registry[operationID].RequestBodySchema["properties"]) + for _, name := range override.omittedProperties { + if _, ok := properties[name]; !ok { + return errors.Errorf("requestBodySchemaOverrides omits property %q missing from the %s request body", name, operationID) + } + } + } return nil } diff --git a/server/mcp/catalog_test.go b/server/mcp/catalog_test.go index 4638c74ca..cc4b40224 100644 --- a/server/mcp/catalog_test.go +++ b/server/mcp/catalog_test.go @@ -2,6 +2,7 @@ package mcp import ( "encoding/json" + "maps" "strconv" "strings" "testing" @@ -155,7 +156,7 @@ func TestBuildToolFromOperationTailorsRequestBodySchemas(t *testing.T) { omittedProperties: []string{"name"}, }, { - name: "upsert reaction gets name from path", + name: "upsert reaction gets parent from path", operationID: "MemoService_UpsertMemoReaction", arguments: map[string]any{ "memo": "memos/abc123", @@ -165,7 +166,7 @@ func TestBuildToolFromOperationTailorsRequestBodySchemas(t *testing.T) { }, }, }, - omittedProperties: []string{"name"}, + omittedProperties: []string{"parent"}, }, } @@ -376,6 +377,29 @@ func TestValidateOperationOverridesRejectsStaleKey(t *testing.T) { require.ErrorContains(t, validateOperationOverrides(registry), "MemoService_UpdateMemo") } +func TestValidateOperationOverridesRejectsStaleOmittedProperty(t *testing.T) { + spec, err := loadOpenAPISpec("../../proto/gen/openapi.yaml") + require.NoError(t, err) + registry, err := buildOperationRegistry(spec) + require.NoError(t, err) + + // A binding rename (name -> parent) must be reported instead of leaving the + // path-bound field exposed in the body schema. + operation := registry["MemoService_UpsertMemoReaction"] + renamed := cloneSchema(operation.RequestBodySchema) + properties := maps.Clone(schemaProperties(renamed["properties"])) + properties["name"] = properties["parent"] + delete(properties, "parent") + renamed["properties"] = properties + registry["MemoService_UpsertMemoReaction"] = &openAPIOperation{ + OperationID: operation.OperationID, + RequestBodySchema: renamed, + } + err = validateOperationOverrides(registry) + require.ErrorContains(t, err, `omits property "parent"`) + require.ErrorContains(t, err, "MemoService_UpsertMemoReaction") +} + func TestBuildCuratedToolsRejectsDuplicateToolNames(t *testing.T) { registry := make(map[string]*openAPIOperation, len(curatedOperationIDs)) for _, operationID := range curatedOperationIDs { diff --git a/server/mcp/evals/README.md b/server/mcp/evals/README.md index bbc857672..52344c931 100644 --- a/server/mcp/evals/README.md +++ b/server/mcp/evals/README.md @@ -42,22 +42,59 @@ update both this summary and `memos_eval.xml`; nothing in CI catches drift. ## Running an evaluation -1. Launch a throwaway demo-mode instance (SQLite, auto-seeded) on a free port: +The seed users have random, discarded passwords and the seed ships no access +token, so the instance needs a way to sign in as `steven`. Questions 10 and 11 +depend on the caller being `steven`. Declare that user in a configuration +directory: the seed runs after the configuration is applied and attaches its +content to the declared `steven` instead of creating a second one. + +1. Write a configuration that declares `steven` as an administrator with a + password you choose: + + ```bash + mkdir -p /tmp/memos-evals/config + cat > /tmp/memos-evals/config/instance.json <<'EOF' + { + "users": [ + { "name": "users/steven", "username": "steven", "role": "ADMIN", "password": "${env:MEMOS_EVAL_PASSWORD}" } + ] + } + EOF + export MEMOS_EVAL_PASSWORD=evals-only + ``` + +2. Launch a throwaway demo-mode instance (SQLite, auto-seeded) on a free port: ```bash go run -buildvcs=true ./cmd/memos --demo --driver sqlite \ --port 8099 --data "$(mktemp -d)" \ + --config-dir /tmp/memos-evals/config \ --instance-url http://localhost:8099 ``` -2. The MCP endpoint is `http://localhost:8099/mcp`. Authenticate with the seed's - demo personal access token, which belongs to `steven`: +3. Sign in as `steven` and create a personal access token for the harness. The + sign-in access token is short-lived; the personal access token is not. - ``` - Authorization: Bearer memos_pat_demo + ```bash + ACCESS_TOKEN=$(curl -s -X POST http://localhost:8099/api/auth/signin \ + -H 'Content-Type: application/json' \ + -d "{\"passwordCredentials\":{\"username\":\"steven\",\"password\":\"$MEMOS_EVAL_PASSWORD\"}}" \ + | jq -r .accessToken) + MEMOS_TOKEN=$(curl -s -X POST http://localhost:8099/api/users/steven/personalAccessTokens \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"description":"mcp evals"}' \ + | jq -r .token) ``` -3. Point an MCP client / eval harness at that endpoint and have the model answer +4. The MCP endpoint is `http://localhost:8099/mcp`. Authenticate every request + with that token: + + ``` + Authorization: Bearer $MEMOS_TOKEN + ``` + +5. Point an MCP client / eval harness at that endpoint and have the model answer each ``, then string-compare against each ``. Quick manual check of a single tool call: @@ -66,7 +103,7 @@ update both this summary and `memos_eval.xml`; nothing in CI catches drift. curl -s -X POST http://localhost:8099/mcp \ -H 'Content-Type: application/json' \ -H 'Accept: application/json, text/event-stream' \ - -H 'Authorization: Bearer memos_pat_demo' \ + -H "Authorization: Bearer $MEMOS_TOKEN" \ -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"memo_list_memos","arguments":{"filter":"pinned == true"}}}' ```