mirror of
https://github.com/usememos/memos.git
synced 2026-11-19 10:29:11 +01:00
313 lines
9.6 KiB
Go
313 lines
9.6 KiB
Go
package configuration
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
"google.golang.org/protobuf/encoding/protojson"
|
|
"google.golang.org/protobuf/reflect/protoreflect"
|
|
"google.golang.org/protobuf/types/descriptorpb"
|
|
|
|
apipb "github.com/usememos/memos/proto/gen/api"
|
|
)
|
|
|
|
var (
|
|
referencePattern = regexp.MustCompile(`^\$\{(env|file):([^}]*)\}$`)
|
|
envNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
|
|
|
// protojson errors can quote the rejected value, which may be a
|
|
// substituted secret. Only these value-free forms are passed on.
|
|
unknownFieldPattern = regexp.MustCompile(`unknown field "[^"]+"`)
|
|
duplicateFieldPattern = regexp.MustCompile(`duplicate field "[^"]+"`)
|
|
invalidValuePattern = regexp.MustCompile(`invalid value for [a-z0-9]+ field [A-Za-z0-9_]+`)
|
|
oneofPattern = regexp.MustCompile(`oneof [A-Za-z0-9_.]+ is already set`)
|
|
)
|
|
|
|
// decodeDocument substitutes references in one file and decodes it strictly.
|
|
// It also returns, for each user by index, the optional user fields the file
|
|
// declares.
|
|
func decodeDocument(content []byte, resolver Resolver) (*apipb.Configuration, map[int][]string, []string, []string) {
|
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
|
decoder.UseNumber()
|
|
r := &jsonReader{decoder: decoder, resolver: resolver}
|
|
value, err := r.value("")
|
|
if err != nil {
|
|
return nil, nil, nil, []string{describeJSONError(err, content)}
|
|
}
|
|
if _, err := decoder.Token(); err != io.EOF {
|
|
return nil, nil, nil, []string{"unexpected data after the JSON document"}
|
|
}
|
|
object, ok := value.(map[string]any)
|
|
if !ok {
|
|
return nil, nil, nil, []string{"the document must be a JSON object"}
|
|
}
|
|
|
|
// Marshalling a substituted tree cannot fail: it holds only JSON values.
|
|
normalized, _ := json.Marshal(value)
|
|
document := &apipb.Configuration{}
|
|
if err := (protojson.UnmarshalOptions{}).Unmarshal(normalized, document); err != nil {
|
|
return nil, nil, nil, []string{describeProtoJSONError(err)}
|
|
}
|
|
|
|
inspector := &inspector{}
|
|
inspector.message("", value, document.ProtoReflect().Descriptor())
|
|
return document, declaredUserFields(object), inspector.warnings, inspector.problems
|
|
}
|
|
|
|
// optionalUserFields are the User fields managed only when a file declares them.
|
|
var optionalUserFields = []string{"displayName", "email"}
|
|
|
|
// declaredUserFields returns, for each user in a decoded document by index,
|
|
// which optional fields the document names, in either JSON or proto spelling.
|
|
func declaredUserFields(document map[string]any) map[int][]string {
|
|
users, ok := document["users"].([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
descriptor := (&apipb.User{}).ProtoReflect().Descriptor()
|
|
result := map[int][]string{}
|
|
for i, entry := range users {
|
|
object, ok := entry.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
for _, jsonName := range optionalUserFields {
|
|
field := descriptor.Fields().ByJSONName(jsonName)
|
|
if _, declared := object[jsonName]; declared {
|
|
result[i] = append(result[i], jsonName)
|
|
} else if _, declared := object[string(field.Name())]; declared {
|
|
result[i] = append(result[i], jsonName)
|
|
}
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// jsonReader reads a JSON value token by token, substituting references in
|
|
// string values and reporting paths in the document's own key spelling.
|
|
type jsonReader struct {
|
|
decoder *json.Decoder
|
|
resolver Resolver
|
|
}
|
|
|
|
func (r *jsonReader) value(path string) (any, error) {
|
|
token, err := r.decoder.Token()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
switch t := token.(type) {
|
|
case json.Delim:
|
|
switch t {
|
|
case '{':
|
|
object := map[string]any{}
|
|
for r.decoder.More() {
|
|
keyToken, err := r.decoder.Token()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Inside an object, encoding/json returns every key as a string.
|
|
key, ok := keyToken.(string)
|
|
if !ok {
|
|
return nil, errors.Errorf("%s: object key must be a string", path)
|
|
}
|
|
childPath := joinKey(path, key)
|
|
if _, ok := object[key]; ok {
|
|
return nil, errors.Errorf("%s: duplicate key", childPath)
|
|
}
|
|
child, err := r.value(childPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
object[key] = child
|
|
}
|
|
if _, err := r.decoder.Token(); err != nil {
|
|
return nil, err
|
|
}
|
|
return object, nil
|
|
case '[':
|
|
list := []any{}
|
|
for r.decoder.More() {
|
|
child, err := r.value(fmt.Sprintf("%s[%d]", path, len(list)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
list = append(list, child)
|
|
}
|
|
if _, err := r.decoder.Token(); err != nil {
|
|
return nil, err
|
|
}
|
|
return list, nil
|
|
default:
|
|
return nil, errors.Errorf("unexpected %q", t.String())
|
|
}
|
|
case string:
|
|
return r.substitute(path, t)
|
|
default:
|
|
return token, nil
|
|
}
|
|
}
|
|
|
|
// substitute resolves a string that is exactly one ${env:NAME} or
|
|
// ${file:PATH} reference, and turns each $${ into a literal ${. Errors name
|
|
// the path and the reference, never a resolved value.
|
|
func (r *jsonReader) substitute(path, value string) (string, error) {
|
|
if !hasUnescapedReference(value) {
|
|
return strings.ReplaceAll(value, "$${", "${"), nil
|
|
}
|
|
match := referencePattern.FindStringSubmatch(value)
|
|
if match == nil {
|
|
if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") && strings.Count(value, "${") == 1 {
|
|
return "", errors.Errorf("%s: unsupported reference; use ${env:NAME} or ${file:/absolute/path}", path)
|
|
}
|
|
return "", errors.Errorf("%s: a reference must be the whole string value; write $${ for a literal ${", path)
|
|
}
|
|
switch kind, target := match[1], match[2]; kind {
|
|
case "env":
|
|
if !envNamePattern.MatchString(target) {
|
|
return "", errors.Errorf("%s: invalid environment variable name %q", path, target)
|
|
}
|
|
resolved, ok := r.resolver.LookupEnv(target)
|
|
if !ok {
|
|
return "", errors.Errorf("%s: environment variable %s is not set", path, target)
|
|
}
|
|
return resolved, nil
|
|
default:
|
|
if !filepath.IsAbs(target) {
|
|
return "", errors.Errorf("%s: file reference %q must be an absolute path", path, target)
|
|
}
|
|
content, err := r.resolver.ReadFile(target)
|
|
if err != nil {
|
|
return "", errors.Errorf("%s: cannot read %s: %v", path, target, err)
|
|
}
|
|
if len(content) > maxReferencedFileSize {
|
|
return "", errors.Errorf("%s: %s is larger than %d bytes", path, target, maxReferencedFileSize)
|
|
}
|
|
return strings.TrimRight(string(content), "\r\n"), nil
|
|
}
|
|
}
|
|
|
|
func hasUnescapedReference(value string) bool {
|
|
for i := 0; i+1 < len(value); i++ {
|
|
if value[i] == '$' && value[i+1] == '{' && (i == 0 || value[i-1] != '$') {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func joinKey(path, key string) string {
|
|
if path == "" {
|
|
return key
|
|
}
|
|
return path + "." + key
|
|
}
|
|
|
|
func describeJSONError(err error, content []byte) string {
|
|
var syntaxError *json.SyntaxError
|
|
if errors.As(err, &syntaxError) {
|
|
line, column := position(content, syntaxError.Offset)
|
|
return fmt.Sprintf("line %d, column %d: %s", line, column, syntaxError.Error())
|
|
}
|
|
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
|
return "unexpected end of JSON"
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func position(content []byte, offset int64) (int, int) {
|
|
offset = min(offset, int64(len(content)))
|
|
before := content[:offset]
|
|
line := bytes.Count(before, []byte("\n")) + 1
|
|
column := len(before) - bytes.LastIndexByte(before, '\n') - 1
|
|
return line, column
|
|
}
|
|
|
|
func describeProtoJSONError(err error) string {
|
|
message := err.Error()
|
|
for _, pattern := range []*regexp.Regexp{unknownFieldPattern, duplicateFieldPattern, invalidValuePattern, oneofPattern} {
|
|
if found := pattern.FindString(message); found != "" {
|
|
return found
|
|
}
|
|
}
|
|
return "invalid protobuf JSON; check field names and value types"
|
|
}
|
|
|
|
// inspector walks a decoded document beside its message descriptor and
|
|
// reports what protojson accepts but configuration does not: numeric enum
|
|
// values and output-only fields. Deprecated fields are warnings.
|
|
type inspector struct {
|
|
problems []string
|
|
warnings []string
|
|
}
|
|
|
|
func (in *inspector) message(path string, value any, descriptor protoreflect.MessageDescriptor) {
|
|
object, ok := value.(map[string]any)
|
|
if !ok || strings.HasPrefix(string(descriptor.FullName()), "google.protobuf.") {
|
|
return
|
|
}
|
|
keys := make([]string, 0, len(object))
|
|
for key := range object {
|
|
keys = append(keys, key)
|
|
}
|
|
slices.Sort(keys)
|
|
for _, key := range keys {
|
|
field := descriptor.Fields().ByJSONName(key)
|
|
if field == nil {
|
|
field = descriptor.Fields().ByTextName(key)
|
|
}
|
|
if field == nil || object[key] == nil {
|
|
continue
|
|
}
|
|
in.field(joinKey(path, key), object[key], field)
|
|
}
|
|
}
|
|
|
|
func (in *inspector) field(path string, value any, field protoreflect.FieldDescriptor) {
|
|
if options, ok := field.Options().(*descriptorpb.FieldOptions); ok && options.GetDeprecated() {
|
|
in.warnings = append(in.warnings, path+" is deprecated")
|
|
}
|
|
if isOutputOnly(field) {
|
|
in.problems = append(in.problems, path+" is output only and cannot be configured")
|
|
return
|
|
}
|
|
|
|
element := func(elementPath string, elementValue any, kind protoreflect.Kind, message protoreflect.MessageDescriptor) {
|
|
switch kind {
|
|
case protoreflect.EnumKind:
|
|
if _, isNumber := elementValue.(json.Number); isNumber {
|
|
in.problems = append(in.problems, elementPath+": use the enum value name, not its number")
|
|
}
|
|
case protoreflect.MessageKind, protoreflect.GroupKind:
|
|
in.message(elementPath, elementValue, message)
|
|
default:
|
|
}
|
|
}
|
|
switch {
|
|
case field.IsMap():
|
|
object, ok := value.(map[string]any)
|
|
if !ok {
|
|
return
|
|
}
|
|
for key, entry := range object {
|
|
element(path+"."+key, entry, field.MapValue().Kind(), field.MapValue().Message())
|
|
}
|
|
case field.IsList():
|
|
list, ok := value.([]any)
|
|
if !ok {
|
|
return
|
|
}
|
|
for i, entry := range list {
|
|
element(fmt.Sprintf("%s[%d]", path, i), entry, field.Kind(), field.Message())
|
|
}
|
|
default:
|
|
element(path, value, field.Kind(), field.Message())
|
|
}
|
|
}
|