refactor: name packages by role and enforce import direction (#6317)

This commit is contained in:
ABird
2026-09-13 18:22:54 +08:00
committed by GitHub
parent c506f67d22
commit 45519accd3
286 changed files with 569 additions and 2595 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ jobs:
;;
other)
go test -v -race -coverprofile=coverage.out -covermode=atomic \
./cmd/... ./internal/... ./proto/...
./cmd/... ./core/... ./markdown/... ./filter/... ./provider/... ./proto/...
;;
esac
env:
+2 -2
View File
@@ -45,7 +45,7 @@ jobs:
uses: actions/upload-artifact@v6
with:
name: frontend-dist
path: server/router/frontend/dist
path: server/frontend/dist
retention-days: 1
build-push:
@@ -75,7 +75,7 @@ jobs:
uses: actions/download-artifact@v7
with:
name: frontend-dist
path: server/router/frontend/dist
path: server/frontend/dist
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
+3 -3
View File
@@ -109,7 +109,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: frontend-dist
path: server/router/frontend/dist
path: server/frontend/dist
retention-days: 1
build-binaries:
@@ -147,7 +147,7 @@ jobs:
uses: actions/download-artifact@v4
with:
name: frontend-dist
path: server/router/frontend/dist
path: server/frontend/dist
- name: Build binary
env:
@@ -276,7 +276,7 @@ jobs:
uses: actions/download-artifact@v4
with:
name: frontend-dist
path: server/router/frontend/dist
path: server/frontend/dist
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
+1 -1
View File
@@ -6,7 +6,7 @@
/memos.exe
coverage.out
# Frontend bundles: web/dist from `pnpm build`, and
# server/router/frontend/dist from `pnpm release`, which go:embed bakes
# server/frontend/dist from `pnpm release`, which go:embed bakes
# into the binary.
dist/
+51
View File
@@ -2,6 +2,7 @@ version: "2"
linters:
enable:
- depguard
- revive
- govet
- staticcheck
@@ -17,6 +18,56 @@ linters:
disable:
- errcheck
settings:
# Import direction. Each rule names a layer by file path and lists the
# layers it must not import. See internal/README.md, core/README.md,
# provider/README.md, and the doc.go of server, store, markdown, filter.
depguard:
rules:
internal-has-no-app-imports:
files:
- "**/internal/**"
- "!**/internal/testutil/**"
deny:
- pkg: github.com/usememos/memos/store
desc: internal packages must not know the store; move the code to core or a root package
- pkg: github.com/usememos/memos/core
desc: internal packages must not import core
- pkg: github.com/usememos/memos/server
desc: internal packages must not import server
- pkg: github.com/usememos/memos/provider
desc: internal packages must not import provider
- pkg: github.com/usememos/memos/markdown
desc: internal packages must not import markdown
- pkg: github.com/usememos/memos/filter
desc: internal packages must not import filter
- pkg: github.com/usememos/memos/proto
desc: internal packages must not import generated protos; only internal/testutil may
engines-and-providers-sit-below-store:
files:
- "**/markdown/**"
- "**/filter/**"
- "**/provider/**"
deny:
- pkg: github.com/usememos/memos/store
desc: markdown, filter, and provider must not import store
- pkg: github.com/usememos/memos/core
desc: markdown, filter, and provider must not import core
- pkg: github.com/usememos/memos/server
desc: markdown, filter, and provider must not import server
store-sits-below-core:
files:
- "**/store/**"
deny:
- pkg: github.com/usememos/memos/core
desc: store must not import core; business rules that span store calls live in core
- pkg: github.com/usememos/memos/server
desc: store must not import server
core-sits-below-server:
files:
- "**/core/**"
deny:
- pkg: github.com/usememos/memos/server
desc: core must not import server; transports call core, not the reverse
exhaustive:
explicit-exhaustive-switch: false
staticcheck:
+41 -28
View File
@@ -18,7 +18,7 @@ Memos is a self-hosted note-taking app.
- Keep diffs scoped. Do not do repo-wide cleanup, dependency churn, or generated-file rewrites unless the task requires it.
- Do not hand-edit generated proto outputs. Change `.proto` files, then run `buf generate`.
- Add migrations for all database drivers when schema changes, and update each driver's `LATEST.sql`.
- Add public API endpoints to `server/router/api/v1/acl_config.go`.
- Add public API endpoints to `server/api/v1/acl_config.go`.
- Ask before adding heavy dependencies, changing auth/token behavior, or altering Docker/release workflows.
## Commands
@@ -32,9 +32,9 @@ go test ./... # Run all Go tests
go test -v ./store/... # Store tests, including DB drivers via TestContainers
go test -v -race ./server/... # Server tests with race detector
go test -v -race ./internal/... # Internal package tests with race detector
go test -v -run TestFoo ./pkg/... # Run matching Go tests
go test -v -run TestFoo ./core/... # Run matching Go tests in one tree
go mod tidy -go=1.27.0 # Match CI tidy check
golangci-lint run # Go lint, config: .golangci.yaml
golangci-lint run # Go lint, config: .golangci.yaml (includes depguard layering rules)
golangci-lint run --fix # Auto-fix lint, including goimports
# Frontend
@@ -43,7 +43,7 @@ cd web && pnpm dev # Dev server on :3001, proxying API to :8081
cd web && pnpm lint # Type check + Biome lint
cd web && pnpm test # Vitest unit tests
cd web && pnpm build # Production build
cd web && pnpm release # Build SPA into server/router/frontend/dist
cd web && pnpm release # Build SPA into server/frontend/dist
# Protocol Buffers
cd proto && buf generate # Regenerate Go + TypeScript + OpenAPI
@@ -51,40 +51,52 @@ cd proto && buf lint # Lint proto files
cd proto && buf format -w # Format proto files
```
## Code Map
## Repository Layout
| Path | Purpose |
| --- | --- |
| `cmd/memos/main.go` | Cobra/Viper CLI setup and server startup |
| `server/server.go` | Echo HTTP server, router wiring, and graceful shutdown |
| `server/auth/` | JWT access tokens, refresh tokens, PAT handling |
| `server/router/api/v1/` | Connect/gRPC-Gateway services, ACL config, SSE hub |
| `server/router/frontend/` | Static SPA serving |
| `server/router/fileserver/` | Native HTTP file serving, thumbnails, range requests |
| `server/runner/` | Memo payload rebuilding |
| `store/` | Store facade, cache, migrations, driver interface |
| `store/db/{sqlite,mysql,postgres}/` | Database-specific drivers and SQL |
| `proto/api/v1/` | Public API service definitions |
| `proto/store/` | Internal storage proto messages |
| `internal/` | App-private packages: scheduler, cron, email, CEL filter, markdown, idp, S3 |
| `web/src/connect.ts` | Connect RPC clients, auth interceptor, access-token refresh |
| `web/src/auth-state.ts` | Token storage and BroadcastChannel cross-tab sync |
| `web/src/hooks/` | React Query hooks for server state |
| `web/src/contexts/` | React context for client/UI state |
| `web/src/components/` | Radix/Tailwind UI components and feature components |
| `web/src/themes/` | CSS themes using OKLch color tokens |
Every root folder is named for what it holds. Read the folder's `doc.go` or `README.md` before adding to it.
| Path | Holds | May import |
| --- | --- | --- |
| `cmd/memos/` | Cobra/Viper CLI setup and server startup | anything |
| `server/` | The HTTP process: Echo bootstrap, graceful shutdown, every transport | core, store, provider, markdown, filter, internal |
| `server/api/v1/` | Connect/gRPC-Gateway services, ACL config, SSE hub. Thin handlers; rules go to `core/` | |
| `server/fileserver/` | Native HTTP file serving, thumbnails, range requests | |
| `server/frontend/` | Static SPA serving; `dist/` is the built SPA baked in by `go:embed` | |
| `server/mcp/` | Model Context Protocol server | |
| `server/auth/` | JWT access tokens, refresh tokens, PAT handling | |
| `core/` | Business rules with no HTTP and no SQL: `access`, `notification`, `memopayload` | store, provider, markdown, filter, internal |
| `store/` | Store facade, cache, migrations, `Driver` interface; `store/db/{sqlite,mysql,postgres}/` implement it | provider, markdown, filter, internal |
| `markdown/` | Markdown engine: parser, AST, memos syntax extensions, renderer, memo payload | proto/gen, internal |
| `filter/` | CEL filter compiler: parse to IR, render to SQL per driver, filterable field schema | internal |
| `provider/` | Backends configured by instance settings: `ai`, `idp`, `storage` | proto/gen, internal |
| `internal/` | Private plumbing with no memos vocabulary (`identifier`, `email`, `webhook`, `ratelimit`, …) | third party only |
| `proto/api/v1/`, `proto/store/` | Public API and internal storage proto sources; `proto/gen/` is generated | |
| `web/src/` | React SPA: `connect.ts` clients, `auth-state.ts`, `hooks/`, `contexts/`, `components/`, `themes/` | |
Layering rules, enforced by `depguard` in `.golangci.yaml`:
- Imports point downward only: `cmd` → `server` → `core` → `store` → {`provider`, `markdown`, `filter`} → `internal`.
- `internal/` packages import nothing from this module (except `internal/testutil`, which may import `proto/gen`).
- A package goes in `internal/` only if it could be published as a standalone module unchanged. If it knows what a Memo, Space, or instance setting is, it is a root package.
- Never create `util`, `common`, `base`, `helpers`, `misc`, or `pkg` packages. Name a package for the one thing it does, or put the code next to its only caller.
- New business rules extracted from `server/api/v1` go to `core/<resource>/`, one package per resource.
File convention inside `server/api/v1/`: `<resource>_service.go` holds the RPC handlers, `<resource>_service_converters.go` the
store↔proto conversion, and `<resource>_service_<topic>.go` further splits. Extend the matching file; do not add `*_helpers2.go`.
Black-box service tests live in `server/api/v1/test/`, unit tests next to the code.
## Change Routing
| Change | Update | Verify |
| --- | --- | --- |
| Go service or router behavior | Service code under `server/`, tests near package | `go test -v -race ./server/...` |
| Go service or router behavior | Handlers under `server/`, rules under `core/`, tests near package | `go test -v -race ./server/... ./core/...` |
| Store or migration behavior | `store/`, all three DB driver migrations, `LATEST.sql` | `go test -v ./store/...` |
| Markdown, filter, or provider logic | `markdown/`, `filter/`, `provider/` | `go test -v -race ./markdown/... ./filter/... ./provider/...` |
| Internal package logic | Relevant `internal/` package tests | `go test -v -race ./internal/...` |
| Frontend behavior | Components/hooks/contexts under `web/src/` | `cd web && pnpm lint && pnpm test` |
| Frontend production output | Vite config or release-sensitive UI | `cd web && pnpm build` or `pnpm release` |
| Proto API | `.proto` source plus generated outputs | `cd proto && buf generate && buf lint` |
| Public unauthenticated route | `server/router/api/v1/acl_config.go` | Targeted server test or manual route check |
| Public unauthenticated route | `server/api/v1/acl_config.go` | Targeted server test or manual route check |
## Go Conventions
@@ -119,7 +131,8 @@ cd proto && buf format -w # Format proto files
## CI Reference
- Backend CI: Go 1.27.0, `go mod tidy -go=1.27.0`, golangci-lint v2.13.1, test groups `store`, `server`, `internal`, `other`.
- Backend CI: Go 1.27.0, `go mod tidy -go=1.27.0`, golangci-lint v2.13.1, test groups `store`, `server`, `internal`, `other`
(`cmd`, `core`, `markdown`, `filter`, `provider`, `proto`).
- Frontend CI: Node 24, pnpm 11.0.1, `pnpm lint`, `pnpm test`, `pnpm build`.
- Proto CI: `buf lint` and `buf format` check.
- Docker: `scripts/Dockerfile`, Alpine 3.21 runtime, non-root user, port 5230, multi-arch amd64/arm64/arm/v7.
+16
View File
@@ -0,0 +1,16 @@
# core
Business rules that do not care about HTTP or SQL. A package lands here when it
needs several store calls, or a decision, that more than one transport must
apply the same way.
| Package | Owns |
| --- | --- |
| `access/` | who may read a memo, given viewer, space membership, and share links |
| `notification/` | building and dispatching inbox and email notifications |
| `memopayload/` | rebuilding a memo's derived payload from its markdown |
Layering: `core` may import `store`, `provider`, `markdown`, `filter`,
`proto/gen`, and `internal`. It must not import `server` or `cmd`. New
resource-specific rules extracted from `server/api/v1` belong here, one package
per resource.
@@ -6,7 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/usememos/memos/internal/markdown"
"github.com/usememos/memos/markdown"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
+9
View File
@@ -0,0 +1,9 @@
// Package filter compiles the CEL filter expressions accepted by the list
// endpoints into an intermediate representation and renders that IR into SQL
// for each database driver. The schema of filterable memo and user fields
// lives here.
//
// Layering: filter imports only third-party packages and internal. It must not
// import store, core, or server; the SQL drivers import filter, not the other
// way round.
package filter
+23
View File
@@ -0,0 +1,23 @@
# internal
Private plumbing with no memos vocabulary. A package belongs here only if it
could be published tomorrow as a standalone module without changes: it imports
nothing from `store`, `core`, `server`, `provider`, `markdown`, `filter`, or
`proto/gen`. The `depguard` rules in `.golangci.yaml` enforce that.
| Package | Purpose |
| --- | --- |
| `clientip/` | trusted-proxy aware client IP resolution middleware |
| `email/` | SMTP client, config, and message types |
| `identifier/` | grammar of UIDs and usernames shared by parser, store, and API |
| `linkmeta/` | HTML metadata, oEmbed, and image fetching for link previews |
| `motionphoto/` | Live Photo and Motion Photo container parsing |
| `profile/` | process configuration parsed from flags and environment |
| `random/` | UUIDs and random strings from a secure source |
| `ratelimit/` | sliding-window rate limiter |
| `testutil/` | test fixtures and the fake S3 server (the one package allowed to import `proto/gen`) |
| `version/` | build version and version comparison |
| `webhook/` | signed webhook delivery with SSRF protection |
Do not add `util`, `common`, `base`, or `helpers` packages. Name a package for
the one thing it does, or put the code next to its only caller.
-1
View File
@@ -1 +0,0 @@
package httpgetter
@@ -1,4 +1,4 @@
package base
package identifier
import "regexp"
@@ -1,4 +1,4 @@
package base
package identifier
import (
"testing"
@@ -1,4 +1,4 @@
package base
package identifier
// MaxUsernameLength is the maximum number of ASCII characters in a writable username.
const MaxUsernameLength = 36
@@ -1,4 +1,4 @@
package base
package identifier
import (
"strings"
@@ -1,4 +1,4 @@
package httpgetter
package linkmeta
import (
"context"
@@ -1,4 +1,4 @@
package httpgetter
package linkmeta
import (
"bytes"
@@ -1,4 +1,4 @@
package httpgetter
package linkmeta
import (
"context"
+1
View File
@@ -0,0 +1 @@
package linkmeta
@@ -1,4 +1,4 @@
package httpgetter
package linkmeta
import (
"errors"
@@ -1,4 +1,4 @@
package httpgetter
package linkmeta
import (
"mime"
+33
View File
@@ -0,0 +1,33 @@
// Package random generates identifiers and secrets from a cryptographically
// secure source.
package random
import (
"crypto/rand"
"math/big"
"strings"
"uuid"
)
// UUID returns a new random (version 4) UUID in its canonical string form.
func UUID() string {
return uuid.NewV4().String()
}
var alphabet = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
// String returns a random alphanumeric string of length n.
func String(n int) (string, error) {
var sb strings.Builder
sb.Grow(n)
for i := 0; i < n; i++ {
index, err := rand.Int(rand.Reader, big.NewInt(int64(len(alphabet))))
if err != nil {
return "", err
}
if _, err := sb.WriteRune(alphabet[index.Uint64()]); err != nil {
return "", err
}
}
return sb.String(), nil
}
-367
View File
@@ -1,367 +0,0 @@
# Scheduler Plugin
A production-ready, GitHub Actions-inspired cron job scheduler for Go.
## Features
- **Standard Cron Syntax**: Supports both 5-field and 6-field (with seconds) cron expressions
- **Timezone-Aware**: Explicit timezone handling to avoid DST surprises
- **Middleware Pattern**: Composable job wrappers for logging, metrics, panic recovery, timeouts
- **Graceful Shutdown**: Jobs complete cleanly or cancel when context expires
- **Zero Dependencies**: Core functionality uses only the standard library
- **Type-Safe**: Strong typing with clear error messages
- **Well-Tested**: Comprehensive test coverage
## Installation
This package is included with Memos. No separate installation required.
## Quick Start
```go
package main
import (
"context"
"fmt"
"github.com/usememos/memos/internal/scheduler"
)
func main() {
s := scheduler.New()
s.Register(&scheduler.Job{
Name: "daily-cleanup",
Schedule: "0 2 * * *", // 2 AM daily
Handler: func(ctx context.Context) error {
fmt.Println("Running cleanup...")
return nil
},
})
s.Start()
defer s.Stop(context.Background())
// Keep running...
select {}
}
```
## Cron Expression Format
### 5-Field Format (Standard)
```
┌───────────── minute (0 - 59)
│ ┌───────────── hour (0 - 23)
│ │ ┌───────────── day of month (1 - 31)
│ │ │ ┌───────────── month (1 - 12)
│ │ │ │ ┌───────────── day of week (0 - 7) (Sunday = 0 or 7)
│ │ │ │ │
* * * * *
```
### 6-Field Format (With Seconds)
```
┌───────────── second (0 - 59)
│ ┌───────────── minute (0 - 59)
│ │ ┌───────────── hour (0 - 23)
│ │ │ ┌───────────── day of month (1 - 31)
│ │ │ │ ┌───────────── month (1 - 12)
│ │ │ │ │ ┌───────────── day of week (0 - 7)
│ │ │ │ │ │
* * * * * *
```
### Special Characters
- `*` - Any value (every minute, every hour, etc.)
- `,` - List of values: `1,15,30` (1st, 15th, and 30th)
- `-` - Range: `9-17` (9 AM through 5 PM)
- `/` - Step: `*/15` (every 15 units)
### Common Examples
| Schedule | Description |
|----------|-------------|
| `* * * * *` | Every minute |
| `0 * * * *` | Every hour |
| `0 0 * * *` | Daily at midnight |
| `0 9 * * 1-5` | Weekdays at 9 AM |
| `*/15 * * * *` | Every 15 minutes |
| `0 0 1 * *` | First day of every month |
| `0 0 * * 0` | Every Sunday at midnight |
| `30 14 * * *` | Every day at 2:30 PM |
## Timezone Support
```go
// Global timezone for all jobs
s := scheduler.New(
scheduler.WithTimezone("America/New_York"),
)
// Per-job timezone (overrides global)
s.Register(&scheduler.Job{
Name: "tokyo-report",
Schedule: "0 9 * * *", // 9 AM Tokyo time
Timezone: "Asia/Tokyo",
Handler: func(ctx context.Context) error {
// Runs at 9 AM in Tokyo
return nil
},
})
```
**Important**: Always use IANA timezone names (`America/New_York`, not `EST`).
## Middleware
Middleware wraps job handlers to add cross-cutting behavior. Multiple middleware can be chained together.
### Built-in Middleware
#### Recovery (Panic Handling)
```go
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Recovery(func(jobName string, r interface{}) {
log.Printf("Job %s panicked: %v", jobName, r)
}),
),
)
```
#### Logging
```go
type Logger interface {
Info(msg string, args ...interface{})
Error(msg string, args ...interface{})
}
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Logging(myLogger),
),
)
```
#### Timeout
```go
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Timeout(5 * time.Minute),
),
)
```
### Combining Middleware
```go
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Recovery(panicHandler),
scheduler.Logging(logger),
scheduler.Timeout(10 * time.Minute),
),
)
```
**Order matters**: Middleware are applied left-to-right. In the example above:
1. Recovery (outermost) catches panics from everything
2. Logging logs the execution
3. Timeout (innermost) wraps the actual handler
### Custom Middleware
```go
func Metrics(recorder MetricsRecorder) scheduler.Middleware {
return func(next scheduler.JobHandler) scheduler.JobHandler {
return func(ctx context.Context) error {
start := time.Now()
err := next(ctx)
duration := time.Since(start)
jobName := scheduler.GetJobName(ctx)
recorder.Record(jobName, duration, err)
return err
}
}
}
```
## Graceful Shutdown
Always use `Stop()` with a context to allow jobs to finish cleanly:
```go
// Give jobs up to 30 seconds to complete
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
log.Printf("Shutdown error: %v", err)
}
```
Jobs should respect context cancellation:
```go
Handler: func(ctx context.Context) error {
for i := 0; i < 100; i++ {
select {
case <-ctx.Done():
return ctx.Err() // Canceled
default:
// Do work
}
}
return nil
}
```
## Best Practices
### 1. Always Name Your Jobs
Names are used for logging, metrics, and debugging:
```go
Name: "user-cleanup-job" // Good
Name: "job1" // Bad
```
### 2. Add Descriptions and Tags
```go
s.Register(&scheduler.Job{
Name: "stale-session-cleanup",
Description: "Removes user sessions older than 30 days",
Tags: []string{"maintenance", "security"},
Schedule: "0 3 * * *",
Handler: cleanupSessions,
})
```
### 3. Use Appropriate Middleware
Always include Recovery and Logging in production:
```go
scheduler.New(
scheduler.WithMiddleware(
scheduler.Recovery(logPanic),
scheduler.Logging(logger),
),
)
```
### 4. Avoid Scheduling Exactly on the Hour
Many systems schedule jobs at `:00`, causing load spikes. Stagger your jobs:
```go
"5 2 * * *" // 2:05 AM (good)
"0 2 * * *" // 2:00 AM (often overloaded)
```
### 5. Make Jobs Idempotent
Jobs may run multiple times (crash recovery, etc.). Design them to be safely re-runnable:
```go
Handler: func(ctx context.Context) error {
// Use unique constraint or check-before-insert
db.Exec("INSERT IGNORE INTO processed_items ...")
return nil
}
```
### 6. Handle Timezones Explicitly
Always specify timezone for business-hour jobs:
```go
Timezone: "America/New_York" // Good
// Timezone: "" // Bad (defaults to UTC)
```
### 7. Test Your Cron Expressions
Use a cron expression calculator before deploying:
- [crontab.guru](https://crontab.guru/)
- Write unit tests with the parser
## Testing Jobs
Test job handlers independently of the scheduler:
```go
func TestCleanupJob(t *testing.T) {
ctx := context.Background()
err := cleanupHandler(ctx)
if err != nil {
t.Fatalf("cleanup failed: %v", err)
}
// Verify cleanup occurred
}
```
Test schedule parsing:
```go
func TestScheduleParsing(t *testing.T) {
job := &scheduler.Job{
Name: "test",
Schedule: "0 2 * * *",
Handler: func(ctx context.Context) error { return nil },
}
if err := job.Validate(); err != nil {
t.Fatalf("invalid job: %v", err)
}
}
```
## Comparison to Other Solutions
| Feature | scheduler | robfig/cron | github.com/go-co-op/gocron |
|---------|-----------|-------------|----------------------------|
| Standard cron syntax | ✅ | ✅ | ✅ |
| Seconds support | ✅ | ✅ | ✅ |
| Timezone support | ✅ | ✅ | ✅ |
| Middleware pattern | ✅ | ⚠️ (basic) | ❌ |
| Graceful shutdown | ✅ | ⚠️ (basic) | ✅ |
| Zero dependencies | ✅ | ❌ | ❌ |
| Job metadata | ✅ | ❌ | ⚠️ (limited) |
## API Reference
See [example_test.go](./example_test.go) for comprehensive examples.
### Core Types
- `Scheduler` - Manages scheduled jobs
- `Job` - Job definition with schedule and handler
- `Middleware` - Function that wraps job handlers
### Functions
- `New(opts ...Option) *Scheduler` - Create new scheduler
- `WithTimezone(tz string) Option` - Set default timezone
- `WithMiddleware(mw ...Middleware) Option` - Add middleware
### Methods
- `Register(job *Job) error` - Add job to scheduler
- `Start() error` - Begin executing jobs
- `Stop(ctx context.Context) error` - Graceful shutdown
## License
This package is part of the Memos project and shares its license.
-35
View File
@@ -1,35 +0,0 @@
// Package scheduler provides a GitHub Actions-inspired cron job scheduler.
//
// Features:
// - Standard cron expression syntax (5-field and 6-field formats)
// - Timezone-aware scheduling
// - Middleware pattern for cross-cutting concerns (logging, metrics, recovery)
// - Graceful shutdown with context cancellation
// - Zero external dependencies
//
// Basic usage:
//
// s := scheduler.New()
//
// s.Register(&scheduler.Job{
// Name: "daily-cleanup",
// Schedule: "0 2 * * *", // 2 AM daily
// Handler: func(ctx context.Context) error {
// // Your cleanup logic here
// return nil
// },
// })
//
// s.Start()
// defer s.Stop(context.Background())
//
// With middleware:
//
// s := scheduler.New(
// scheduler.WithTimezone("America/New_York"),
// scheduler.WithMiddleware(
// scheduler.Recovery(),
// scheduler.Logging(),
// ),
// )
package scheduler
-165
View File
@@ -1,165 +0,0 @@
package scheduler_test
import (
"context"
"fmt"
"log/slog"
"os"
"time"
"github.com/usememos/memos/internal/scheduler"
)
// Example demonstrates basic scheduler usage.
func Example_basic() {
s := scheduler.New()
s.Register(&scheduler.Job{
Name: "hello",
Schedule: "*/5 * * * *", // Every 5 minutes
Description: "Say hello",
Handler: func(_ context.Context) error {
fmt.Println("Hello from scheduler!")
return nil
},
})
s.Start()
defer s.Stop(context.Background())
// Scheduler runs in background
time.Sleep(100 * time.Millisecond)
}
// Example demonstrates timezone-aware scheduling.
func Example_timezone() {
s := scheduler.New(
scheduler.WithTimezone("America/New_York"),
)
s.Register(&scheduler.Job{
Name: "daily-report",
Schedule: "0 9 * * *", // 9 AM in New York
Handler: func(_ context.Context) error {
fmt.Println("Generating daily report...")
return nil
},
})
s.Start()
defer s.Stop(context.Background())
}
// Example demonstrates middleware usage.
func Example_middleware() {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Recovery(func(jobName string, r interface{}) {
logger.Error("Job panicked", "job", jobName, "panic", r)
}),
scheduler.Logging(&slogAdapter{logger}),
scheduler.Timeout(5*time.Minute),
),
)
s.Register(&scheduler.Job{
Name: "data-sync",
Schedule: "0 */2 * * *", // Every 2 hours
Handler: func(_ context.Context) error {
// Your sync logic here
return nil
},
})
s.Start()
defer s.Stop(context.Background())
}
// slogAdapter adapts slog.Logger to scheduler.Logger interface.
type slogAdapter struct {
logger *slog.Logger
}
func (a *slogAdapter) Info(msg string, args ...interface{}) {
a.logger.Info(msg, args...)
}
func (a *slogAdapter) Error(msg string, args ...interface{}) {
a.logger.Error(msg, args...)
}
// Example demonstrates multiple jobs with different schedules.
func Example_multipleJobs() {
s := scheduler.New()
// Cleanup old data every night at 2 AM
s.Register(&scheduler.Job{
Name: "cleanup",
Schedule: "0 2 * * *",
Tags: []string{"maintenance"},
Handler: func(_ context.Context) error {
fmt.Println("Cleaning up old data...")
return nil
},
})
// Health check every 5 minutes
s.Register(&scheduler.Job{
Name: "health-check",
Schedule: "*/5 * * * *",
Tags: []string{"monitoring"},
Handler: func(_ context.Context) error {
fmt.Println("Running health check...")
return nil
},
})
// Weekly backup on Sundays at 1 AM
s.Register(&scheduler.Job{
Name: "weekly-backup",
Schedule: "0 1 * * 0",
Tags: []string{"backup"},
Handler: func(_ context.Context) error {
fmt.Println("Creating weekly backup...")
return nil
},
})
s.Start()
defer s.Stop(context.Background())
}
// Example demonstrates graceful shutdown with timeout.
func Example_gracefulShutdown() {
s := scheduler.New()
s.Register(&scheduler.Job{
Name: "long-running",
Schedule: "* * * * *",
Handler: func(ctx context.Context) error {
select {
case <-time.After(30 * time.Second):
fmt.Println("Job completed")
case <-ctx.Done():
fmt.Println("Job canceled, cleaning up...")
return ctx.Err()
}
return nil
},
})
s.Start()
// Simulate shutdown signal
time.Sleep(5 * time.Second)
// Give jobs 10 seconds to finish
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := s.Stop(shutdownCtx); err != nil {
fmt.Printf("Shutdown error: %v\n", err)
}
}
-393
View File
@@ -1,393 +0,0 @@
package scheduler_test
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/usememos/memos/internal/scheduler"
)
// TestRealWorldScenario tests a realistic multi-job scenario.
func TestRealWorldScenario(t *testing.T) {
var (
quickJobCount atomic.Int32
hourlyJobCount atomic.Int32
logEntries []string
logMu sync.Mutex
)
logger := &testLogger{
onInfo: func(msg string, _ ...interface{}) {
logMu.Lock()
logEntries = append(logEntries, fmt.Sprintf("INFO: %s", msg))
logMu.Unlock()
},
onError: func(msg string, _ ...interface{}) {
logMu.Lock()
logEntries = append(logEntries, fmt.Sprintf("ERROR: %s", msg))
logMu.Unlock()
},
}
s := scheduler.New(
scheduler.WithTimezone("UTC"),
scheduler.WithMiddleware(
scheduler.Recovery(func(jobName string, r interface{}) {
t.Logf("Job %s panicked: %v", jobName, r)
}),
scheduler.Logging(logger),
scheduler.Timeout(5*time.Second),
),
)
// Quick job (every second)
s.Register(&scheduler.Job{
Name: "quick-check",
Schedule: "* * * * * *",
Handler: func(_ context.Context) error {
quickJobCount.Add(1)
time.Sleep(100 * time.Millisecond)
return nil
},
})
// Slower job (every 2 seconds)
s.Register(&scheduler.Job{
Name: "slow-process",
Schedule: "*/2 * * * * *",
Handler: func(_ context.Context) error {
hourlyJobCount.Add(1)
time.Sleep(500 * time.Millisecond)
return nil
},
})
// Start scheduler
if err := s.Start(); err != nil {
t.Fatalf("failed to start scheduler: %v", err)
}
// Let it run for 5 seconds
time.Sleep(5 * time.Second)
// Graceful shutdown
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop scheduler: %v", err)
}
// Verify execution counts
quick := quickJobCount.Load()
slow := hourlyJobCount.Load()
t.Logf("Quick job ran %d times", quick)
t.Logf("Slow job ran %d times", slow)
if quick < 4 {
t.Errorf("expected quick job to run at least 4 times, ran %d", quick)
}
if slow < 2 {
t.Errorf("expected slow job to run at least 2 times, ran %d", slow)
}
// Verify logging
logMu.Lock()
defer logMu.Unlock()
hasStartLog := false
hasCompleteLog := false
for _, entry := range logEntries {
if contains(entry, "Job started") {
hasStartLog = true
}
if contains(entry, "Job completed") {
hasCompleteLog = true
}
}
if !hasStartLog {
t.Error("expected job start logs")
}
if !hasCompleteLog {
t.Error("expected job completion logs")
}
}
// TestCancellationDuringExecution verifies jobs can be canceled mid-execution.
func TestCancellationDuringExecution(t *testing.T) {
var canceled atomic.Bool
var started atomic.Bool
s := scheduler.New()
s.Register(&scheduler.Job{
Name: "long-job",
Schedule: "* * * * * *",
Handler: func(ctx context.Context) error {
started.Store(true)
// Simulate long-running work
for i := 0; i < 100; i++ {
select {
case <-ctx.Done():
canceled.Store(true)
return ctx.Err()
case <-time.After(100 * time.Millisecond):
// Keep working
}
}
return nil
},
})
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Wait until job starts
for i := 0; i < 30; i++ {
if started.Load() {
break
}
time.Sleep(100 * time.Millisecond)
}
if !started.Load() {
t.Fatal("job did not start within timeout")
}
// Stop with reasonable timeout
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Logf("stop returned error (may be expected): %v", err)
}
if !canceled.Load() {
t.Error("expected job to detect cancellation")
}
}
// TestTimezoneHandling verifies timezone-aware scheduling.
func TestTimezoneHandling(t *testing.T) {
// Parse a schedule in a specific timezone
schedule, err := scheduler.ParseCronExpression("0 9 * * *") // 9 AM
if err != nil {
t.Fatalf("failed to parse schedule: %v", err)
}
// Test in New York timezone
nyc, err := time.LoadLocation("America/New_York")
if err != nil {
t.Fatalf("failed to load timezone: %v", err)
}
// Current time: 8:30 AM in New York
now := time.Date(2025, 1, 15, 8, 30, 0, 0, nyc)
// Next run should be 9:00 AM same day
next := schedule.Next(now)
expected := time.Date(2025, 1, 15, 9, 0, 0, 0, nyc)
if !next.Equal(expected) {
t.Errorf("next = %v, expected %v", next, expected)
}
// If it's already past 9 AM
now = time.Date(2025, 1, 15, 9, 30, 0, 0, nyc)
next = schedule.Next(now)
expected = time.Date(2025, 1, 16, 9, 0, 0, 0, nyc)
if !next.Equal(expected) {
t.Errorf("next = %v, expected %v", next, expected)
}
}
// TestErrorPropagation verifies error handling.
func TestErrorPropagation(t *testing.T) {
var errorLogged atomic.Bool
logger := &testLogger{
onError: func(msg string, _ ...interface{}) {
if msg == "Job failed" {
errorLogged.Store(true)
}
},
}
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Logging(logger),
),
)
s.Register(&scheduler.Job{
Name: "failing-job",
Schedule: "* * * * * *",
Handler: func(_ context.Context) error {
return errors.New("intentional error")
},
})
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Let it run once
time.Sleep(1500 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop: %v", err)
}
if !errorLogged.Load() {
t.Error("expected error to be logged")
}
}
// TestPanicRecovery verifies panic recovery middleware.
func TestPanicRecovery(t *testing.T) {
var panicRecovered atomic.Bool
s := scheduler.New(
scheduler.WithMiddleware(
scheduler.Recovery(func(jobName string, r interface{}) {
panicRecovered.Store(true)
t.Logf("Recovered from panic in job %s: %v", jobName, r)
}),
),
)
s.Register(&scheduler.Job{
Name: "panicking-job",
Schedule: "* * * * * *",
Handler: func(_ context.Context) error {
panic("intentional panic for testing")
},
})
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Let it run once
time.Sleep(1500 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop: %v", err)
}
if !panicRecovered.Load() {
t.Error("expected panic to be recovered")
}
}
// TestMultipleJobsWithDifferentSchedules verifies concurrent job execution.
func TestMultipleJobsWithDifferentSchedules(t *testing.T) {
var (
job1Count atomic.Int32
job2Count atomic.Int32
job3Count atomic.Int32
)
s := scheduler.New()
// Job 1: Every second
s.Register(&scheduler.Job{
Name: "job-1sec",
Schedule: "* * * * * *",
Handler: func(_ context.Context) error {
job1Count.Add(1)
return nil
},
})
// Job 2: Every 2 seconds
s.Register(&scheduler.Job{
Name: "job-2sec",
Schedule: "*/2 * * * * *",
Handler: func(_ context.Context) error {
job2Count.Add(1)
return nil
},
})
// Job 3: Every 3 seconds
s.Register(&scheduler.Job{
Name: "job-3sec",
Schedule: "*/3 * * * * *",
Handler: func(_ context.Context) error {
job3Count.Add(1)
return nil
},
})
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Let them run for 6 seconds
time.Sleep(6 * time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop: %v", err)
}
// Verify counts (allowing for timing variance)
c1 := job1Count.Load()
c2 := job2Count.Load()
c3 := job3Count.Load()
t.Logf("Job 1 ran %d times, Job 2 ran %d times, Job 3 ran %d times", c1, c2, c3)
if c1 < 5 {
t.Errorf("expected job1 to run at least 5 times, ran %d", c1)
}
if c2 < 2 {
t.Errorf("expected job2 to run at least 2 times, ran %d", c2)
}
if c3 < 1 {
t.Errorf("expected job3 to run at least 1 time, ran %d", c3)
}
}
// Helpers
type testLogger struct {
onInfo func(msg string, args ...interface{})
onError func(msg string, args ...interface{})
}
func (l *testLogger) Info(msg string, args ...interface{}) {
if l.onInfo != nil {
l.onInfo(msg, args...)
}
}
func (l *testLogger) Error(msg string, args ...interface{}) {
if l.onError != nil {
l.onError(msg, args...)
}
}
func contains(s, substr string) bool {
return strings.Contains(s, substr)
}
-58
View File
@@ -1,58 +0,0 @@
package scheduler
import (
"context"
"github.com/pkg/errors"
)
// JobHandler is the function signature for scheduled job handlers.
// The context passed to the handler will be canceled if the scheduler is shutting down.
type JobHandler func(ctx context.Context) error
// Job represents a scheduled task.
type Job struct {
// Name is a unique identifier for this job (required).
// Used for logging and metrics.
Name string
// Schedule is a cron expression defining when this job runs (required).
// Supports standard 5-field format: "minute hour day month weekday"
// Examples: "0 * * * *" (hourly), "0 0 * * *" (daily at midnight)
Schedule string
// Timezone for schedule evaluation (optional, defaults to UTC).
// Use IANA timezone names: "America/New_York", "Europe/London", etc.
Timezone string
// Handler is the function to execute when the job triggers (required).
Handler JobHandler
// Description provides human-readable context about what this job does (optional).
Description string
// Tags allow categorizing jobs for filtering/monitoring (optional).
Tags []string
}
// Validate checks if the job definition is valid.
func (j *Job) Validate() error {
if j.Name == "" {
return errors.New("job name is required")
}
if j.Schedule == "" {
return errors.New("job schedule is required")
}
// Validate cron expression using parser
if _, err := ParseCronExpression(j.Schedule); err != nil {
return errors.Wrap(err, "invalid cron expression")
}
if j.Handler == nil {
return errors.New("job handler is required")
}
return nil
}
-90
View File
@@ -1,90 +0,0 @@
package scheduler
import (
"context"
"testing"
)
func TestJobDefinition(t *testing.T) {
callCount := 0
job := &Job{
Name: "test-job",
Handler: func(_ context.Context) error {
callCount++
return nil
},
}
if job.Name != "test-job" {
t.Errorf("expected name 'test-job', got %s", job.Name)
}
// Test handler execution
if err := job.Handler(context.Background()); err != nil {
t.Fatalf("handler failed: %v", err)
}
if callCount != 1 {
t.Errorf("expected handler to be called once, called %d times", callCount)
}
}
func TestJobValidation(t *testing.T) {
tests := []struct {
name string
job *Job
wantErr bool
}{
{
name: "valid job",
job: &Job{
Name: "valid",
Schedule: "0 * * * *",
Handler: func(_ context.Context) error { return nil },
},
wantErr: false,
},
{
name: "missing name",
job: &Job{
Schedule: "0 * * * *",
Handler: func(_ context.Context) error { return nil },
},
wantErr: true,
},
{
name: "missing schedule",
job: &Job{
Name: "test",
Handler: func(_ context.Context) error { return nil },
},
wantErr: true,
},
{
name: "invalid cron expression",
job: &Job{
Name: "test",
Schedule: "invalid cron",
Handler: func(_ context.Context) error { return nil },
},
wantErr: true,
},
{
name: "missing handler",
job: &Job{
Name: "test",
Schedule: "0 * * * *",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.job.Validate()
if (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
-121
View File
@@ -1,121 +0,0 @@
package scheduler
import (
"context"
"slices"
"time"
"github.com/pkg/errors"
)
// Middleware wraps a JobHandler to add cross-cutting behavior.
type Middleware func(JobHandler) JobHandler
// Chain combines multiple middleware into a single middleware.
// Middleware are applied in the order they're provided (left to right).
func Chain(middlewares ...Middleware) Middleware {
return func(handler JobHandler) JobHandler {
// Apply middleware in reverse order so first middleware wraps outermost
for _, middleware := range slices.Backward(middlewares) {
handler = middleware(handler)
}
return handler
}
}
// Recovery recovers from panics in job handlers and converts them to errors.
func Recovery(onPanic func(jobName string, recovered interface{})) Middleware {
return func(next JobHandler) JobHandler {
return func(ctx context.Context) (err error) {
defer func() {
if r := recover(); r != nil {
jobName := getJobName(ctx)
if onPanic != nil {
onPanic(jobName, r)
}
err = errors.Errorf("job %q panicked: %v", jobName, r)
}
}()
return next(ctx)
}
}
}
// Logger is a minimal logging interface.
type Logger interface {
Info(msg string, args ...interface{})
Error(msg string, args ...interface{})
}
// Logging adds execution logging to jobs.
func Logging(logger Logger) Middleware {
return func(next JobHandler) JobHandler {
return func(ctx context.Context) error {
jobName := getJobName(ctx)
start := time.Now()
logger.Info("Job started", "job", jobName)
err := next(ctx)
duration := time.Since(start)
if err != nil {
logger.Error("Job failed", "job", jobName, "duration", duration, "error", err)
} else {
logger.Info("Job completed", "job", jobName, "duration", duration)
}
return err
}
}
}
// Timeout wraps a job handler with a timeout.
func Timeout(duration time.Duration) Middleware {
return func(next JobHandler) JobHandler {
return func(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, duration)
defer cancel()
done := make(chan error, 1)
go func() {
done <- next(ctx)
}()
select {
case err := <-done:
return err
case <-ctx.Done():
return errors.Errorf("job %q timed out after %v", getJobName(ctx), duration)
}
}
}
}
// Context keys for job metadata.
type contextKey int
const (
jobNameKey contextKey = iota
)
// withJobName adds the job name to the context.
func withJobName(ctx context.Context, name string) context.Context {
return context.WithValue(ctx, jobNameKey, name)
}
// getJobName retrieves the job name from the context.
func getJobName(ctx context.Context) string {
if name, ok := ctx.Value(jobNameKey).(string); ok {
return name
}
return "unknown"
}
// GetJobName retrieves the job name from the context (public API).
// Returns empty string if not found.
//
//nolint:revive // GetJobName is the public API, getJobName is internal
func GetJobName(ctx context.Context) string {
return getJobName(ctx)
}
-146
View File
@@ -1,146 +0,0 @@
package scheduler
import (
"context"
"errors"
"sync/atomic"
"testing"
)
func TestMiddlewareChaining(t *testing.T) {
var order []string
mw1 := func(next JobHandler) JobHandler {
return func(ctx context.Context) error {
order = append(order, "before-1")
err := next(ctx)
order = append(order, "after-1")
return err
}
}
mw2 := func(next JobHandler) JobHandler {
return func(ctx context.Context) error {
order = append(order, "before-2")
err := next(ctx)
order = append(order, "after-2")
return err
}
}
handler := func(_ context.Context) error {
order = append(order, "handler")
return nil
}
chain := Chain(mw1, mw2)
wrapped := chain(handler)
if err := wrapped(context.Background()); err != nil {
t.Fatalf("wrapped handler failed: %v", err)
}
expected := []string{"before-1", "before-2", "handler", "after-2", "after-1"}
if len(order) != len(expected) {
t.Fatalf("expected %d calls, got %d", len(expected), len(order))
}
for i, want := range expected {
if order[i] != want {
t.Errorf("order[%d] = %q, want %q", i, order[i], want)
}
}
}
func TestRecoveryMiddleware(t *testing.T) {
var panicRecovered atomic.Bool
onPanic := func(_ string, _ interface{}) {
panicRecovered.Store(true)
}
handler := func(_ context.Context) error {
panic("simulated panic")
}
recovery := Recovery(onPanic)
wrapped := recovery(handler)
// Should not panic, error should be returned
err := wrapped(withJobName(context.Background(), "test-job"))
if err == nil {
t.Error("expected error from recovered panic")
}
if !panicRecovered.Load() {
t.Error("panic handler was not called")
}
}
func TestLoggingMiddleware(t *testing.T) {
var loggedStart, loggedEnd atomic.Bool
var loggedError atomic.Bool
logger := &testLogger{
onInfo: func(msg string, _ ...interface{}) {
if msg == "Job started" {
loggedStart.Store(true)
} else if msg == "Job completed" {
loggedEnd.Store(true)
}
},
onError: func(msg string, _ ...interface{}) {
if msg == "Job failed" {
loggedError.Store(true)
}
},
}
// Test successful execution
handler := func(_ context.Context) error {
return nil
}
logging := Logging(logger)
wrapped := logging(handler)
if err := wrapped(withJobName(context.Background(), "test-job")); err != nil {
t.Fatalf("handler failed: %v", err)
}
if !loggedStart.Load() {
t.Error("start was not logged")
}
if !loggedEnd.Load() {
t.Error("end was not logged")
}
// Test error handling
handlerErr := func(_ context.Context) error {
return errors.New("job error")
}
wrappedErr := logging(handlerErr)
_ = wrappedErr(withJobName(context.Background(), "test-job-error"))
if !loggedError.Load() {
t.Error("error was not logged")
}
}
type testLogger struct {
onInfo func(msg string, args ...interface{})
onError func(msg string, args ...interface{})
}
func (l *testLogger) Info(msg string, args ...interface{}) {
if l.onInfo != nil {
l.onInfo(msg, args...)
}
}
func (l *testLogger) Error(msg string, args ...interface{}) {
if l.onError != nil {
l.onError(msg, args...)
}
}
-240
View File
@@ -1,240 +0,0 @@
package scheduler
import (
"slices"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
)
// Schedule represents a parsed cron expression.
type Schedule struct {
seconds fieldMatcher // 0-59 (optional, for 6-field format)
minutes fieldMatcher // 0-59
hours fieldMatcher // 0-23
days fieldMatcher // 1-31
months fieldMatcher // 1-12
weekdays fieldMatcher // 0-7 (0 and 7 are Sunday)
hasSecs bool
dayWildcard bool
}
// fieldMatcher determines if a field value matches.
type fieldMatcher interface {
matches(value int) bool
}
// ParseCronExpression parses a cron expression and returns a Schedule.
// Supports both 5-field (minute hour day month weekday) and 6-field (second minute hour day month weekday) formats.
func ParseCronExpression(expr string) (*Schedule, error) {
if expr == "" {
return nil, errors.New("empty cron expression")
}
fields := strings.Fields(expr)
if len(fields) != 5 && len(fields) != 6 {
return nil, errors.Errorf("invalid cron expression: expected 5 or 6 fields, got %d", len(fields))
}
s := &Schedule{
hasSecs: len(fields) == 6,
}
var err error
offset := 0
// Parse seconds (if 6-field format)
if s.hasSecs {
s.seconds, err = parseField(fields[0], 0, 59)
if err != nil {
return nil, errors.Wrap(err, "invalid seconds field")
}
offset = 1
} else {
s.seconds = &exactMatcher{value: 0} // Default to 0 seconds
}
// Parse minutes
s.minutes, err = parseField(fields[offset], 0, 59)
if err != nil {
return nil, errors.Wrap(err, "invalid minutes field")
}
// Parse hours
s.hours, err = parseField(fields[offset+1], 0, 23)
if err != nil {
return nil, errors.Wrap(err, "invalid hours field")
}
// Parse days
s.days, err = parseField(fields[offset+2], 1, 31)
if err != nil {
return nil, errors.Wrap(err, "invalid days field")
}
// Parse months
s.months, err = parseField(fields[offset+3], 1, 12)
if err != nil {
return nil, errors.Wrap(err, "invalid months field")
}
// Parse weekdays (0-7, where both 0 and 7 represent Sunday)
s.weekdays, err = parseField(fields[offset+4], 0, 7)
if err != nil {
return nil, errors.Wrap(err, "invalid weekdays field")
}
s.dayWildcard = strings.HasPrefix(fields[offset+2], "*") || strings.HasPrefix(fields[offset+4], "*")
return s, nil
}
// Next returns the next time the schedule should run after the given time.
func (s *Schedule) Next(from time.Time) time.Time {
// Start from the next second/minute
if s.hasSecs {
from = from.Add(1 * time.Second).Truncate(time.Second)
} else {
from = from.Add(1 * time.Minute).Truncate(time.Minute)
}
// Cap search at 4 years to prevent infinite loops
maxTime := from.AddDate(4, 0, 0)
for from.Before(maxTime) {
if s.matches(from) {
return from
}
// Advance to next potential match
if s.hasSecs {
from = from.Add(1 * time.Second)
} else {
from = from.Add(1 * time.Minute)
}
}
// Should never reach here with valid cron expressions
return time.Time{}
}
// matches checks if the given time matches the schedule.
func (s *Schedule) matches(t time.Time) bool {
if !s.seconds.matches(t.Second()) || !s.minutes.matches(t.Minute()) ||
!s.hours.matches(t.Hour()) || !s.months.matches(int(t.Month())) {
return false
}
dayMatches := s.days.matches(t.Day())
weekdayMatches := s.weekdays.matches(int(t.Weekday()))
if t.Weekday() == time.Sunday {
weekdayMatches = weekdayMatches || s.weekdays.matches(7)
}
// Cron combines restricted day fields with OR, but wildcard fields (including steps) use AND.
calendarMatches := dayMatches || weekdayMatches
if s.dayWildcard {
calendarMatches = dayMatches && weekdayMatches
}
return calendarMatches
}
// parseField parses a single cron field (supports *, ranges, lists, steps).
func parseField(field string, min, max int) (fieldMatcher, error) {
// Wildcard
if field == "*" {
return &wildcardMatcher{}, nil
}
// Step values (*/N)
if strings.HasPrefix(field, "*/") {
step, err := strconv.Atoi(field[2:])
if err != nil || step < 1 || step > max {
return nil, errors.Errorf("invalid step value: %s", field)
}
return &stepMatcher{step: step, min: min, max: max}, nil
}
// List (1,2,3)
if strings.Contains(field, ",") {
parts := strings.Split(field, ",")
values := make([]int, 0, len(parts))
for _, p := range parts {
val, err := strconv.Atoi(strings.TrimSpace(p))
if err != nil || val < min || val > max {
return nil, errors.Errorf("invalid list value: %s", p)
}
values = append(values, val)
}
return &listMatcher{values: values}, nil
}
// Range (1-5)
if strings.Contains(field, "-") {
parts := strings.Split(field, "-")
if len(parts) != 2 {
return nil, errors.Errorf("invalid range: %s", field)
}
start, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
end, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
if err1 != nil || err2 != nil || start < min || end > max || start > end {
return nil, errors.Errorf("invalid range: %s", field)
}
return &rangeMatcher{start: start, end: end}, nil
}
// Exact value
val, err := strconv.Atoi(field)
if err != nil || val < min || val > max {
return nil, errors.Errorf("invalid value: %s (must be between %d and %d)", field, min, max)
}
return &exactMatcher{value: val}, nil
}
// wildcardMatcher matches any value.
type wildcardMatcher struct{}
func (*wildcardMatcher) matches(_ int) bool {
return true
}
// exactMatcher matches a specific value.
type exactMatcher struct {
value int
}
func (m *exactMatcher) matches(value int) bool {
return value == m.value
}
// rangeMatcher matches values in a range.
type rangeMatcher struct {
start, end int
}
func (m *rangeMatcher) matches(value int) bool {
return value >= m.start && value <= m.end
}
// listMatcher matches any value in a list.
type listMatcher struct {
values []int
}
func (m *listMatcher) matches(value int) bool {
return slices.Contains(m.values, value)
}
// stepMatcher matches values at regular intervals.
type stepMatcher struct {
step, min, max int
}
func (m *stepMatcher) matches(value int) bool {
if value < m.min || value > m.max {
return false
}
return (value-m.min)%m.step == 0
}
@@ -1,58 +0,0 @@
package scheduler
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestScheduleNextCalendarDays(t *testing.T) {
tests := []struct {
name string
expr string
from string
want string
}{
{"weekly", "0 0 * * 0", "2025-01-01T10:00:00Z", "2025-01-05T00:00:00Z"},
{"monthly", "0 0 1 * *", "2025-01-02T00:00:00Z", "2025-02-01T00:00:00Z"},
{"weekdays skip weekend", "0 9 * * 1-5", "2025-01-03T09:00:00Z", "2025-01-06T09:00:00Z"},
{"day step with weekday", "0 0 */2 * 3", "2025-01-01T00:00:00Z", "2025-01-15T00:00:00Z"},
{"weekday step with day", "0 0 13 * */2", "2025-01-01T00:00:00Z", "2025-02-13T00:00:00Z"},
{"both day steps", "0 0 */2 * */2", "2025-01-01T00:00:00Z", "2025-01-05T00:00:00Z"},
{"restricted fields match day", "0 0 15 * 1", "2025-01-13T00:00:00Z", "2025-01-15T00:00:00Z"},
{"restricted fields match weekday", "0 0 15 * 1", "2025-01-15T00:00:00Z", "2025-01-20T00:00:00Z"},
{"explicit full range uses OR", "0 0 1-31 * 1", "2025-01-01T00:00:00Z", "2025-01-02T00:00:00Z"},
{"Sunday zero with restricted day", "0 0 31 * 0", "2025-01-04T00:00:00Z", "2025-01-05T00:00:00Z"},
{"Sunday seven with restricted day", "0 0 31 * 7", "2025-01-04T00:00:00Z", "2025-01-05T00:00:00Z"},
{"Sunday seven with wildcard day", "0 0 * * 7", "2025-01-06T00:00:00Z", "2025-01-12T00:00:00Z"},
{"Sunday seven in list", "0 0 31 * 1,7", "2025-01-04T00:00:00Z", "2025-01-05T00:00:00Z"},
{"Sunday seven in range", "0 0 31 * 5-7", "2025-01-04T00:00:00Z", "2025-01-05T00:00:00Z"},
{"Sunday aliases do not repeat", "0 0 * * 0,7", "2025-01-05T00:00:00Z", "2025-01-12T00:00:00Z"},
}
for _, format := range []struct {
name string
prefix string
}{
{"five fields", ""},
{"six fields", "0 "},
} {
t.Run(format.name, func(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
from, err := time.Parse(time.RFC3339, tt.from)
require.NoError(t, err)
want, err := time.Parse(time.RFC3339, tt.want)
require.NoError(t, err)
schedule, err := ParseCronExpression(format.prefix + tt.expr)
require.NoError(t, err)
next := schedule.Next(from)
require.Equal(t, want, next)
})
}
})
}
}
-127
View File
@@ -1,127 +0,0 @@
package scheduler
import (
"testing"
"time"
)
func TestParseCronExpression(t *testing.T) {
tests := []struct {
name string
expr string
wantErr bool
}{
// Standard 5-field format
{"every minute", "* * * * *", false},
{"hourly", "0 * * * *", false},
{"daily midnight", "0 0 * * *", false},
{"weekly sunday", "0 0 * * 0", false},
{"monthly", "0 0 1 * *", false},
{"specific time", "30 14 * * *", false}, // 2:30 PM daily
{"range", "0 9-17 * * *", false}, // Every hour 9 AM - 5 PM
{"step", "*/15 * * * *", false}, // Every 15 minutes
{"list", "0 8,12,18 * * *", false}, // 8 AM, 12 PM, 6 PM
// 6-field format with seconds
{"with seconds", "0 * * * * *", false},
{"every 30 seconds", "*/30 * * * * *", false},
// Invalid expressions
{"empty", "", true},
{"too few fields", "* * *", true},
{"too many fields", "* * * * * * *", true},
{"invalid minute", "60 * * * *", true},
{"invalid hour", "0 24 * * *", true},
{"invalid day", "0 0 32 * *", true},
{"invalid month", "0 0 1 13 *", true},
{"invalid weekday", "0 0 * * 8", true},
{"garbage", "not a cron expression", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
schedule, err := ParseCronExpression(tt.expr)
if (err != nil) != tt.wantErr {
t.Errorf("ParseCronExpression(%q) error = %v, wantErr %v", tt.expr, err, tt.wantErr)
return
}
if !tt.wantErr && schedule == nil {
t.Errorf("ParseCronExpression(%q) returned nil schedule without error", tt.expr)
}
})
}
}
func TestScheduleNext(t *testing.T) {
tests := []struct {
name string
expr string
from time.Time
expected time.Time
}{
{
name: "every minute from start of hour",
expr: "* * * * *",
from: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC),
expected: time.Date(2025, 1, 1, 10, 1, 0, 0, time.UTC),
},
{
name: "hourly at minute 30",
expr: "30 * * * *",
from: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC),
expected: time.Date(2025, 1, 1, 10, 30, 0, 0, time.UTC),
},
{
name: "hourly at minute 30 (already past)",
expr: "30 * * * *",
from: time.Date(2025, 1, 1, 10, 45, 0, 0, time.UTC),
expected: time.Date(2025, 1, 1, 11, 30, 0, 0, time.UTC),
},
{
name: "daily at 2 AM",
expr: "0 2 * * *",
from: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC),
expected: time.Date(2025, 1, 2, 2, 0, 0, 0, time.UTC),
},
{
name: "every 15 minutes",
expr: "*/15 * * * *",
from: time.Date(2025, 1, 1, 10, 7, 0, 0, time.UTC),
expected: time.Date(2025, 1, 1, 10, 15, 0, 0, time.UTC),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
schedule, err := ParseCronExpression(tt.expr)
if err != nil {
t.Fatalf("failed to parse expression: %v", err)
}
next := schedule.Next(tt.from)
if !next.Equal(tt.expected) {
t.Errorf("Next(%v) = %v, expected %v", tt.from, next, tt.expected)
}
})
}
}
func TestScheduleNextWithTimezone(t *testing.T) {
nyc, _ := time.LoadLocation("America/New_York")
// Schedule for 9 AM in New York
schedule, err := ParseCronExpression("0 9 * * *")
if err != nil {
t.Fatalf("failed to parse expression: %v", err)
}
// Current time: 8 AM in New York
from := time.Date(2025, 1, 1, 8, 0, 0, 0, nyc)
next := schedule.Next(from)
// Should be 9 AM same day in New York
expected := time.Date(2025, 1, 1, 9, 0, 0, 0, nyc)
if !next.Equal(expected) {
t.Errorf("Next(%v) = %v, expected %v", from, next, expected)
}
}
-202
View File
@@ -1,202 +0,0 @@
package scheduler
import (
"context"
"sync"
"time"
"github.com/pkg/errors"
)
// Scheduler manages scheduled jobs.
type Scheduler struct {
jobs map[string]*registeredJob
jobsMu sync.RWMutex
timezone *time.Location
middleware Middleware
running bool
runningMu sync.RWMutex
stopCh chan struct{}
wg sync.WaitGroup
}
// registeredJob wraps a Job with runtime state.
type registeredJob struct {
job *Job
cancelFn context.CancelFunc
}
// Option configures a Scheduler.
type Option func(*Scheduler)
// WithTimezone sets the default timezone for all jobs.
func WithTimezone(tz string) Option {
return func(s *Scheduler) {
loc, err := time.LoadLocation(tz)
if err != nil {
// Default to UTC on invalid timezone
loc = time.UTC
}
s.timezone = loc
}
}
// WithMiddleware sets middleware to wrap all job handlers.
func WithMiddleware(mw ...Middleware) Option {
return func(s *Scheduler) {
if len(mw) > 0 {
s.middleware = Chain(mw...)
}
}
}
// New creates a new Scheduler with optional configuration.
func New(opts ...Option) *Scheduler {
s := &Scheduler{
jobs: make(map[string]*registeredJob),
timezone: time.UTC,
stopCh: make(chan struct{}),
}
for _, opt := range opts {
opt(s)
}
return s
}
// Register adds a job to the scheduler.
// Jobs must be registered before calling Start().
func (s *Scheduler) Register(job *Job) error {
if job == nil {
return errors.New("job cannot be nil")
}
if err := job.Validate(); err != nil {
return errors.Wrap(err, "invalid job")
}
s.jobsMu.Lock()
defer s.jobsMu.Unlock()
if _, exists := s.jobs[job.Name]; exists {
return errors.Errorf("job with name %q already registered", job.Name)
}
s.jobs[job.Name] = &registeredJob{job: job}
return nil
}
// Start begins executing scheduled jobs.
func (s *Scheduler) Start() error {
s.runningMu.Lock()
defer s.runningMu.Unlock()
if s.running {
return errors.New("scheduler already running")
}
s.jobsMu.RLock()
defer s.jobsMu.RUnlock()
// Parse and schedule all jobs
for _, rj := range s.jobs {
schedule, err := ParseCronExpression(rj.job.Schedule)
if err != nil {
return errors.Wrapf(err, "failed to parse schedule for job %q", rj.job.Name)
}
ctx, cancel := context.WithCancel(context.Background())
rj.cancelFn = cancel
s.wg.Add(1)
go s.runJobWithSchedule(ctx, rj, schedule)
}
s.running = true
return nil
}
// runJobWithSchedule executes a job according to its cron schedule.
func (s *Scheduler) runJobWithSchedule(ctx context.Context, rj *registeredJob, schedule *Schedule) {
defer s.wg.Done()
// Apply middleware to handler
handler := rj.job.Handler
if s.middleware != nil {
handler = s.middleware(handler)
}
for {
// Calculate next run time
now := time.Now()
if rj.job.Timezone != "" {
loc, err := time.LoadLocation(rj.job.Timezone)
if err == nil {
now = now.In(loc)
}
} else if s.timezone != nil {
now = now.In(s.timezone)
}
next := schedule.Next(now)
duration := time.Until(next)
timer := time.NewTimer(duration)
select {
case <-timer.C:
// Add job name to context and execute
jobCtx := withJobName(ctx, rj.job.Name)
if err := handler(jobCtx); err != nil {
// Error already handled by middleware (if any)
_ = err
}
case <-ctx.Done():
// Stop the timer to prevent it from firing. The timer will be garbage collected.
timer.Stop()
return
case <-s.stopCh:
// Stop the timer to prevent it from firing. The timer will be garbage collected.
timer.Stop()
return
}
}
}
// Stop gracefully shuts down the scheduler.
// It waits for all running jobs to complete or until the context is canceled.
func (s *Scheduler) Stop(ctx context.Context) error {
s.runningMu.Lock()
if !s.running {
s.runningMu.Unlock()
return errors.New("scheduler not running")
}
s.running = false
s.runningMu.Unlock()
// Cancel all job contexts
s.jobsMu.RLock()
for _, rj := range s.jobs {
if rj.cancelFn != nil {
rj.cancelFn()
}
}
s.jobsMu.RUnlock()
// Signal stop and wait for jobs to finish
close(s.stopCh)
done := make(chan struct{})
go func() {
s.wg.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
-165
View File
@@ -1,165 +0,0 @@
package scheduler
import (
"context"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestSchedulerCreation(t *testing.T) {
s := New()
if s == nil {
t.Fatal("New() returned nil")
}
}
func TestSchedulerWithTimezone(t *testing.T) {
s := New(WithTimezone("America/New_York"))
if s == nil {
t.Fatal("New() with timezone returned nil")
}
}
func TestJobRegistration(t *testing.T) {
s := New()
job := &Job{
Name: "test-registration",
Schedule: "0 * * * *",
Handler: func(_ context.Context) error { return nil },
}
if err := s.Register(job); err != nil {
t.Fatalf("failed to register valid job: %v", err)
}
// Registering duplicate name should fail
if err := s.Register(job); err == nil {
t.Error("expected error when registering duplicate job name")
}
}
func TestSchedulerStartStop(t *testing.T) {
s := New()
var runCount atomic.Int32
job := &Job{
Name: "test-start-stop",
Schedule: "* * * * * *", // Every second (6-field format)
Handler: func(_ context.Context) error {
runCount.Add(1)
return nil
},
}
if err := s.Register(job); err != nil {
t.Fatalf("failed to register job: %v", err)
}
// Start scheduler
if err := s.Start(); err != nil {
t.Fatalf("failed to start scheduler: %v", err)
}
// Let it run for 2.5 seconds
time.Sleep(2500 * time.Millisecond)
// Stop scheduler
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop scheduler: %v", err)
}
count := runCount.Load()
// Should have run at least twice (at 0s and 1s, maybe 2s)
if count < 2 {
t.Errorf("expected job to run at least 2 times, ran %d times", count)
}
// Verify it stopped (count shouldn't increase)
finalCount := count
time.Sleep(1500 * time.Millisecond)
if runCount.Load() != finalCount {
t.Error("scheduler did not stop - job continued running")
}
}
func TestSchedulerWithMiddleware(t *testing.T) {
var executionLog []string
var logMu sync.Mutex
logger := &testLogger{
onInfo: func(msg string, _ ...interface{}) {
logMu.Lock()
executionLog = append(executionLog, fmt.Sprintf("INFO: %s", msg))
logMu.Unlock()
},
onError: func(msg string, _ ...interface{}) {
logMu.Lock()
executionLog = append(executionLog, fmt.Sprintf("ERROR: %s", msg))
logMu.Unlock()
},
}
s := New(WithMiddleware(
Recovery(func(jobName string, r interface{}) {
logMu.Lock()
executionLog = append(executionLog, fmt.Sprintf("PANIC: %s - %v", jobName, r))
logMu.Unlock()
}),
Logging(logger),
))
job := &Job{
Name: "test-middleware",
Schedule: "* * * * * *", // Every second
Handler: func(_ context.Context) error {
return nil
},
}
if err := s.Register(job); err != nil {
t.Fatalf("failed to register job: %v", err)
}
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
time.Sleep(1500 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.Stop(ctx); err != nil {
t.Fatalf("failed to stop: %v", err)
}
logMu.Lock()
defer logMu.Unlock()
// Should have at least one start and one completion log
hasStart := false
hasCompletion := false
for _, log := range executionLog {
if strings.Contains(log, "Job started") {
hasStart = true
}
if strings.Contains(log, "Job completed") {
hasCompletion = true
}
}
if !hasStart {
t.Error("expected job start log")
}
if !hasCompletion {
t.Error("expected job completion log")
}
}
-98
View File
@@ -1,98 +0,0 @@
package util //nolint:revive // util namespace is intentional for shared helpers
import (
"crypto/rand"
"math/big"
"net/mail"
"strconv"
"strings"
"uuid"
"github.com/pkg/errors"
)
// ConvertStringToInt32 converts a string to int32.
func ConvertStringToInt32(src string) (int32, error) {
parsed, err := strconv.ParseInt(src, 10, 32)
if err != nil {
return 0, err
}
return int32(parsed), nil
}
// HasPrefixes returns true if the string s has any of the given prefixes.
func HasPrefixes(src string, prefixes ...string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(src, prefix) {
return true
}
}
return false
}
// maxEmailLength is the longest address accepted for storage. RFC 5321 caps a
// forward path at 256 octets including the angle brackets, so 254 is the
// widely used practical limit.
const maxEmailLength = 254
// NormalizeEmail validates an email address and returns its canonical stored
// form: surrounding whitespace removed and every character lowercased. An
// empty input (after trimming) is valid and returns "", meaning no address.
//
// Display-name forms such as "Alice <alice@example.com>" are rejected even
// though the mail parser accepts them, because the stored value must be the
// bare address. The local part is lowercased along with the domain: every
// mainstream provider treats it case-insensitively, and folding both halves
// is what makes the instance-wide uniqueness rule match what users expect.
func NormalizeEmail(email string) (string, error) {
email = strings.TrimSpace(email)
if email == "" {
return "", nil
}
if len(email) > maxEmailLength {
return "", errors.Errorf("email must be at most %d characters", maxEmailLength)
}
address, err := mail.ParseAddress(email)
if err != nil {
return "", errors.New("email is not a valid address")
}
if address.Address != email {
return "", errors.New("email must be a bare address without a display name")
}
return strings.ToLower(email), nil
}
func GenUUID() string {
return uuid.NewV4().String()
}
var letters = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
// RandomString returns a random string with length n.
func RandomString(n int) (string, error) {
var sb strings.Builder
sb.Grow(n)
for i := 0; i < n; i++ {
// The reason for using crypto/rand instead of math/rand is that
// the former relies on hardware to generate random numbers and
// thus has a stronger source of random numbers.
randNum, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
if err != nil {
return "", err
}
if _, err := sb.WriteRune(letters[randNum.Uint64()]); err != nil {
return "", err
}
}
return sb.String(), nil
}
// ReplaceString replaces all occurrences of old in slice with new.
func ReplaceString(slice []string, old, new string) []string {
for i, s := range slice {
if s == old {
slice[i] = new
}
}
return slice
}
+24 -26
View File
@@ -19,8 +19,6 @@ import (
"uuid"
"github.com/pkg/errors"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
)
var (
@@ -37,17 +35,17 @@ var (
},
}
asyncPostQueue = make(chan *WebhookRequestPayload, 128)
asyncPostQueue = make(chan *Request, 128)
)
func init() {
for range 4 {
go func() {
for payload := range asyncPostQueue {
if err := Post(payload); err != nil {
for request := range asyncPostQueue {
if err := Post(request); err != nil {
slog.Warn("Failed to dispatch webhook asynchronously",
slog.String("url", payload.URL),
slog.String("activityType", payload.ActivityType),
slog.String("url", request.URL),
slog.String("label", request.Label),
slog.Any("err", err))
}
}
@@ -99,17 +97,17 @@ func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error
return nil, errors.Errorf("webhook: host %q resolved to no IP addresses", host)
}
type WebhookRequestPayload struct {
// The target URL for the webhook request.
URL string `json:"url"`
// The type of activity that triggered this webhook.
ActivityType string `json:"activityType"`
// The resource name of the creator. Format: users/{user}
Creator string `json:"creator"`
// The memo that triggered this webhook (if applicable).
Memo *v1pb.Memo `json:"memo"`
// Optional signing secret for HMAC-SHA256 signature. Not serialized to JSON.
SigningSecret string `json:"-"`
// Request is one webhook delivery: Payload is JSON-encoded and posted to URL,
// signed with SigningSecret when one is set.
type Request struct {
// URL is the destination endpoint.
URL string
// Label identifies the delivery in logs, for example an activity type.
Label string
// SigningSecret enables Standard Webhooks HMAC-SHA256 signing when non-empty.
SigningSecret string
// Payload is the value encoded as the JSON request body.
Payload any
}
// resolveSigningKey returns the raw HMAC key for a signing secret. Secrets using
@@ -139,9 +137,9 @@ func GenerateSigningSecret() (string, error) {
return "whsec_" + base64.StdEncoding.EncodeToString(buf), nil
}
// Post posts the message to webhook endpoint.
func Post(requestPayload *WebhookRequestPayload) error {
body, err := json.Marshal(requestPayload)
// Post delivers the request synchronously and returns the receiver's verdict.
func Post(requestPayload *Request) error {
body, err := json.Marshal(requestPayload.Payload)
if err != nil {
return errors.Wrapf(err, "failed to marshal webhook request to %s", requestPayload.URL)
}
@@ -202,11 +200,11 @@ func Post(requestPayload *WebhookRequestPayload) error {
return nil
}
// PostAsync posts the message to webhook endpoint asynchronously.
// It enqueues the request for bounded asynchronous dispatch and does not wait for the response.
func PostAsync(requestPayload *WebhookRequestPayload) {
// PostAsync enqueues the request for bounded asynchronous delivery and does
// not wait for the response.
func PostAsync(requestPayload *Request) {
if requestPayload == nil {
slog.Warn("Dropped webhook dispatch because payload is nil")
slog.Warn("Dropped webhook dispatch because request is nil")
return
}
select {
@@ -214,6 +212,6 @@ func PostAsync(requestPayload *WebhookRequestPayload) {
default:
slog.Warn("Dropped webhook dispatch because the async queue is full",
slog.String("url", requestPayload.URL),
slog.String("activityType", requestPayload.ActivityType))
slog.String("label", requestPayload.Label))
}
}
+8 -11
View File
@@ -125,7 +125,7 @@ func TestPostChecksRedirectDestinationAtDialTime(t *testing.T) {
defer redirectSource.Close()
sourceURL := strings.Replace(redirectSource.URL, "127.0.0.1", "localhost", 1)
err := Post(&WebhookRequestPayload{URL: sourceURL})
err := Post(&Request{URL: sourceURL})
require.ErrorContains(t, err, "connection to reserved/private IP address is not allowed")
require.False(t, redirectedRequestReceived.Load())
}
@@ -149,7 +149,7 @@ func TestPostAllowsExplicitPrivateIPPolicyAtDialTime(t *testing.T) {
}))
defer server.Close()
require.NoError(t, Post(&WebhookRequestPayload{URL: server.URL}))
require.NoError(t, Post(&Request{URL: server.URL}))
})
}
}
@@ -255,10 +255,9 @@ func TestPostSignsRequest(t *testing.T) {
}))
defer server.Close()
err := Post(&WebhookRequestPayload{
err := Post(&Request{
URL: server.URL,
ActivityType: "memos.memo.created",
Creator: "users/1",
Label: "memos.memo.created",
SigningSecret: tc.secret,
})
require.NoError(t, err)
@@ -292,10 +291,9 @@ func TestPostWithoutSecretSetsNoSignatureHeaders(t *testing.T) {
}))
defer server.Close()
err := Post(&WebhookRequestPayload{
err := Post(&Request{
URL: server.URL,
ActivityType: "memos.memo.created",
Creator: "users/1",
Label: "memos.memo.created",
})
require.NoError(t, err)
require.False(t, hasSignatureHeaders, "no signature headers should be set when no secret is configured")
@@ -316,10 +314,9 @@ func TestPostRejectsUnspecifiedAddressDestination(t *testing.T) {
_, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://"))
require.NoError(t, err)
err = Post(&WebhookRequestPayload{
err = Post(&Request{
URL: "http://0.0.0.0:" + port,
ActivityType: "memos.memo.created",
Creator: "users/1",
Label: "memos.memo.created",
})
require.Error(t, err)
require.False(t, receivedRequest.Load(), "the webhook must not reach a local service through the unspecified address")
+8
View File
@@ -0,0 +1,8 @@
// Package markdown is the markdown engine: parsing memo content into the AST
// in markdown/ast, the memos-specific syntax in markdown/extensions and
// markdown/parser (tags, mentions, math, GFM extras), rendering, and the
// derived memo payload (tags, properties, snippet).
//
// Layering: markdown may import proto/gen and internal. It must not import
// store, core, or server.
package markdown
@@ -13,8 +13,8 @@ import (
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
mast "github.com/usememos/memos/internal/markdown/ast"
mparser "github.com/usememos/memos/internal/markdown/parser"
mast "github.com/usememos/memos/markdown/ast"
mparser "github.com/usememos/memos/markdown/parser"
)
const gfmDomainPattern = `(?:[\p{L}\p{N}_-]+\.)*[\p{L}\p{N}-]+\.[\p{L}\p{N}-]+`
@@ -7,9 +7,9 @@ import (
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
"github.com/usememos/memos/internal/base"
mast "github.com/usememos/memos/internal/markdown/ast"
mparser "github.com/usememos/memos/internal/markdown/parser"
"github.com/usememos/memos/internal/identifier"
mast "github.com/usememos/memos/markdown/ast"
mparser "github.com/usememos/memos/markdown/parser"
)
type mentionExtension struct{}
@@ -43,7 +43,7 @@ func (*mentionASTTransformer) Transform(document *ast.Document, reader text.Read
func replaceMentionsInText(textNode *ast.Text, source []byte) {
segment := textNode.Segment
runHasLeftBoundary := segment.Padding > 0 || segment.Start == 0 || !base.IsUsernameCharacter(source[segment.Start-1])
runHasLeftBoundary := segment.Padding > 0 || segment.Start == 0 || !identifier.IsUsernameCharacter(source[segment.Start-1])
matches := mparser.FindMentionMatches(segment.Value(source), runHasLeftBoundary)
if len(matches) == 0 {
return
@@ -8,7 +8,7 @@ import (
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/text"
mast "github.com/usememos/memos/internal/markdown/ast"
mast "github.com/usememos/memos/markdown/ast"
)
func TestReplaceMentionsInPaddedText(t *testing.T) {
@@ -8,8 +8,8 @@ import (
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
mast "github.com/usememos/memos/internal/markdown/ast"
mparser "github.com/usememos/memos/internal/markdown/parser"
mast "github.com/usememos/memos/markdown/ast"
mparser "github.com/usememos/memos/markdown/parser"
)
type tagExtension struct{}
@@ -14,10 +14,10 @@ import (
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/text"
"github.com/usememos/memos/internal/base"
mast "github.com/usememos/memos/internal/markdown/ast"
"github.com/usememos/memos/internal/markdown/extensions"
"github.com/usememos/memos/internal/markdown/renderer"
"github.com/usememos/memos/internal/identifier"
mast "github.com/usememos/memos/markdown/ast"
"github.com/usememos/memos/markdown/extensions"
"github.com/usememos/memos/markdown/renderer"
storepb "github.com/usememos/memos/proto/gen/store"
)
@@ -506,7 +506,7 @@ func ParseManagedAttachmentImageURL(raw string) (uid string, managed, valid bool
}
parts := strings.Split(strings.TrimPrefix(parsed.Path, "/file/attachments/"), "/")
if (len(parts) != 1 && len(parts) != 2) || !base.UIDMatcher.MatchString(parts[0]) {
if (len(parts) != 1 && len(parts) != 2) || !identifier.UIDMatcher.MatchString(parts[0]) {
return "", true, false
}
if len(parts) == 2 && parts[1] == "" {
@@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/base"
"github.com/usememos/memos/internal/identifier"
)
func TestNewService(t *testing.T) {
@@ -351,7 +351,7 @@ func TestExtractAllMentions(t *testing.T) {
assert.ElementsMatch(t, []string{"Alice", "bob"}, data.Mentions)
assert.ElementsMatch(t, []string{"tag"}, data.Tags)
maxLengthUsername := "a" + strings.Repeat("b", base.MaxUsernameLength-1)
maxLengthUsername := "a" + strings.Repeat("b", identifier.MaxUsernameLength-1)
data, err = svc.ExtractAll([]byte("@" + maxLengthUsername))
require.NoError(t, err)
assert.Equal(t, []string{maxLengthUsername}, data.Mentions)
@@ -10,7 +10,7 @@ import (
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
mast "github.com/usememos/memos/internal/markdown/ast"
mast "github.com/usememos/memos/markdown/ast"
)
type inlineMathParser struct{}
@@ -11,7 +11,7 @@ import (
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
mast "github.com/usememos/memos/internal/markdown/ast"
mast "github.com/usememos/memos/markdown/ast"
)
func TestInlineMathParserDollarBoundaries(t *testing.T) {
@@ -3,7 +3,7 @@ package parser
import (
"github.com/yuin/goldmark/util"
"github.com/usememos/memos/internal/base"
"github.com/usememos/memos/internal/identifier"
)
// MentionMatch is one username reference in an eligible literal-source run.
@@ -20,7 +20,7 @@ func hasMentionLeftBoundary(source []byte, pos int, runHasLeftBoundary bool) boo
if pos == 0 {
return runHasLeftBoundary
}
return !base.IsUsernameCharacter(source[pos-1])
return !identifier.IsUsernameCharacter(source[pos-1])
}
// FindMentionMatches enumerates username references in one eligible literal-source run.
@@ -38,11 +38,11 @@ func FindMentionMatches(source []byte, runHasLeftBoundary bool) []MentionMatch {
}
end := pos + 1
for end < len(source) && base.IsUsernameCharacter(source[end]) {
for end < len(source) && identifier.IsUsernameCharacter(source[end]) {
end++
}
username := source[pos+1 : end]
if base.IsValidUsername(string(username)) {
if identifier.IsValidUsername(string(username)) {
matches = append(matches, MentionMatch{
Start: pos,
End: end,
@@ -6,7 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/usememos/memos/internal/base"
"github.com/usememos/memos/internal/identifier"
)
func TestFindMentionMatches(t *testing.T) {
@@ -17,8 +17,8 @@ func TestFindMentionMatches(t *testing.T) {
want []string
}{
{name: "valid", source: "@alice @Alice-2 @123 @123-456", runHasLeftBoundary: true, want: []string{"alice", "Alice-2", "123", "123-456"}},
{name: "maximum length", source: "@a" + strings.Repeat("b", base.MaxUsernameLength-1), runHasLeftBoundary: true, want: []string{"a" + strings.Repeat("b", base.MaxUsernameLength-1)}},
{name: "too long", source: "@a" + strings.Repeat("b", base.MaxUsernameLength), runHasLeftBoundary: true},
{name: "maximum length", source: "@a" + strings.Repeat("b", identifier.MaxUsernameLength-1), runHasLeftBoundary: true, want: []string{"a" + strings.Repeat("b", identifier.MaxUsernameLength-1)}},
{name: "too long", source: "@a" + strings.Repeat("b", identifier.MaxUsernameLength), runHasLeftBoundary: true},
{name: "invalid shapes", source: "@-alice @alice- @álîçé", runHasLeftBoundary: true},
{name: "username character left boundary", source: "hello@alice foo-@bob foo_@carol", runHasLeftBoundary: true, want: []string{"carol"}},
{name: "Unicode left boundary", source: "中文@alice", runHasLeftBoundary: true, want: []string{"alice"}},
@@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
mast "github.com/usememos/memos/internal/markdown/ast"
mast "github.com/usememos/memos/markdown/ast"
)
func TestFindTagMatches(t *testing.T) {
@@ -8,7 +8,7 @@ import (
gast "github.com/yuin/goldmark/ast"
east "github.com/yuin/goldmark/extension/ast"
mast "github.com/usememos/memos/internal/markdown/ast"
mast "github.com/usememos/memos/markdown/ast"
)
// MarkdownRenderer renders goldmark AST back to markdown text.
@@ -10,7 +10,7 @@ import (
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/text"
"github.com/usememos/memos/internal/markdown/extensions"
"github.com/usememos/memos/markdown/extensions"
)
func TestMarkdownRenderer(t *testing.T) {
+15
View File
@@ -0,0 +1,15 @@
# provider
Backends that memos talks to on behalf of an instance, each configured by an
instance setting and reached through an interface so the implementation can be
swapped.
| Package | Backend |
| --- | --- |
| `ai/` | speech-to-text and audio-capable LLM providers (OpenAI, Gemini) |
| `idp/` | identity providers for SSO (OAuth2) |
| `storage/` | attachment object storage (local disk, S3) |
Layering: `provider` may import `proto/gen` and `internal`. It must not import
`store`, `core`, or `server`; `store` resolves the configured provider and hands
it to callers.
@@ -1,6 +1,6 @@
// Package audiollm defines the multimodal-audio capability for AI providers.
// Implementations call chat-completions or generate-content style APIs that
// accept audio as input. For deterministic transcription, prefer internal/ai/stt
// accept audio as input. For deterministic transcription, prefer provider/ai/stt
// where a dedicated STT endpoint exists.
package audiollm
@@ -13,9 +13,9 @@ import (
"github.com/pkg/errors"
"google.golang.org/genai"
"github.com/usememos/memos/internal/ai"
"github.com/usememos/memos/internal/ai/audio"
"github.com/usememos/memos/internal/ai/audiollm"
"github.com/usememos/memos/provider/ai"
"github.com/usememos/memos/provider/ai/audio"
"github.com/usememos/memos/provider/ai/audiollm"
)
const (
@@ -12,9 +12,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/ai"
"github.com/usememos/memos/internal/ai/audiollm"
audiollmgemini "github.com/usememos/memos/internal/ai/audiollm/gemini"
"github.com/usememos/memos/provider/ai"
"github.com/usememos/memos/provider/ai/audiollm"
audiollmgemini "github.com/usememos/memos/provider/ai/audiollm/gemini"
)
func TestGenerateFromAudio(t *testing.T) {
@@ -13,8 +13,8 @@ import (
openaioption "github.com/openai/openai-go/v3/option"
"github.com/pkg/errors"
"github.com/usememos/memos/internal/ai"
"github.com/usememos/memos/internal/ai/stt"
"github.com/usememos/memos/provider/ai"
"github.com/usememos/memos/provider/ai/stt"
)
const defaultEndpoint = "https://api.openai.com/v1"
@@ -11,9 +11,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/ai"
"github.com/usememos/memos/internal/ai/stt"
sttopenai "github.com/usememos/memos/internal/ai/stt/openai"
"github.com/usememos/memos/provider/ai"
"github.com/usememos/memos/provider/ai/stt"
sttopenai "github.com/usememos/memos/provider/ai/stt/openai"
)
func TestTranscribe(t *testing.T) {
@@ -1,7 +1,7 @@
// Package stt defines the speech-to-text capability for AI providers.
// Implementations call dedicated STT endpoints (e.g. OpenAI /audio/transcriptions)
// and return deterministic transcription output. For multimodal LLMs that
// happen to accept audio input, see internal/ai/audiollm.
// happen to accept audio input, see provider/ai/audiollm.
package stt
import (
@@ -12,8 +12,8 @@ import (
"github.com/pkg/errors"
"golang.org/x/oauth2"
"github.com/usememos/memos/internal/idp"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/provider/idp"
)
// IdentityProvider represents an OAuth2 Identity Provider.
@@ -13,8 +13,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/idp"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/provider/idp"
)
func TestNewIdentityProvider(t *testing.T) {
@@ -6,8 +6,8 @@ import (
"github.com/pkg/errors"
"github.com/usememos/memos/internal/storage/s3"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/provider/storage/s3"
)
// ErrRangeNotSatisfiable reports a ranged read whose byte range falls outside

Some files were not shown because too many files have changed in this diff Show More