diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index ff2fb0f60..15ba85673 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -9,6 +9,7 @@ on: - "go.mod" - "go.sum" - "**.go" + - "store/migration/**" - ".golangci.yaml" - ".github/actions/setup-go/**" - ".github/workflows/backend-tests.yml" @@ -60,7 +61,7 @@ jobs: run: | case "${{ matrix.test-group }}" in store) - # Run store tests for all drivers (sqlite, mysql, postgres) + # Run all database targets (SQLite, MySQL, MariaDB, PostgreSQL). DRIVER= go test -v -coverprofile=coverage.out -covermode=atomic ./store/... ;; server) diff --git a/.github/workflows/upgrade-smoke.yml b/.github/workflows/upgrade-smoke.yml index 9ec68df3f..d5ea6dfec 100644 --- a/.github/workflows/upgrade-smoke.yml +++ b/.github/workflows/upgrade-smoke.yml @@ -15,11 +15,15 @@ on: branches: [main] paths: # Anything that can change how an existing database is opened or upgraded. + - "go.mod" + - "go.sum" + - "filter/**" - "store/migration/**" - "store/migrator.go" - "store/schema_version*.go" - "store/db/**" - "store/test/containers.go" + - "store/test/main_test.go" - "store/test/migrator*_test.go" - "store/test/store.go" - "server/server.go" @@ -41,12 +45,24 @@ concurrency: jobs: migration-upgrade: - name: Upgrade from previous stable (${{ matrix.driver }}) + name: Upgrade from previous stable (${{ matrix.target }}) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - driver: [sqlite, mysql, postgres] + include: + - target: sqlite + driver: sqlite + mysql-variant: "" + - target: mysql + driver: mysql + mysql-variant: mysql + - target: mariadb + driver: mysql + mysql-variant: mariadb + - target: postgres + driver: postgres + mysql-variant: "" steps: - name: Checkout code uses: actions/checkout@v6 @@ -64,6 +80,7 @@ jobs: go test -v -timeout 30m -run 'TestMigration|TestUpgrade|TestFreshInstall' ./store/test/... env: DRIVER: ${{ matrix.driver }} + MYSQL_VARIANT: ${{ matrix.mysql-variant }} release-smoke: name: Fresh install and upgrade through the release image diff --git a/AGENTS.md b/AGENTS.md index 9fcaae8ac..96f607481 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ Memos is a self-hosted note-taking app. - Backend: Go 1.27.4, net/http, Connect RPC, gRPC-Gateway, Protocol Buffers. - Frontend: React 19, TypeScript 7, Vite 8, Tailwind CSS v4, `@usememos/ui` on Base UI, React Query v5. -- Storage: SQLite, MySQL, PostgreSQL. +- Storage: SQLite, MySQL, PostgreSQL; MariaDB 11.4.13 compatibility is tested through the MySQL driver. - Generated API outputs: `proto/gen/` for Go/OpenAPI, `web/src/types/proto/` for TypeScript. ## Working Rules @@ -40,6 +40,7 @@ go run -buildvcs=true ./cmd/memos --port 8081 # Start backend with commit-derive go run -buildvcs=true ./cmd/memos config plan --data DIR --config-dir DIR # Declarative configuration: also validate, apply, state go test ./... # Run all Go tests go test -v ./store/... # Store tests, including DB drivers via TestContainers +DRIVER=mysql MYSQL_VARIANT=mariadb go test -v ./store/test/... # MariaDB compatibility tests go test -v -race ./server/... # Server tests with race detector go test -v -race ./internal/... # Internal package tests with race detector go test -v -run TestFoo ./core/... # Run matching Go tests in one tree @@ -150,6 +151,8 @@ Black-box service tests live in `server/api/test/`, unit tests next to the code. ## Database And Proto Rules +- Prefer MySQL SQL and metadata checks shared with MariaDB when correctness, performance, and maintainability are preserved. + Keep MariaDB on the MySQL driver and dialect; verify shared SQL changes against both database targets. - Schema changes require SQLite, MySQL, and PostgreSQL migrations plus `LATEST.sql` updates. - Fresh-install SQL and incremental migrations must stay equivalent. - Proto field changes must preserve compatibility unless the task explicitly allows a breaking API change. Record every diff --git a/docs/design/README.md b/docs/design/README.md index c39cdc70f..df8117456 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -17,6 +17,7 @@ behavior; a design document does not have to. Implemented, awaiting release: - [Markup: image annotation and saving](drawing-annotation-and-recovery.md) +- [MariaDB compatibility testing](mariadb-compatibility-testing.md) Implemented and frozen: diff --git a/docs/design/mariadb-compatibility-testing.md b/docs/design/mariadb-compatibility-testing.md new file mode 100644 index 000000000..eeb154976 --- /dev/null +++ b/docs/design/mariadb-compatibility-testing.md @@ -0,0 +1,293 @@ +# MariaDB compatibility testing + +Status: Implemented; awaiting a tagged release. + +Date: 2026-11-16 + +Related: [PR #6571](https://github.com/usememos/memos/pull/6571), [store tests](../../store/test/README.md), +[migration procedure](../../store/migration/README.md). + +## Decision and scope + +Test MariaDB through the existing `mysql` application driver, SQL dialect, and migrations. Add a fourth database test target +alongside SQLite, MySQL, and PostgreSQL. Run the same store and upgrade tests against it. + +MySQL remains the primary implementation target. Prefer SQL and metadata checks shared with MariaDB when they preserve +correctness, performance, and maintainability. A MySQL-specific feature needs a concrete benefit that justifies the compatibility +cost. Do not add server-version detection or a second SQL dialect for the differences identified here. + +The initial MariaDB target is `mariadb:11.4.13`. The local `mariadb:11.4` image used for design probes reports that version. +Pin the patch tag in the test setup; keep the existing `mysql:8.4` target. MariaDB 11.4 has community maintenance through May 2029, +according to the [MariaDB maintenance policy](https://mariadb.org/about/). + +The compatibility target is MariaDB 11.4.13 using `driver=mysql`. This does not establish compatibility with every MariaDB version, +MySQL-to-MariaDB database conversion, or cross-product replication. + +This change covers test containers, confirmed SQL incompatibilities, CI, and contributor guidance. It adds no application driver, +schema version, migration directory, API, or user-facing configuration option. + +## Audit baseline and confirmed gaps + +The source inspection used main at `bdc41c236`, which includes #6571. + +- `store/test/main_test.go` runs three driver subprocesses when `DRIVER` is empty. An explicit `DRIVER` runs one suite. +- `GetMySQLDSN` starts one shared MySQL 8.4 container and creates an isolated database for each test. +- Released-binary upgrade tests also connect to that container through the Docker network alias `memos-mysql`. +- Backend Tests invokes the aggregate store run. Upgrade Smoke has a three-driver matrix and is also called by Release. +- `requireLookupIndexes` still queries `IS_VISIBLE`, even though #6571 removed it from the migration. +- `filter/render.go` casts JSON boolean literals with `CAST(... AS JSON)`. +- `store/db/mysql/memo.go` uses the same cast for tags and property objects in `RefreshMemoDerivedPayload`. +- `ListSpaces` selects `space.payload` without grouping it. MySQL accepts the primary-key functional dependency; MariaDB + rejects the query when `ONLY_FULL_GROUP_BY` is enabled. +- `store/user_unique.go` relies on `user_identity` in the constraint target. MariaDB reports the bare index names `provider` + and `user_id`, so identity conflicts are not classified as `ErrUserIdentityTaken`. + +Direct SQL probes against MySQL 8.4 and MariaDB 11.4.13 confirmed that MariaDB rejects `CAST('true' AS JSON)` with error 1064. +Both servers accept `JSON_EXTRACT(document, '$')`. In the probes, it preserves nested arrays and objects passed to `JSON_SET`. +Using it for boolean constants also gives the same results for true, false, missing values, JSON null, the string `"true"`, and +the number 1. The broader audit and suite results are recorded below. These initial diagnostic overlays did not change repository +source files; the implementation now uses the shared expressions and container selector described here. + +See the [MariaDB CAST reference](https://mariadb.com/docs/server/reference/sql-functions/string-functions/cast) and +[JSON_EXTRACT reference](https://mariadb.com/docs/server/reference/sql-functions/special-functions/json-functions/json_extract). + +## SQL audit coverage and results + +The audit covered all 26 production Go files in `store/db/mysql`, every current MySQL schema and migration file, the store-level +migration executor and unique-error classifier, and the MySQL SQL emitted by `filter/render.go` and `filter/schema.go`. + +| Source | Reviewed behavior | Result on the audited versions | +| --- | --- | --- | +| `store/migration/mysql/LATEST.sql` | All 13 tables, indexes, JSON columns, CHECK constraints, timestamp expression defaults, and share foreign key | Complete script applies on both servers; Unicode content round-trips; invalid JSON and member roles are rejected; deleting a memo cascades to its shares | +| Both incremental MySQL migration files | Creator indexes and attachment lookup index, including upgrade and retry paths | Upgrade suites execute them; the remaining `IS_VISIBLE` failure is in the test assertion | +| `mysql.go`, `common.go`, `executor.go` | DSN merging, metadata queries, multi-statement execution, JSON decoding, and executor interfaces | No additional SQL incompatibility found | +| `user.go`, `user_identity.go` | User CRUD, unique nullable email, identity transactions, generated IDs, and admin retention | Existing store behavior tests pass after the three initial overlay corrections; stronger identity-error assertions expose an additional classifier gap | +| `user_delete.go`, `space_delete.go` | Batched deletes, relationship cleanup, and parent row locking | Existing data-preservation and concurrency tests pass on MariaDB | +| `space.go` | Membership lifecycle, summaries, aggregates, pagination, and locks | Default mode passes; two existing Space tests fail under `ONLY_FULL_GROUP_BY`; explicitly grouping payload fixes those tests | +| `memo.go`, `memo_attachment.go` | Memo CRUD, guarded payload refresh, attachment binding, timestamps, and serializable mutations | JSON casts need replacement; remaining tested operations pass | +| `memo_access.go`, `memo_policy.go`, `memo_delete_policy.go`, `memo_comment.go`, `memo_relation_policy.go` | Access predicates, EXISTS queries, authorization snapshots, subtree traversal, and locking | No additional SQL incompatibility found in source or existing MariaDB tests | +| `attachment.go`, `attachment_delete.go`, `memo_relation.go`, `memo_share.go` | Attachment queries and updates, filters, relation upserts, shares, limits, and usage sums | Existing MariaDB tests pass, including binary content guards and relation cleanup | +| `reaction.go`, `reaction_policy.go` | INSERT SELECT with LOCK IN SHARE MODE, upserts, readback, and participation checks | Existing MariaDB tests pass, including concurrent memo deletion | +| `auth_config.go`, `instance_setting.go`, `idp.go` | FOR UPDATE, serializable configuration mutations, INSERT IGNORE, ON DUPLICATE KEY UPDATE, affected-row handling, and binary compare-and-swap | Existing MariaDB tests pass; retry detection still uses errors 1205 and 1213 | +| `user_setting.go`, `inbox.go` | JSON_SEARCH for PAT hashes and JSON_EXTRACT comparisons for inbox enums | Existing MariaDB tests and direct SQL probes pass | +| `filter/render.go`, `filter/schema.go` | JSON booleans, existence, JSON_TABLE tag comprehensions, binary tag matching, size, related-row predicates, string filters, regex, resource expressions, and date accessors | Boolean casts need replacement; existing execution tests pass with the candidate expressions | +| `store/user_unique.go` | Mapping database uniqueness errors to typed store errors | MariaDB identity constraints need recognition without the MySQL table qualifier | + +### Diagnostic runs + +The audit used MySQL 8.4.11 from `mysql:8.4` and MariaDB 11.4.13 from `mariadb:11.4`. Temporary `go test -overlay` files changed +the test image and candidate expressions without editing application or test source in the worktree. + +- With only the image replaced, `DRIVER=mysql go test -v -count=1 -timeout 30m ./store/...` reported 13 failing top-level tests. + The reported causes were `IS_VISIBLE` in the test assertion and the two production JSON-cast sites. +- With those three sites corrected in an overlay, the same command passed on MariaDB. Across the store packages, 388 top-level + tests passed; existing driver-specific and opt-in skips remained. Released `0.30.0` through `0.31.0` upgrades and direct + `0.31.0` upgrades both passed. +- The same candidate SQL and index assertion passed the full store run on MySQL 8.4.11 as well, with 388 top-level passes. +- Enabling `ONLY_FULL_GROUP_BY` reproduced failures in `TestSpaceStoreMembershipGuards` and + `TestSpaceStoreInvitationLifecycle`. Adding `space.payload` to the grouping clause made the focused Space selection pass. +- Strengthening the existing identity conflict assertions from `require.Error` to `require.ErrorIs` exposed failures in + `TestUserIdentityUniqueConflict`, `TestUserIdentitySameUserSameProviderConflicts`, and the identity-conflict subtest of + `TestCreateUserWithIdentityIsAtomic`. The strengthened selection passed on MySQL. Username classification still passed on MariaDB. + +The existing suite becoming green after three corrections does not settle compatibility: the stricter SQL mode and typed-error +checks found gaps outside its original assertions. Final acceptance includes those stronger checks. + +### Differences to retain in the compatibility boundary + +Both audited images defaulted to InnoDB and utf8mb4, but their table collations differed: MySQL used `utf8mb4_0900_ai_ci`, while +MariaDB used `utf8mb4_uca1400_ai_ci`. Username and email already specify `utf8mb4_bin`; tag comparisons and payload content guards +use explicit binary comparisons. Keep those exact comparisons. Do not introduce a collation migration to make all text ordering +identical. Document utf8mb4 and InnoDB as deployment requirements rather than assuming every MariaDB installation has the +audited image's defaults. + +MariaDB's JSON implementation differs from MySQL's, but the current JSON column declarations preserve validation on both. +Keep the schema's JSON types. The audited defaults, constraints, and indexes do not require a `LATEST.sql` change or a new migration. + +The existing REGEXP operator uses different regex engines across database products. Common patterns pass the existing tests; +Unicode category and boundary semantics are not established as identical. Likewise, date accessors continue to use the database +session timezone. These are documented portability limits, not reasons to silently change filter semantics or remove features. +See [MariaDB REGEXP](https://mariadb.com/docs/server/reference/sql-functions/string-functions/regular-expressions-functions/regexp) +and [MySQL grouping behavior](https://dev.mysql.com/doc/refman/8.4/en/group-by-handling.html). + +## Test target selection + +Keep `DRIVER` as the application driver. Add a test-only selector `MYSQL_VARIANT`, with `mysql` as its default and `mariadb` as +the other accepted value. + +| Target label | DRIVER | MYSQL_VARIANT | Container | +| --- | --- | --- | --- | +| sqlite | sqlite | unset | none | +| mysql | mysql | mysql | mysql:8.4 | +| mariadb | mysql | mariadb | mariadb:11.4.13 | +| postgres | postgres | unset | postgres:18 | + +An explicit MySQL run without `MYSQL_VARIANT` retains its current behavior. Reject unknown variants before starting containers. +The selector only controls the MySQL test fixture. It never changes `Profile.Driver`, filter dialect selection, or `MEMOS_DRIVER`. + +The default aggregate run adds a `mariadb` entry. Each subprocess receives explicit selector values, including an empty value +for non-MySQL targets, so an inherited environment cannot turn the MySQL run into a second MariaDB run. Labels and startup +logs identify the selected server and image. MariaDB errors must fail that run; they must not select MySQL as a fallback. + +## Container setup and upgrade connectivity + +Use the [Testcontainers MariaDB module](https://golang.testcontainers.org/modules/mariadb/), pinned to `v0.44.0` to match the +existing Testcontainers dependencies. It handles MariaDB credentials and startup behavior. This adds a test module, with no +new application database driver or Testcontainers upgrade. + +Extend `GetMySQLDSN` to select the appropriate container startup branch. Reuse database creation, the existing SQL connection +check, and `multiStatements=true`. Preserve one selected server per test process and one database per test. Clean up the selected +container and network through the existing teardown path, including partially started containers when startup returns an error. + +Enable `ONLY_FULL_GROUP_BY` for the MariaDB fixture while retaining its strict-mode settings. MySQL already enables this check +by default. This keeps the shared queries from relying on MariaDB's looser default grouping rules without adding another CI target. + +Give either server the existing `memos-mysql` network alias. The variant runs in a separate process and network, so the alias +does not conflict. `getContainerDSN` can retain its MySQL address rewrite, and released Memos containers continue to receive +`MEMOS_DRIVER=mysql`. Verify both host-to-database and Memos-container-to-database connections. + +Do not duplicate the upgrade fixtures or add MariaDB cases to every application-driver switch. + +## SQL changes and behavior to preserve + +### Index assertions + +The attachment index migration adds a performance index. It does not audit an operator's existing index definition or implement +a separate migration recovery mechanism. MySQL uses ordinary `CREATE INDEX`, matching the 26.10 scripts. SQLite and PostgreSQL +use `CREATE INDEX IF NOT EXISTS`. The index declarations are the same as before. + +The migrator skips files whose schema version is already recorded, so a completed upgrade can restart. MySQL commits DDL before +the version marker; an interrupted migration may therefore require manual inspection and recovery. That affects MySQL migrations +generally and is not solved specially in the 26.11 script. See [MySQL CREATE INDEX](https://dev.mysql.com/doc/refman/8.4/en/create-index.html). + +Remove `IS_VISIBLE` from the test assertion, matching #6571. Fresh-install and ordinary upgrade tests still check ordered columns, +full-column coverage, non-unique status, and BTREE structure. The attachment migration test starts with the index absent, upgrades, +verifies preserved data and the schema marker, then verifies a restart. It no longer claims recovery for partial DDL or custom +index definitions. There is no invisible-index branch in the migration or tests. + +### JSON boolean filters + +Replace JSON literal casts with `JSON_EXTRACT('true', '$')` and `JSON_EXTRACT('false', '$')` in the MySQL renderer. Preserve the +existing `COALESCE` and negation behavior. A missing flag counts as false, and `flag == false` matches the same memos as `!flag`. +The change also applies to the payload half of `has_image`. + +Update the nearby SQL expectation tests. Extend existing store filter tests only where needed to verify actual database behavior, +including explicit JSON null and false values. Do not replace boolean comparisons with loose numeric or string coercion. + +### Derived payload refresh + +Replace `CAST(? AS JSON)` with `JSON_EXTRACT(?, '$')` for tags and property values in `RefreshMemoDerivedPayload`. +Keep the single guarded `UPDATE`, binary content comparison, and partial `JSON_SET` update. Tags must remain an array and property +an object; a refresh must preserve unrelated payload keys and timestamps and must not overwrite a concurrent content edit. + +Reuse `TestRefreshMemoDerivedPayload`, which already covers these user-visible guarantees. Add a type assertion only if the +existing round-trip assertions do not expose an array or object accidentally stored as a string. + +### Space aggregation + +Include `space.payload` in `ListSpaces`' grouped columns. It is already selected, and the primary key keeps each Space as one +group, so the change preserves the intended result on both servers. Keep member counting and current-user role behavior. +Run the existing Space tests with the MariaDB fixture's strict grouping mode enabled. + +### Identity constraint errors + +Recognize MariaDB's unqualified identity index names in the unique-error classifier. Restrict matching to the constraint target +and known identity indexes; duplicate values must not affect classification. Keep the existing typed-error API and transactional +rollback behavior. No constraint renaming or schema migration is needed. + +Strengthen the existing identity-conflict tests to require `ErrUserIdentityTaken` and the atomic username-conflict test to require +`ErrUsernameTaken`. Add the observed MariaDB error strings to `store/user_unique_test.go` alongside the MySQL fixtures. + +Run the full suite to find further differences. Record each failure before deciding on a shared fix. Do not skip failing MariaDB +behavior tests to make the target green or broaden this change into speculative SQL cleanup. + +## CI and release checks + +Keep the existing Backend Tests store job. Its aggregate invocation will run four targets sequentially. This is the smallest CI +change; measure the added duration before deciding whether separate database jobs are worthwhile. Add `store/migration/**` to +its pull-request paths so SQL-only changes also run the full store suite. + +Expand Upgrade Smoke to four named targets using explicit driver and variant values from the table above. Include the new +selector's entry point, dependency manifests, and filter implementation in its pull-request paths. Keep the existing migration +and released-binary upgrade selection. + +MariaDB failures should block the same checks as the other tested targets once all suites pass. Upgrade Smoke is reused by Release, +so this also adds MariaDB to the release prerequisite. This is an intended change to the release gate. It requires no edit to +`release.yml`, Dockerfiles, image publication, or release scripts. The separate release-image smoke test retains its current scope. + +## Implementation files + +| Files | Change | +| --- | --- | +| `store/test/containers.go` | Select MySQL or MariaDB, preserve host/network DSNs, and clean up both startup branches | +| `store/test/main_test.go` | Add the MariaDB aggregate target and validate the test selector | +| `go.mod`, `go.sum` | Add the matching MariaDB Testcontainers module | +| `store/test/migrator_test.go` | Align index assertions with the shared migration contract | +| `store/migration/{sqlite,mysql,postgres}/26.11/00__add_attachment_memo_index.sql` | Use ordinary index creation without definition auditing or special recovery | +| `store/test/migrator_attachment_index_test.go` | Cover index creation, data preservation, and restart after a completed upgrade | +| `filter/render.go`, `filter/engine_test.go`, `filter/README.md` | Use shared JSON expressions and update expectations and guidance | +| `store/db/mysql/memo.go` | Preserve structured JSON in derived payload updates using shared SQL | +| `store/db/mysql/space.go` | Explicitly group the selected payload column | +| `store/user_unique.go`, `store/user_unique_test.go` | Recognize unqualified MariaDB identity index names | +| `store/test/user_identity_test.go` | Assert typed conflict errors as well as transaction rollback | +| `store/test/memo_filter_test.go` | Fill any gaps in boolean/null behavior coverage | +| `store/test/memo_payload_refresh_test.go` | Extend only if needed to detect JSON type regressions | +| `.github/workflows/backend-tests.yml` | Update target comments and migration path coverage | +| `.github/workflows/upgrade-smoke.yml` | Add the MariaDB target and relevant trigger paths | +| `store/test/README.md`, `AGENTS.md` | Document commands, tested scope, and the shared-SQL preference | +| `store/migration/README.md`, `docs/design/memo-attachment-filters.md` | Document the simpler index migration contract | + +`store/test/store.go` and `migrator_upgrade_test.go` should keep their existing driver dispatch. Update their comments only if +needed. The existing attachment index migration is simplified on all three drivers. It creates the same indexes, so there is no +new schema version, new migration, or `LATEST.sql` edit. + +## Implementation order and acceptance + +1. Add the selector and container fixture locally, including strict grouping mode. Reproduce the audited failures using the real fixture. +2. Fix the index assertion, JSON expressions, Space grouping, and identity-error classification. Run focused tests on MySQL and + MariaDB, then the full store suites with the stronger conflict assertions. +3. Add the CI target and contributor guidance after both suites pass. Verify SQLite and PostgreSQL through the aggregate run. +4. Run the upgrade selection against all four targets, including released `0.30.0` through `0.31.0` and direct `0.31.0` upgrades. + A historical-binary failure must be investigated and documented rather than silently removed from MariaDB coverage. + +Planned verification commands, from the repository root: + +```bash +DRIVER=mysql MYSQL_VARIANT=mysql go test -v -count=1 -run 'TestMigrationAttachmentIndex|TestRefreshMemoDerivedPayload|TestMemoFilter|TestSpaceStore|TestListSpaces|TestUserIdentity|TestCreateUserWithIdentity' ./store/test/... +DRIVER=mysql MYSQL_VARIANT=mariadb go test -v -count=1 -run 'TestMigrationAttachmentIndex|TestRefreshMemoDerivedPayload|TestMemoFilter|TestSpaceStore|TestListSpaces|TestUserIdentity|TestCreateUserWithIdentity' ./store/test/... +go test -v -race ./filter/... +go test -v ./store/... +DRIVER=mysql MYSQL_VARIANT=mariadb go test -v -count=1 -timeout 30m -run 'TestMigration|TestUpgrade|TestFreshInstall' ./store/test/... +go mod tidy -go=1.27.4 +go fix -diff ./... +golangci-lint run +git diff --check +``` + +Run the upgrade selection with the SQLite, MySQL, and PostgreSQL target environments as well. Acceptance requires the full store +suite on each target, strict grouping and typed conflict-error checks, data preservation and restart checks on upgrades, and no +MySQL behavior regression. Passing the diagnostic overlays alone does not satisfy implementation acceptance. Record suite +results and CI duration when implementation is complete. + +## Implementation verification + +Local verification completed on 2026-11-16 using the implementation, without diagnostic overlays: + +| Check | Result | +| --- | --- | +| Full `./store/...` suite with `DRIVER=sqlite` | Passed | +| Full `./store/...` suite with `DRIVER=mysql MYSQL_VARIANT=mysql` | Passed | +| Full `./store/...` suite with `DRIVER=mysql MYSQL_VARIANT=mariadb` | Passed, with ONLY_FULL_GROUP_BY enabled | +| Full `./store/...` suite with `DRIVER=postgres` | Passed | +| Fresh installs, attachment index upgrades, released 0.30.0 through 0.31.0 upgrades, and direct 0.31.0 upgrades | Passed on all four targets as part of their full suites | +| `go test -v -race ./server/... ./core/...` | Passed | +| `go test -v -race ./markdown/... ./filter/... ./provider/...` | Passed | +| golangci-lint v2.14.0 built with the repository's Go toolchain | Passed, zero issues | +| `go fix -diff ./...`, `go mod tidy -go=1.27.4`, and `git diff --check` | Passed | +| Workflow YAML and four-target matrix validation | Passed | +| Unknown MYSQL_VARIANT | Rejected before container startup | + +The aggregate selector was also exercised with an inherited MariaDB variant and ran all four database targets. Final store +suites were run separately after simplifying the index scripts. Local parallel runs of `store/test` took approximately 39 seconds +for SQLite, 259 seconds for MySQL, 221 seconds for MariaDB, and 149 seconds for PostgreSQL; these are not CI timing measurements. +GitHub Actions execution remains to be verified after pushing the change. diff --git a/docs/design/memo-attachment-filters.md b/docs/design/memo-attachment-filters.md index 2d2cbc637..09116f670 100644 --- a/docs/design/memo-attachment-filters.md +++ b/docs/design/memo-attachment-filters.md @@ -265,3 +265,13 @@ all instances and other writers, migrating once, then restarting, as described i [the migration procedure](../../store/migration/README.md#coordinating-an-upgrade). The migrations reject conflicting index definitions; MySQL also accepts a completed index when retrying after an implicit DDL commit. + +### 2026-11-16: Simplify index migration scripts + +The attachment lookup index migration adds a performance index without auditing +existing definitions. MySQL uses ordinary `CREATE INDEX`, matching its 26.10 +migrations; SQLite and PostgreSQL use `CREATE INDEX IF NOT EXISTS`. Migration +versions handle restarts after a completed upgrade. Recovery after a partially +committed MySQL migration remains an execution-wide concern, with no special +recovery SQL in this script. Tests verify fresh installs, ordinary upgrades, +data preservation, and restarts after successful upgrades. diff --git a/filter/MAINTENANCE.md b/filter/MAINTENANCE.md index 2b3bda2ff..24afb049e 100644 --- a/filter/MAINTENANCE.md +++ b/filter/MAINTENANCE.md @@ -49,5 +49,6 @@ extending or debugging the package. - `go test ./store/...` ensures all dialect tests consume the engine correctly. - Add targeted unit tests whenever new IR nodes or renderer paths are introduced. -- When changing boolean or JSON handling, verify all three dialect test suites - (SQLite, MySQL, Postgres) to avoid regression. +- When changing boolean or JSON handling, verify SQLite, MySQL, MariaDB, and + PostgreSQL execution through the store tests. MariaDB uses the MySQL dialect; + see `store/test/README.md` for the test selector. diff --git a/filter/README.md b/filter/README.md index bf39ca2a3..fa9aae264 100644 --- a/filter/README.md +++ b/filter/README.md @@ -60,7 +60,7 @@ stmt, _ := engine.CompileToStatement(ctx, `has_task_list && visibility == "PUBLI string operands are literal. Hierarchy works through implied ancestors already present in the memo tag set, not through prefix matching. - **Boolean Flags** — Fields such as `has_task_list` render as an `IS TRUE` check - (a comparison against `CAST('true' AS JSON)` on MySQL). Memo payloads omit false + (a comparison against `JSON_EXTRACT('true', '$')` on MySQL). Memo payloads omit false flags, so a missing key counts as false: `flag == false` and `flag != true` render as the negated check and match the same memos as `!flag`. - **Presence Flags** — `has_location` renders as a JSON key-existence check on diff --git a/filter/engine_test.go b/filter/engine_test.go index 86bb31e10..322f2aed4 100644 --- a/filter/engine_test.go +++ b/filter/engine_test.go @@ -673,7 +673,7 @@ func TestRenderJSONBoolComparisonsPerDialect(t *testing.T) { predicates := map[DialectName]string{ DialectSQLite: "JSON_EXTRACT(`memo`.`payload`, '$.property.hasLink') IS TRUE", - DialectMySQL: "COALESCE(JSON_EXTRACT(`memo`.`payload`, '$.property.hasLink'), CAST('false' AS JSON)) = CAST('true' AS JSON)", + DialectMySQL: "COALESCE(JSON_EXTRACT(`memo`.`payload`, '$.property.hasLink'), JSON_EXTRACT('false', '$')) = JSON_EXTRACT('true', '$')", DialectPostgres: "(memo.payload->'property'->>'hasLink')::boolean IS TRUE", } for dialect, predicate := range predicates { @@ -770,7 +770,7 @@ func TestRenderExistsComparisonsPerDialect(t *testing.T) { "has_image": { DialectSQLite: "(JSON_EXTRACT(`memo`.`payload`, '$.property.hasInlineImage') IS TRUE OR " + "EXISTS (SELECT 1 FROM `attachment` WHERE `attachment`.`memo_id` = `memo`.`id` AND LOWER(`attachment`.`type`) LIKE 'image/%'))", - DialectMySQL: "(COALESCE(JSON_EXTRACT(`memo`.`payload`, '$.property.hasInlineImage'), CAST('false' AS JSON)) = CAST('true' AS JSON) OR " + + DialectMySQL: "(COALESCE(JSON_EXTRACT(`memo`.`payload`, '$.property.hasInlineImage'), JSON_EXTRACT('false', '$')) = JSON_EXTRACT('true', '$') OR " + "EXISTS (SELECT 1 FROM `attachment` WHERE `attachment`.`memo_id` = `memo`.`id` AND LOWER(`attachment`.`type`) LIKE 'image/%'))", DialectPostgres: "((memo.payload->'property'->>'hasInlineImage')::boolean IS TRUE OR " + "EXISTS (SELECT 1 FROM attachment WHERE attachment.memo_id = memo.id AND LOWER(attachment.type) LIKE 'image/%'))", diff --git a/filter/render.go b/filter/render.go index f178294b3..167ddfe53 100644 --- a/filter/render.go +++ b/filter/render.go @@ -848,7 +848,7 @@ func (r *renderer) jsonBoolPredicate(field Field) (string, error) { case DialectSQLite: return fmt.Sprintf("%s IS TRUE", expr), nil case DialectMySQL: - return fmt.Sprintf("COALESCE(%s, CAST('false' AS JSON)) = CAST('true' AS JSON)", expr), nil + return fmt.Sprintf("COALESCE(%s, JSON_EXTRACT('false', '$')) = JSON_EXTRACT('true', '$')", expr), nil case DialectPostgres: return fmt.Sprintf("(%s)::boolean IS TRUE", expr), nil default: diff --git a/go.mod b/go.mod index ff2208b9a..d9592b10f 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.44.0 + github.com/testcontainers/testcontainers-go/modules/mariadb v0.44.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/yuin/goldmark v1.8.5 diff --git a/go.sum b/go.sum index 6e8d77fd7..c24dbd700 100644 --- a/go.sum +++ b/go.sum @@ -247,6 +247,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/testcontainers/testcontainers-go/modules/mariadb v0.44.0 h1:tiQ6mPa+Q1Cp79xXxKxy0EDftUvrsGulYEYFbPc6H7g= +github.com/testcontainers/testcontainers-go/modules/mariadb v0.44.0/go.mod h1:tepyUQIx4NyelTgbCvnf/C1IuKbG2L99DLKlVBAfc9I= github.com/testcontainers/testcontainers-go/modules/mysql v0.44.0 h1:oJPJPxNE6YQ0zlq6mZKh06JOlyimCky4ruQUimdDet4= github.com/testcontainers/testcontainers-go/modules/mysql v0.44.0/go.mod h1:MSOAU6ukCpehJVHQDN1k9JgOZXZuqHD+2pT20M3JkIg= github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM= diff --git a/store/db/mysql/memo.go b/store/db/mysql/memo.go index eae473197..340d6af30 100644 --- a/store/db/mysql/memo.go +++ b/store/db/mysql/memo.go @@ -364,7 +364,7 @@ func (d *DB) RefreshMemoDerivedPayload(ctx context.Context, refresh *store.Refre return err } if _, err := d.db.ExecContext(ctx, - "UPDATE `memo` SET `payload` = JSON_SET(COALESCE(`payload`, JSON_OBJECT()), '$.tags', CAST(? AS JSON), '$.property', CAST(? AS JSON)) WHERE `id` = ? AND CAST(`content` AS BINARY) = CAST(? AS BINARY)", + "UPDATE `memo` SET `payload` = JSON_SET(COALESCE(`payload`, JSON_OBJECT()), '$.tags', JSON_EXTRACT(?, '$'), '$.property', JSON_EXTRACT(?, '$')) WHERE `id` = ? AND CAST(`content` AS BINARY) = CAST(? AS BINARY)", tags, property, refresh.ID, refresh.Content, ); err != nil { return errors.Wrap(err, "failed to refresh memo payload") diff --git a/store/db/mysql/space.go b/store/db/mysql/space.go index aba2202f7..d26393c94 100644 --- a/store/db/mysql/space.go +++ b/store/db/mysql/space.go @@ -90,7 +90,7 @@ func (d *DB) ListSpaces(ctx context.Context, find *store.FindSpace) ([]*store.Sp JOIN user active_user ON active_user.id = active_member.user_id AND active_user.row_status = 'NORMAL'` where = append(where, "viewer_member.user_id = ?", "viewer_member.status = 'ACTIVE'", "viewer_member.role IN ('ADMIN', 'USER')", "viewer_user.row_status = 'NORMAL'") args = append(args, *find.MemberUserID) - groupBy = " GROUP BY space.id, space.uid, space.title, space.description, viewer_member.role" + groupBy = " GROUP BY space.id, space.uid, space.title, space.description, space.payload, viewer_member.role" } query := "SELECT " + selectFields + " FROM space" + joins + " WHERE " + strings.Join(where, " AND ") + groupBy + " ORDER BY space.id DESC" query = appendMySQLLimit(query, find.Limit, find.Offset) diff --git a/store/migration/README.md b/store/migration/README.md index fa786d881..43b5c51fa 100644 --- a/store/migration/README.md +++ b/store/migration/README.md @@ -40,7 +40,8 @@ time on large attachment tables and block concurrent writes; SQLite also allows only one writer. The attachment lookup index in schema 26.11.1 uses this maintenance window rather than an online build. PostgreSQL builds it inside the migration transaction, so failure rolls back both the index and schema marker. -MySQL commits DDL implicitly; this migration accepts an already completed index -on retry, but rejects a conflicting index definition. If a migration reports a -conflict, inspect the named index before retrying; do not advance the schema -marker manually. +MySQL commits DDL implicitly, so an interrupted migration can leave schema +changes behind an old version marker. MySQL migrations use ordinary DDL; they +do not provide automatic recovery for that partial state. Inspect and reconcile +the database before retrying an interrupted migration. A completed migration +is skipped on restart because its schema version is already recorded. diff --git a/store/migration/mysql/26.11/00__add_attachment_memo_index.sql b/store/migration/mysql/26.11/00__add_attachment_memo_index.sql index 40bf4a248..9b263eec8 100644 --- a/store/migration/mysql/26.11/00__add_attachment_memo_index.sql +++ b/store/migration/mysql/26.11/00__add_attachment_memo_index.sql @@ -1,16 +1 @@ --- MySQL commits DDL before the schema marker. Accept a completed index on retry, --- but let CREATE fail for a conflicting definition instead of advancing the marker. -SET @memos_attachment_index_sql = IF( - (SELECT COUNT(*) = 2 - AND SUM(SEQ_IN_INDEX = 1 AND COLUMN_NAME = 'memo_id') = 1 - AND SUM(SEQ_IN_INDEX = 2 AND COLUMN_NAME = 'type') = 1 - AND SUM(SUB_PART IS NOT NULL OR NON_UNIQUE <> 1 OR INDEX_TYPE <> 'BTREE') = 0 - FROM information_schema.STATISTICS - WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attachment' - AND INDEX_NAME = 'idx_attachment_memo_id_type'), - 'SELECT 1', - 'CREATE INDEX `idx_attachment_memo_id_type` ON `attachment`(`memo_id`, `type`)' -); -PREPARE memos_attachment_index FROM @memos_attachment_index_sql; -EXECUTE memos_attachment_index; -DEALLOCATE PREPARE memos_attachment_index; +CREATE INDEX `idx_attachment_memo_id_type` ON `attachment`(`memo_id`, `type`); diff --git a/store/migration/postgres/26.11/00__add_attachment_memo_index.sql b/store/migration/postgres/26.11/00__add_attachment_memo_index.sql index aafe9ff50..91ced5c59 100644 --- a/store/migration/postgres/26.11/00__add_attachment_memo_index.sql +++ b/store/migration/postgres/26.11/00__add_attachment_memo_index.sql @@ -1,20 +1,2 @@ -- Index the bounded lookup key; attachment.type is unbounded text on PostgreSQL. -DO $$ -BEGIN - IF to_regclass('idx_attachment_memo_id') IS NULL THEN - CREATE INDEX idx_attachment_memo_id ON attachment(memo_id); - ELSIF NOT EXISTS ( - SELECT 1 FROM pg_index i - JOIN pg_class c ON c.oid = i.indexrelid - JOIN pg_am am ON am.oid = c.relam - WHERE i.indexrelid = to_regclass('idx_attachment_memo_id') - AND i.indrelid = 'attachment'::regclass - AND i.indisvalid AND i.indisready AND NOT i.indisunique - AND i.indpred IS NULL AND i.indexprs IS NULL - AND i.indnatts = 1 AND i.indnkeyatts = 1 - AND pg_get_indexdef(i.indexrelid, 1, true) = 'memo_id' - AND am.amname = 'btree' - ) THEN - RAISE EXCEPTION 'conflicting definition for idx_attachment_memo_id'; - END IF; -END $$; +CREATE INDEX IF NOT EXISTS idx_attachment_memo_id ON attachment(memo_id); diff --git a/store/migration/sqlite/26.11/00__add_attachment_memo_index.sql b/store/migration/sqlite/26.11/00__add_attachment_memo_index.sql index 63087739d..7124d27d7 100644 --- a/store/migration/sqlite/26.11/00__add_attachment_memo_index.sql +++ b/store/migration/sqlite/26.11/00__add_attachment_memo_index.sql @@ -1,17 +1 @@ CREATE INDEX IF NOT EXISTS idx_attachment_memo_id_type ON attachment(memo_id, type); - --- IF NOT EXISTS alone would silently accept an operator-created conflicting index. -CREATE TEMP TABLE attachment_index_check ( - valid INTEGER CONSTRAINT idx_attachment_memo_id_type_definition CHECK (valid = 1) -); -INSERT INTO attachment_index_check -SELECT EXISTS ( - SELECT 1 FROM pragma_index_list('attachment') - WHERE name = 'idx_attachment_memo_id_type' AND "unique" = 0 AND partial = 0 -) AND ( - SELECT COUNT(*) = 2 - AND SUM(seqno = 0 AND name = 'memo_id') = 1 - AND SUM(seqno = 1 AND name = 'type') = 1 - FROM pragma_index_info('idx_attachment_memo_id_type') -); -DROP TABLE attachment_index_check; diff --git a/store/test/README.md b/store/test/README.md index a19cf9d33..53afa6ed4 100644 --- a/store/test/README.md +++ b/store/test/README.md @@ -1,20 +1,30 @@ # Store tests -Run from the repository root. Docker must be running for MySQL, PostgreSQL, and released-binary upgrade tests. +Run from the repository root. Docker must be running for MySQL, MariaDB, PostgreSQL, and released-binary upgrade tests. ```bash -go test -v ./store/... # SQLite, MySQL, and PostgreSQL; TestContainers creates isolated databases -DRIVER=mysql go test -v ./store/test/... # One driver only +go test -v ./store/... # SQLite, MySQL, MariaDB, PostgreSQL; isolated databases per test +DRIVER=mysql go test -v ./store/test/... # MySQL 8.4 +DRIVER=mysql MYSQL_VARIANT=mariadb go test -v ./store/test/... # MariaDB 11.4.13 ``` +MariaDB uses the application's `mysql` driver, dialect, and migrations. `MYSQL_VARIANT` is test-only; accepted values are `mysql` +(the default) and `mariadb`. The aggregate run sets each target explicitly, regardless of an inherited variant. The MariaDB +container enables `ONLY_FULL_GROUP_BY` alongside its strict SQL mode so shared queries are checked under strict grouping rules. + +MariaDB 11.4.13 is the tested compatibility target. Deployments need InnoDB and utf8mb4. Other MariaDB versions, database conversion +between MySQL and MariaDB, and replication between them are outside this coverage. Text ordering and regex behavior can differ; +timestamp accessors use the database session timezone. Exact tag and content-guard comparisons retain explicit binary semantics. + ## Upgrade coverage ```bash DRIVER=sqlite go test -v -count=1 -timeout 30m -run 'TestMigration|TestUpgrade|TestFreshInstall' ./store/test/... ``` -Use `DRIVER=mysql` or `DRIVER=postgres` for the other drivers. The existing Upgrade Smoke workflow runs this selection for -all three drivers before release. +Use `DRIVER=mysql`, `DRIVER=mysql MYSQL_VARIANT=mariadb`, or `DRIVER=postgres` for the other database targets. Upgrade Smoke runs +this selection for all four targets before release. Both ordinary store tests and upgrade tests use the selected container, +including the private-network connection from released Memos binaries. - Before `v0.31.0`: `TestUpgradeThroughBaselineRelease` initializes a database with released `0.30.0`, verifies the current migrator refuses a direct upgrade without modifying data or settings, then upgrades through released `0.31.0` to current. diff --git a/store/test/containers.go b/store/test/containers.go index db26223d4..5a8916245 100644 --- a/store/test/containers.go +++ b/store/test/containers.go @@ -17,6 +17,7 @@ import ( "github.com/moby/moby/api/types/container" "github.com/pkg/errors" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/mariadb" "github.com/testcontainers/testcontainers-go/modules/mysql" "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/network" @@ -42,6 +43,7 @@ const ( var ( mysqlContainer atomic.Pointer[mysql.MySQLContainer] + mariadbContainer atomic.Pointer[mariadb.MariaDBContainer] postgresContainer atomic.Pointer[postgres.PostgresContainer] mysqlOnce sync.Once postgresOnce sync.Once @@ -88,8 +90,23 @@ func skipIfContainerProviderUnavailable(t *testing.T) { testcontainers.SkipIfProviderIsNotHealthy(t) } -// GetMySQLDSN starts a MySQL container (if not already running) and creates a fresh database for this test. +func getMySQLVariant() (string, error) { + switch variant := os.Getenv("MYSQL_VARIANT"); variant { + case "", "mysql": + return "mysql", nil + case "mariadb": + return variant, nil + default: + return "", errors.Errorf("unsupported MYSQL_VARIANT %q: use mysql or mariadb", variant) + } +} + +// GetMySQLDSN starts the selected MySQL-compatible server and creates a fresh database for this test. func GetMySQLDSN(t *testing.T) string { + variant, err := getMySQLVariant() + if err != nil { + t.Fatal(err) + } skipIfContainerProviderUnavailable(t) ctx := context.Background() @@ -100,34 +117,48 @@ func GetMySQLDSN(t *testing.T) string { t.Fatalf("failed to create test network: %v", err) } - container, err := mysql.Run(ctx, - "mysql:8.4", - mysql.WithDatabase("init_db"), - mysql.WithUsername("root"), - mysql.WithPassword(testPassword), - testcontainers.WithEnv(map[string]string{ - "MYSQL_ROOT_PASSWORD": testPassword, - }), + opts := []testcontainers.ContainerCustomizer{ testcontainers.WithWaitStrategy( wait.ForAll( wait.ForLog("ready for connections").WithOccurrence(2), wait.ForListeningPort("3306/tcp"), - ).WithDeadline(120*time.Second), + ).WithDeadline(120 * time.Second), ), network.WithNetwork([]string{mysqlNetworkAlias}, nw), - ) - if err != nil { - t.Fatalf("failed to start MySQL container: %v", err) } - mysqlContainer.Store(container) - - dsn, err := container.ConnectionString(ctx, "multiStatements=true") + var dsn string + if variant == "mariadb" { + t.Log("Starting MariaDB container: mariadb:11.4.13") + container, startErr := mariadb.Run(ctx, "mariadb:11.4.13", append(opts, + mariadb.WithDatabase("init_db"), + mariadb.WithUsername(testUser), + mariadb.WithPassword(testPassword), + testcontainers.WithCmd("--sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION"), + )...) + mariadbContainer.Store(container) + if startErr != nil { + t.Fatalf("failed to start MariaDB container: %v", startErr) + } + dsn, err = container.ConnectionString(ctx, "multiStatements=true") + } else { + t.Log("Starting MySQL container: mysql:8.4") + container, startErr := mysql.Run(ctx, "mysql:8.4", append(opts, + mysql.WithDatabase("init_db"), + mysql.WithUsername(testUser), + mysql.WithPassword(testPassword), + )...) + mysqlContainer.Store(container) + if startErr != nil { + t.Fatalf("failed to start MySQL container: %v", startErr) + } + dsn, err = container.ConnectionString(ctx, "multiStatements=true") + } if err != nil { - t.Fatalf("failed to get MySQL connection string: %v", err) + t.Fatalf("failed to get %s connection string: %v", variant, err) } if err := waitForDB("mysql", dsn, 30*time.Second); err != nil { - t.Fatalf("MySQL not ready for connections: %v", err) + t.Fatalf("%s not ready for connections: %v", variant, err) } mysqlBaseDSN.Store(dsn) @@ -135,7 +166,7 @@ func GetMySQLDSN(t *testing.T) string { dsn, ok := mysqlBaseDSN.Load().(string) if !ok || dsn == "" { - t.Fatal("MySQL container failed to start in a previous test") + t.Fatalf("%s container failed to start in a previous test", variant) } // Serialize database creation to avoid "table already exists" race conditions @@ -264,6 +295,9 @@ func TerminateContainers() { if container := mysqlContainer.Load(); container != nil { _ = container.Terminate(ctx) } + if container := mariadbContainer.Load(); container != nil { + _ = container.Terminate(ctx) + } if container := postgresContainer.Load(); container != nil { _ = container.Terminate(ctx) } diff --git a/store/test/main_test.go b/store/test/main_test.go index 276e618f5..c4cb553dc 100644 --- a/store/test/main_test.go +++ b/store/test/main_test.go @@ -10,6 +10,10 @@ import ( ) func TestMain(m *testing.M) { + if _, err := getMySQLVariant(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } // If DRIVER is set, run tests for that driver only if os.Getenv("DRIVER") != "" { defer TerminateContainers() @@ -17,7 +21,7 @@ func TestMain(m *testing.M) { return } - // No DRIVER set - run tests for all drivers sequentially + // No DRIVER set - run tests for all database targets sequentially runAllDrivers() } @@ -27,9 +31,10 @@ func runAllDrivers() { label string env []string }{ - {"sqlite", []string{"DRIVER=sqlite"}}, - {"mysql", []string{"DRIVER=mysql"}}, - {"postgres", []string{"DRIVER=postgres"}}, + {"sqlite", []string{"DRIVER=sqlite", "MYSQL_VARIANT="}}, + {"mysql", []string{"DRIVER=mysql", "MYSQL_VARIANT=mysql"}}, + {"mariadb", []string{"DRIVER=mysql", "MYSQL_VARIANT=mariadb"}}, + {"postgres", []string{"DRIVER=postgres", "MYSQL_VARIANT="}}, } _, currentFile, _, _ := runtime.Caller(0) projectRoot := filepath.Dir(filepath.Dir(filepath.Dir(currentFile))) diff --git a/store/test/memo_filter_test.go b/store/test/memo_filter_test.go index 9bc305771..f69771370 100644 --- a/store/test/memo_filter_test.go +++ b/store/test/memo_filter_test.go @@ -1,6 +1,7 @@ package test import ( + "fmt" "strings" "testing" "time" @@ -1210,11 +1211,24 @@ func TestMemoFilterNullPayload(t *testing.T) { tc := NewMemoFilterTestContext(t) defer tc.Close() - tc.CreateMemo(NewMemoBuilder("memo-null-payload", tc.User.ID).Content("Null payload")) - - // Test: has_link should not crash and return no results - memos := tc.ListWithFilter(`has_link`) - require.Len(t, memos, 0) + query := "UPDATE memo SET payload = ? WHERE id = ?" + if getDriverFromEnv() == "postgres" { + query = "UPDATE memo SET payload = $1 WHERE id = $2" + } + for i, payload := range []string{ + `{}`, + `{"property":{}}`, + `{"property":{"hasLink":false}}`, + `{"property":{"hasLink":null}}`, + } { + memo := tc.CreateMemo(NewMemoBuilder(fmt.Sprintf("memo-null-payload-%d", i), tc.User.ID).Content("Missing or false flag")) + _, err := tc.Store.GetDriver().GetDB().ExecContext(tc.Ctx, query, payload, memo.ID) + require.NoError(t, err) + } + require.Empty(t, tc.ListWithFilter(`has_link`)) + for _, expression := range []string{`!has_link`, `has_link == false`, `has_link != true`} { + require.Len(t, tc.ListWithFilter(expression), 4, expression) + } } func TestMemoFilterNoMatches(t *testing.T) { diff --git a/store/test/migrator_attachment_index_test.go b/store/test/migrator_attachment_index_test.go index 9e168e750..26bead625 100644 --- a/store/test/migrator_attachment_index_test.go +++ b/store/test/migrator_attachment_index_test.go @@ -20,61 +20,40 @@ func attachmentLookupIndex(driver string) string { return "idx_attachment_memo_id_type" } -// TestMigrationAttachmentIndex covers a new index, a retry after DDL committed, -// and a conflicting operator-created index without losing attachment data. +// TestMigrationAttachmentIndex preserves attachment data through the index upgrade and a restart. func TestMigrationAttachmentIndex(t *testing.T) { - for _, state := range []string{"absent", "present", "conflict"} { - t.Run(state, func(t *testing.T) { - t.Parallel() - ctx := context.Background() - ts := NewTestingStore(ctx, t) - t.Cleanup(func() { require.NoError(t, ts.Close()) }) - driver := getDriverFromEnv() - index := attachmentLookupIndex(driver) - user, err := createTestingHostUser(ctx, ts) - require.NoError(t, err) - memo, err := ts.CreateMemo(ctx, &store.Memo{UID: "index-upgrade", CreatorID: user.ID, Content: "preserved", Visibility: store.Private}) - require.NoError(t, err) - attachment, err := ts.CreateAttachment(ctx, &store.Attachment{ - UID: "index-attachment", CreatorID: user.ID, MemoID: &memo.ID, - Type: "image/png", Blob: []byte("preserved blob"), Size: 14, StorageType: storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED, - }) - require.NoError(t, err) + t.Parallel() + ctx := context.Background() + ts := NewTestingStore(ctx, t) + t.Cleanup(func() { require.NoError(t, ts.Close()) }) + driver := getDriverFromEnv() + user, err := createTestingHostUser(ctx, ts) + require.NoError(t, err) + memo, err := ts.CreateMemo(ctx, &store.Memo{UID: "index-upgrade", CreatorID: user.ID, Content: "preserved", Visibility: store.Private}) + require.NoError(t, err) + attachment, err := ts.CreateAttachment(ctx, &store.Attachment{ + UID: "index-attachment", CreatorID: user.ID, MemoID: &memo.ID, + Type: "image/png", Blob: []byte("preserved blob"), Size: 14, StorageType: storepb.AttachmentStorageType_ATTACHMENT_STORAGE_TYPE_UNSPECIFIED, + }) + require.NoError(t, err) - if state != "present" { - query := "DROP INDEX " + index - if driver == "mysql" { - query += " ON attachment" - } - _, err = ts.GetDriver().GetDB().ExecContext(ctx, query) - require.NoError(t, err) - } - if state == "conflict" { - _, err = ts.GetDriver().GetDB().ExecContext(ctx, "CREATE INDEX "+index+" ON attachment(creator_id)") - require.NoError(t, err) - } - setSchemaVersion(ctx, t, ts, "26.10.1") - err = ts.Migrate(ctx) - if state == "conflict" { - require.ErrorContains(t, err, index) - } else { - require.NoError(t, err) - requireLookupIndexes(ctx, t, ts, driver) - require.NoError(t, ts.Migrate(ctx), "a completed upgrade can restart") - } - setting, err := ts.GetUncachedInstanceSetting(ctx, &store.FindInstanceSetting{Name: "BASIC"}) - require.NoError(t, err) - expected := "26.11.1" - if state == "conflict" { - expected = "26.10.1" - } - require.Equal(t, expected, setting.GetBasicSetting().SchemaVersion) - stored, err := ts.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID, GetBlob: true}) - require.NoError(t, err) - require.Equal(t, []byte("preserved blob"), stored.Blob) - require.Equal(t, attachment.MemoID, stored.MemoID) - }) + query := "DROP INDEX " + attachmentLookupIndex(driver) + if driver == "mysql" { + query += " ON attachment" } + _, err = ts.GetDriver().GetDB().ExecContext(ctx, query) + require.NoError(t, err) + setSchemaVersion(ctx, t, ts, "26.10.1") + require.NoError(t, ts.Migrate(ctx)) + requireLookupIndexes(ctx, t, ts, driver) + require.NoError(t, ts.Migrate(ctx), "a completed upgrade can restart") + setting, err := ts.GetUncachedInstanceSetting(ctx, &store.FindInstanceSetting{Name: "BASIC"}) + require.NoError(t, err) + require.Equal(t, "26.11.1", setting.GetBasicSetting().SchemaVersion) + stored, err := ts.GetAttachment(ctx, &store.FindAttachment{ID: &attachment.ID, GetBlob: true}) + require.NoError(t, err) + require.Equal(t, []byte("preserved blob"), stored.Blob) + require.Equal(t, attachment.MemoID, stored.MemoID) } // TestMigrationAttachmentLongType preserves PostgreSQL's unbounded MIME field diff --git a/store/test/migrator_test.go b/store/test/migrator_test.go index 8bb763f74..f76543d3b 100644 --- a/store/test/migrator_test.go +++ b/store/test/migrator_test.go @@ -108,9 +108,9 @@ func requireLookupIndexes(ctx context.Context, t *testing.T, ts *store.Store, dr require.Equal(t, 1, count, "index %s must exist", indexName) } + columns := []string{"memo_id", "type"} index := attachmentLookupIndex(driver) var query string - columns := []string{"memo_id", "type"} switch driver { case "sqlite": query = `SELECT info.name FROM pragma_index_list('attachment') AS list @@ -119,7 +119,7 @@ func requireLookupIndexes(ctx context.Context, t *testing.T, ts *store.Store, dr case "mysql": query = `SELECT COLUMN_NAME FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'attachment' AND INDEX_NAME = ? - AND NON_UNIQUE = 1 AND SUB_PART IS NULL AND INDEX_TYPE = 'BTREE' AND IS_VISIBLE = 'YES' + AND NON_UNIQUE = 1 AND SUB_PART IS NULL AND INDEX_TYPE = 'BTREE' ORDER BY SEQ_IN_INDEX` case "postgres": columns = []string{"memo_id"} diff --git a/store/test/store.go b/store/test/store.go index 54c3c202d..d35af66b1 100644 --- a/store/test/store.go +++ b/store/test/store.go @@ -21,7 +21,7 @@ import ( // NewTestingStore creates a new testing store with a fresh database. // Each test gets its own isolated database: // - SQLite: new temp file per test -// - MySQL/PostgreSQL: new database per test in shared container +// - MySQL/MariaDB/PostgreSQL: new database per test in shared container func NewTestingStore(ctx context.Context, t *testing.T) *store.Store { driver := getDriverFromEnv() profile := getTestingProfileForDriver(t, driver) diff --git a/store/test/user_identity_test.go b/store/test/user_identity_test.go index 04a345c44..5b10084fc 100644 --- a/store/test/user_identity_test.go +++ b/store/test/user_identity_test.go @@ -91,20 +91,28 @@ func TestUserIdentityUniqueConflict(t *testing.T) { userB, err := createTestingUserWithRole(ctx, ts, "conflict_user", store.RoleUser) require.NoError(t, err) - _, err = ts.CreateUserIdentity(ctx, &store.UserIdentity{ - UserID: userA.ID, - Provider: "idp-A", - ExternUID: "sub-1", - }) - require.NoError(t, err) + for _, tc := range []struct{ provider, subject string }{ + {provider: "idp-A", subject: "sub-1"}, + {provider: "idp-key-marker", subject: "x for key 'username"}, + {provider: "idp-sqlite-marker", subject: "UNIQUE constraint failed: user.email"}, + {provider: "idp-postgres-marker", subject: `violates unique constraint "idx_user_email"`}, + } { + t.Run(tc.provider, func(t *testing.T) { + _, err := ts.CreateUserIdentity(ctx, &store.UserIdentity{ + UserID: userA.ID, + Provider: tc.provider, + ExternUID: tc.subject, + }) + require.NoError(t, err) - // Second insert with the same (provider, extern_uid) must fail regardless of user_id. - _, err = ts.CreateUserIdentity(ctx, &store.UserIdentity{ - UserID: userB.ID, - Provider: "idp-A", - ExternUID: "sub-1", - }) - require.Error(t, err) + _, err = ts.CreateUserIdentity(ctx, &store.UserIdentity{ + UserID: userB.ID, + Provider: tc.provider, + ExternUID: tc.subject, + }) + require.ErrorIs(t, err, store.ErrUserIdentityTaken) + }) + } } func TestUserIdentitySameExternUIDDifferentProviders(t *testing.T) { @@ -158,7 +166,7 @@ func TestUserIdentitySameUserSameProviderConflicts(t *testing.T) { Provider: "idp-A", ExternUID: "sub-2", }) - require.Error(t, err) + require.ErrorIs(t, err, store.ErrUserIdentityTaken) } func TestCreateUserWithIdentityIsAtomic(t *testing.T) { @@ -203,7 +211,7 @@ func TestCreateUserWithIdentityIsAtomic(t *testing.T) { Provider: "conflict-provider", ExternUID: "conflict-subject", }) - require.Error(t, err) + require.ErrorIs(t, err, store.ErrUserIdentityTaken) username := "rolled-back-user" user, err := ts.GetUser(ctx, &store.FindUser{Username: &username}) @@ -222,7 +230,7 @@ func TestCreateUserWithIdentityIsAtomic(t *testing.T) { Provider: "unused-provider", ExternUID: "unused-subject", }) - require.Error(t, err) + require.ErrorIs(t, err, store.ErrUsernameTaken) provider := "unused-provider" externUID := "unused-subject" diff --git a/store/user_unique.go b/store/user_unique.go index 32df7b759..e643b7c82 100644 --- a/store/user_unique.go +++ b/store/user_unique.go @@ -42,21 +42,27 @@ func classifyUserUniqueViolation(err error) error { return nil } message := err.Error() - for _, marker := range uniqueViolationMarkers { - _, after, ok := strings.Cut(message, marker) - if !ok { - continue - } - target := after - switch { - case strings.Contains(target, "user_identity"): - return errors.Wrap(ErrUserIdentityTaken, message) - case strings.Contains(target, "email"): - return errors.Wrap(ErrEmailTaken, message) - case strings.Contains(target, "username"): - return errors.Wrap(ErrUsernameTaken, message) + marker, markerIndex := "", -1 + // Duplicate values can contain markers from any database's error format. + for _, candidate := range uniqueViolationMarkers { + if index := strings.LastIndex(message, candidate); index > markerIndex { + marker, markerIndex = candidate, index } + } + if markerIndex < 0 { return nil } + target := message[markerIndex+len(marker):] + if marker == " for key '" { + target, _, _ = strings.Cut(target, "'") + } + switch { + case strings.Contains(target, "user_identity") || (marker == " for key '" && (target == "provider" || target == "user_id")): + return errors.Wrap(ErrUserIdentityTaken, message) + case strings.Contains(target, "email"): + return errors.Wrap(ErrEmailTaken, message) + case strings.Contains(target, "username"): + return errors.Wrap(ErrUsernameTaken, message) + } return nil } diff --git a/store/user_unique_test.go b/store/user_unique_test.go index 51481dc23..92b3b5240 100644 --- a/store/user_unique_test.go +++ b/store/user_unique_test.go @@ -19,6 +19,16 @@ func TestClassifyUserUniqueViolation(t *testing.T) { {name: "mysql email", message: "Error 1062 (23000): Duplicate entry 'a@b.c' for key 'user.idx_user_email'", want: ErrEmailTaken}, {name: "mysql username", message: "Error 1062 (23000): Duplicate entry 'alice' for key 'user.username'", want: ErrUsernameTaken}, {name: "mysql identity", message: "Error 1062 (23000): Duplicate entry 'idp-x' for key 'user_identity.provider'", want: ErrUserIdentityTaken}, + {name: "mysql identity value contains key marker", message: "Error 1062 (23000): Duplicate entry 'idp-x for key 'username' for key 'user_identity.provider'", want: ErrUserIdentityTaken}, + {name: "mysql username value contains key marker", message: "Error 1062 (23000): Duplicate entry 'x for key 'user_identity.provider' for key 'user.username'", want: ErrUsernameTaken}, + {name: "mariadb email", message: "Error 1062 (23000): Duplicate entry 'a@b.c' for key 'idx_user_email'", want: ErrEmailTaken}, + {name: "mariadb identity subject", message: "Error 1062 (23000): Duplicate entry 'idp-x' for key 'provider'", want: ErrUserIdentityTaken}, + {name: "mariadb identity value contains key marker", message: "Error 1062 (23000): Duplicate entry 'idp-x for key 'username' for key 'provider'", want: ErrUserIdentityTaken}, + {name: "mariadb identity value contains sqlite marker", message: "Error 1062 (23000): Duplicate entry 'idp-UNIQUE constraint failed: user.email' for key 'provider'", want: ErrUserIdentityTaken}, + {name: "mariadb identity value contains postgres marker", message: `Error 1062 (23000): Duplicate entry 'idp-violates unique constraint "idx_user_email"' for key 'provider'`, want: ErrUserIdentityTaken}, + {name: "mariadb identity user", message: "Error 1062 (23000): Duplicate entry '1-idp' for key 'user_id'", want: ErrUserIdentityTaken}, + {name: "identity key spelled as username", message: "Error 1062 (23000): Duplicate entry 'provider' for key 'username'", want: ErrUsernameTaken}, + {name: "unrelated qualified key", message: "Error 1062 (23000): Duplicate entry 'x' for key 'other.provider'", want: nil}, {name: "postgres email", message: `pq: duplicate key value violates unique constraint "idx_user_email"`, want: ErrEmailTaken}, {name: "postgres username", message: `pq: duplicate key value violates unique constraint "user_username_key"`, want: ErrUsernameTaken}, {name: "postgres identity", message: `pq: duplicate key value violates unique constraint "user_identity_provider_extern_uid_key"`, want: ErrUserIdentityTaken},