happyz
/
memos
mirror of https://github.com/usememos/memos.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: improve migration tests stability and maintainability 

- Fix linting issues and address testcontainers deprecation in store/test/containers.go
- Extract MemosStartupWaitStrategy for consistent container health checks
- Refactor migrator_test.go to use a consolidated testMigration helper, reducing duplication
- Add store/test/Dockerfile for optimized local test image builds
This commit is contained in:
Johnny 2026-01-07 22:40:45 +08:00
parent 013ea52519
commit f4cf2e9559
4 changed files with 375 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -9,6 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.18.16
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3
github.com/docker/docker v28.5.1+incompatible
github.com/go-sql-driver/mysql v1.9.3
github.com/google/cel-go v0.26.1
github.com/google/uuid v1.6.0
@ -50,7 +51,6 @@ require (
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/docker v28.5.1+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect

13
store/test/Dockerfile Normal file
View File

@ -0,0 +1,13 @@
FROM golang:1.25-alpine AS backend
WORKDIR /backend-build
COPY . .
RUN go build -o memos ./cmd/memos
FROM alpine:latest
WORKDIR /usr/local/memos
COPY --from=backend /backend-build/memos /usr/local/memos/
EXPOSE 5230
RUN mkdir -p /var/opt/memos
ENV MEMOS_MODE="prod"
ENV MEMOS_PORT="5230"
ENTRYPOINT ["./memos"]

242
store/test/containers.go
View File

@ -10,10 +10,12 @@ import (
"testing"
"time"
"github.com/docker/docker/api/types/container"
"github.com/pkg/errors"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mysql"
"github.com/testcontainers/testcontainers-go/modules/postgres"
"github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
// Database drivers for connection verification.
@ -24,9 +26,21 @@ import (
const (
testUser = "root"
testPassword = "test"
// Memos container settings for migration testing.
MemosDockerImage = "neosmemo/memos"
StableMemosVersion = "stable"
)
var (
// MemosStartupWaitStrategy defines the wait strategy for Memos container startup.
// It waits for the "started" log message (compatible with both old and new versions)
// and checks if port 5230 is listening.
MemosStartupWaitStrategy = wait.ForAll(
wait.ForLog("started"),
wait.ForListeningPort("5230/tcp"),
).WithDeadline(180 * time.Second)
mysqlContainer *mysql.MySQLContainer
postgresContainer *postgres.PostgresContainer
mysqlOnce sync.Once
@ -34,13 +48,36 @@ var (
mysqlBaseDSN string
postgresBaseDSN string
dbCounter atomic.Int64
// Network for container communication.
testDockerNetwork *testcontainers.DockerNetwork
testNetworkOnce sync.Once
)
// getTestNetwork creates or returns the shared Docker network for container communication.
func getTestNetwork(ctx context.Context) (*testcontainers.DockerNetwork, error) {
var networkErr error
testNetworkOnce.Do(func() {
nw, err := network.New(ctx, network.WithDriver("bridge"))
if err != nil {
networkErr = err
return
}
testDockerNetwork = nw
})
return testDockerNetwork, networkErr
}
// GetMySQLDSN starts a MySQL container (if not already running) and creates a fresh database for this test.
func GetMySQLDSN(t *testing.T) string {
ctx := context.Background()
mysqlOnce.Do(func() {
nw, err := getTestNetwork(ctx)
if err != nil {
t.Fatalf("failed to create test network: %v", err)
}
container, err := mysql.Run(ctx,
"mysql:8",
mysql.WithDatabase("init_db"),
@ -55,6 +92,7 @@ func GetMySQLDSN(t *testing.T) string {
wait.ForListeningPort("3306/tcp"),
).WithDeadline(120*time.Second),
),
network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start MySQL container: %v", err)
@ -130,6 +168,11 @@ func GetPostgresDSN(t *testing.T) string {
ctx := context.Background()
postgresOnce.Do(func() {
nw, err := getTestNetwork(ctx)
if err != nil {
t.Fatalf("failed to create test network: %v", err)
}
container, err := postgres.Run(ctx,
"postgres:18",
postgres.WithDatabase("init_db"),
@ -141,6 +184,7 @@ func GetPostgresDSN(t *testing.T) string {
wait.ForListeningPort("5432/tcp"),
).WithDeadline(120*time.Second),
),
network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start PostgreSQL container: %v", err)
@ -179,7 +223,106 @@ func GetPostgresDSN(t *testing.T) string {
return strings.Replace(postgresBaseDSN, "/init_db?", "/"+dbName+"?", 1)
}
// TerminateContainers cleans up all running containers.
// GetDedicatedMySQLDSN starts a dedicated MySQL container for migration testing.
// This is needed because older Memos versions have bugs when connecting to a MySQL
// server that has other initialized databases (they incorrectly query migration_history
// on a fresh database without checking if the DB is initialized).
// Returns: DSN for host access, container hostname for internal network access, cleanup function.
func GetDedicatedMySQLDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {
ctx := context.Background()
nw, err := getTestNetwork(ctx)
if err != nil {
t.Fatalf("failed to create test network: %v", err)
}
container, err := mysql.Run(ctx,
"mysql:8",
mysql.WithDatabase("memos"),
mysql.WithUsername("root"),
mysql.WithPassword(testPassword),
testcontainers.WithEnv(map[string]string{
"MYSQL_ROOT_PASSWORD": testPassword,
}),
testcontainers.WithWaitStrategy(
wait.ForAll(
wait.ForLog("ready for connections").WithOccurrence(2),
wait.ForListeningPort("3306/tcp"),
).WithDeadline(120*time.Second),
),
network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start dedicated MySQL container: %v", err)
}
hostDSN, err := container.ConnectionString(ctx, "multiStatements=true")
if err != nil {
container.Terminate(ctx)
t.Fatalf("failed to get MySQL connection string: %v", err)
}
if err := waitForDB("mysql", hostDSN, 30*time.Second); err != nil {
container.Terminate(ctx)
t.Fatalf("MySQL not ready for connections: %v", err)
}
name, _ := container.Name(ctx)
host := strings.TrimPrefix(name, "/")
return hostDSN, host, func() {
container.Terminate(ctx)
}
}
// GetDedicatedPostgresDSN starts a dedicated PostgreSQL container for migration testing.
// This is needed for isolation when testing migrations with older Memos versions.
// Returns: DSN for host access, container hostname for internal network access, cleanup function.
func GetDedicatedPostgresDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {
ctx := context.Background()
nw, err := getTestNetwork(ctx)
if err != nil {
t.Fatalf("failed to create test network: %v", err)
}
container, err := postgres.Run(ctx,
"postgres:18",
postgres.WithDatabase("memos"),
postgres.WithUsername(testUser),
postgres.WithPassword(testPassword),
testcontainers.WithWaitStrategy(
wait.ForAll(
wait.ForLog("database system is ready to accept connections").WithOccurrence(2),
wait.ForListeningPort("5432/tcp"),
).WithDeadline(120*time.Second),
),
network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start dedicated PostgreSQL container: %v", err)
}
hostDSN, err := container.ConnectionString(ctx, "sslmode=disable")
if err != nil {
container.Terminate(ctx)
t.Fatalf("failed to get PostgreSQL connection string: %v", err)
}
if err := waitForDB("postgres", hostDSN, 30*time.Second); err != nil {
container.Terminate(ctx)
t.Fatalf("PostgreSQL not ready for connections: %v", err)
}
name, _ := container.Name(ctx)
host := strings.TrimPrefix(name, "/")
return hostDSN, host, func() {
container.Terminate(ctx)
}
}
// TerminateContainers cleans up all running containers and network.
// This is typically called from TestMain.
func TerminateContainers() {
ctx := context.Background()
@ -189,4 +332,101 @@ func TerminateContainers() {
if postgresContainer != nil {
_ = postgresContainer.Terminate(ctx)
}
if testDockerNetwork != nil {
_ = testDockerNetwork.Remove(ctx)
}
}
// GetMySQLContainerHost returns the MySQL container hostname for use within the Docker network.
func GetMySQLContainerHost() string {
if mysqlContainer == nil {
return ""
}
name, _ := mysqlContainer.Name(context.Background())
// Remove leading slash from container name
return strings.TrimPrefix(name, "/")
}
// GetPostgresContainerHost returns the PostgreSQL container hostname for use within the Docker network.
func GetPostgresContainerHost() string {
if postgresContainer == nil {
return ""
}
name, _ := postgresContainer.Name(context.Background())
return strings.TrimPrefix(name, "/")
}
// MemosContainerConfig holds configuration for starting a Memos container.
type MemosContainerConfig struct {
Version string // Memos version tag (e.g., "0.25")
Driver string // Database driver: sqlite, mysql, postgres
DSN string // Database DSN (for mysql/postgres)
DataDir string // Host directory to mount for SQLite data
}
// StartMemosContainer starts a Memos container for migration testing.
// For SQLite, it mounts the dataDir to /var/opt/memos.
// For MySQL/PostgreSQL, it connects to the provided DSN via the test network.
// If Version is "local", builds the image from the local Dockerfile.
func StartMemosContainer(ctx context.Context, cfg MemosContainerConfig) (testcontainers.Container, error) {
env := map[string]string{
"MEMOS_MODE": "prod",
}
var mounts []testcontainers.ContainerMount
var opts []testcontainers.ContainerCustomizer
switch cfg.Driver {
case "sqlite":
env["MEMOS_DRIVER"] = "sqlite"
opts = append(opts, testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) {
hc.Binds = append(hc.Binds, fmt.Sprintf("%s:%s", cfg.DataDir, "/var/opt/memos"))
}))
case "mysql":
env["MEMOS_DRIVER"] = "mysql"
env["MEMOS_DSN"] = cfg.DSN
opts = append(opts, network.WithNetwork(nil, testDockerNetwork))
case "postgres":
env["MEMOS_DRIVER"] = "postgres"
env["MEMOS_DSN"] = cfg.DSN
opts = append(opts, network.WithNetwork(nil, testDockerNetwork))
default:
return nil, errors.Errorf("unsupported driver: %s", cfg.Driver)
}
req := testcontainers.ContainerRequest{
Env: env,
Mounts: testcontainers.Mounts(mounts...),
ExposedPorts: []string{"5230/tcp"},
WaitingFor: MemosStartupWaitStrategy,
}
// Use local Dockerfile build or remote image
if cfg.Version == "local" {
req.FromDockerfile = testcontainers.FromDockerfile{
Context: "../../",
Dockerfile: "store/test/Dockerfile", // Simple Dockerfile without BuildKit requirements
}
} else {
req.Image = fmt.Sprintf("%s:%s", MemosDockerImage, cfg.Version)
}
genericReq := testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
}
// Apply network options
for _, opt := range opts {
if err := opt.Customize(&genericReq); err != nil {
return nil, errors.Wrap(err, "failed to apply container option")
}
}
container, err := testcontainers.GenericContainer(ctx, genericReq)
if err != nil {
return nil, errors.Wrap(err, "failed to start memos container")
}
return container, nil
}

120
store/test/migrator_test.go
View File

@ -2,7 +2,10 @@ package test
import (
"context"
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
@ -15,3 +18,120 @@ func TestGetCurrentSchemaVersion(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "0.26.1", currentSchemaVersion)
}
// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database.
// This is essentially what NewTestingStore already does, but we make it explicit.
func TestFreshInstall(t *testing.T) {
ctx := context.Background()
// NewTestingStore creates a fresh database and runs Migrate()
// which applies LATEST.sql for uninitialized databases
ts := NewTestingStore(ctx, t)
// Verify migration completed successfully
currentSchemaVersion, err := ts.GetCurrentSchemaVersion()
require.NoError(t, err)
require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install")
// Verify we can read instance settings (basic sanity check)
instanceSetting, err := ts.GetInstanceBasicSetting(ctx)
require.NoError(t, err)
require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion)
}
// testMigration is a helper function that orchestrates the migration test flow.
// It starts the stable version, waits for initialization, and then starts the local version.
func testMigration(t *testing.T, driver string, prepareFunc func() (MemosContainerConfig, func())) {
if getDriverFromEnv() != driver {
t.Skipf("skipping %s migration test for non-%s driver", driver, driver)
}
ctx := context.Background()
// Prepare resources (temp dir or dedicated container)
cfg, cleanup := prepareFunc()
if cleanup != nil {
defer cleanup()
}
// 1. Start Old Memos container (Stable)
cfg.Version = StableMemosVersion
t.Logf("Starting Memos %s container to initialize %s database...", cfg.Version, driver)
oldContainer, err := StartMemosContainer(ctx, cfg)
require.NoError(t, err, "failed to start old memos container")
// Wait for database to be fully initialized
time.Sleep(5 * time.Second)
// Stop the old container
err = oldContainer.Terminate(ctx)
require.NoError(t, err, "failed to stop old memos container")
t.Log("Old Memos container stopped, starting new container with local build...")
// 2. Start New Memos container (Local)
cfg.Version = "local" // Triggers local build in StartMemosContainer
newContainer, err := StartMemosContainer(ctx, cfg)
require.NoError(t, err, "failed to start new memos container - migration may have failed")
defer newContainer.Terminate(ctx)
t.Logf("Migration successful: %s -> local build", StableMemosVersion)
}
// TestMigrationFromPreviousVersion_SQLite verifies that migrating from the previous
// Memos version to the current version works correctly for SQLite.
func TestMigrationFromPreviousVersion_SQLite(t *testing.T) {
testMigration(t, "sqlite", func() (MemosContainerConfig, func()) {
// Create a temp directory for SQLite data that persists across container restarts
dataDir := t.TempDir()
return MemosContainerConfig{
Driver: "sqlite",
DataDir: dataDir,
}, nil
})
}
// TestMigrationFromPreviousVersion_MySQL verifies that migrating from the previous
// Memos version to the current version works correctly for MySQL.
func TestMigrationFromPreviousVersion_MySQL(t *testing.T) {
testMigration(t, "mysql", func() (MemosContainerConfig, func()) {
// For migration testing, we need a dedicated MySQL container
dsn, containerHost, cleanup := GetDedicatedMySQLDSN(t)
// Extract database name from DSN
parts := strings.Split(dsn, "/")
dbNameWithParams := parts[len(parts)-1]
dbName := strings.Split(dbNameWithParams, "?")[0]
// Container DSN uses internal network hostname
containerDSN := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", testUser, testPassword, containerHost, dbName)
return MemosContainerConfig{
Driver: "mysql",
DSN: containerDSN,
}, cleanup
})
}
// TestMigrationFromPreviousVersion_Postgres verifies that migrating from the previous
// Memos version to the current version works correctly for PostgreSQL.
func TestMigrationFromPreviousVersion_Postgres(t *testing.T) {
testMigration(t, "postgres", func() (MemosContainerConfig, func()) {
// For migration testing, we need a dedicated PostgreSQL container
dsn, containerHost, cleanup := GetDedicatedPostgresDSN(t)
// Extract database name from DSN
parts := strings.Split(dsn, "/")
dbNameWithParams := parts[len(parts)-1]
dbName := strings.Split(dbNameWithParams, "?")[0]
// Container DSN uses internal network hostname
containerDSN := fmt.Sprintf("postgres://%s:%s@%s:5432/%s?sslmode=disable",
testUser, testPassword, containerHost, dbName)
return MemosContainerConfig{
Driver: "postgres",
DSN: containerDSN,
}, cleanup
})
}