diff --git a/go.mod b/go.mod index cd7f5b5e1..fc85404e7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.18.16 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4 github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 + github.com/docker/docker v28.5.1+incompatible github.com/go-sql-driver/mysql v1.9.3 github.com/google/cel-go v0.26.1 github.com/google/uuid v1.6.0 @@ -50,7 +51,6 @@ require ( github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.5.1+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/store/test/Dockerfile b/store/test/Dockerfile new file mode 100644 index 000000000..6b25a7079 --- /dev/null +++ b/store/test/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.25-alpine AS backend +WORKDIR /backend-build +COPY . . +RUN go build -o memos ./cmd/memos + +FROM alpine:latest +WORKDIR /usr/local/memos +COPY --from=backend /backend-build/memos /usr/local/memos/ +EXPOSE 5230 +RUN mkdir -p /var/opt/memos +ENV MEMOS_MODE="prod" +ENV MEMOS_PORT="5230" +ENTRYPOINT ["./memos"] diff --git a/store/test/containers.go b/store/test/containers.go index bd65ea40e..c5e139306 100644 --- a/store/test/containers.go +++ b/store/test/containers.go @@ -10,10 +10,12 @@ import ( "testing" "time" + "github.com/docker/docker/api/types/container" "github.com/pkg/errors" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/mysql" "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" // Database drivers for connection verification. @@ -24,9 +26,21 @@ import ( const ( testUser = "root" testPassword = "test" + + // Memos container settings for migration testing. + MemosDockerImage = "neosmemo/memos" + StableMemosVersion = "stable" ) var ( + // MemosStartupWaitStrategy defines the wait strategy for Memos container startup. + // It waits for the "started" log message (compatible with both old and new versions) + // and checks if port 5230 is listening. + MemosStartupWaitStrategy = wait.ForAll( + wait.ForLog("started"), + wait.ForListeningPort("5230/tcp"), + ).WithDeadline(180 * time.Second) + mysqlContainer *mysql.MySQLContainer postgresContainer *postgres.PostgresContainer mysqlOnce sync.Once @@ -34,13 +48,36 @@ var ( mysqlBaseDSN string postgresBaseDSN string dbCounter atomic.Int64 + + // Network for container communication. + testDockerNetwork *testcontainers.DockerNetwork + testNetworkOnce sync.Once ) +// getTestNetwork creates or returns the shared Docker network for container communication. +func getTestNetwork(ctx context.Context) (*testcontainers.DockerNetwork, error) { + var networkErr error + testNetworkOnce.Do(func() { + nw, err := network.New(ctx, network.WithDriver("bridge")) + if err != nil { + networkErr = err + return + } + testDockerNetwork = nw + }) + return testDockerNetwork, networkErr +} + // GetMySQLDSN starts a MySQL container (if not already running) and creates a fresh database for this test. func GetMySQLDSN(t *testing.T) string { ctx := context.Background() mysqlOnce.Do(func() { + nw, err := getTestNetwork(ctx) + if err != nil { + t.Fatalf("failed to create test network: %v", err) + } + container, err := mysql.Run(ctx, "mysql:8", mysql.WithDatabase("init_db"), @@ -55,6 +92,7 @@ func GetMySQLDSN(t *testing.T) string { wait.ForListeningPort("3306/tcp"), ).WithDeadline(120*time.Second), ), + network.WithNetwork(nil, nw), ) if err != nil { t.Fatalf("failed to start MySQL container: %v", err) @@ -130,6 +168,11 @@ func GetPostgresDSN(t *testing.T) string { ctx := context.Background() postgresOnce.Do(func() { + nw, err := getTestNetwork(ctx) + if err != nil { + t.Fatalf("failed to create test network: %v", err) + } + container, err := postgres.Run(ctx, "postgres:18", postgres.WithDatabase("init_db"), @@ -141,6 +184,7 @@ func GetPostgresDSN(t *testing.T) string { wait.ForListeningPort("5432/tcp"), ).WithDeadline(120*time.Second), ), + network.WithNetwork(nil, nw), ) if err != nil { t.Fatalf("failed to start PostgreSQL container: %v", err) @@ -179,7 +223,106 @@ func GetPostgresDSN(t *testing.T) string { return strings.Replace(postgresBaseDSN, "/init_db?", "/"+dbName+"?", 1) } -// TerminateContainers cleans up all running containers. +// GetDedicatedMySQLDSN starts a dedicated MySQL container for migration testing. +// This is needed because older Memos versions have bugs when connecting to a MySQL +// server that has other initialized databases (they incorrectly query migration_history +// on a fresh database without checking if the DB is initialized). +// Returns: DSN for host access, container hostname for internal network access, cleanup function. +func GetDedicatedMySQLDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) { + ctx := context.Background() + + nw, err := getTestNetwork(ctx) + if err != nil { + t.Fatalf("failed to create test network: %v", err) + } + + container, err := mysql.Run(ctx, + "mysql:8", + mysql.WithDatabase("memos"), + mysql.WithUsername("root"), + mysql.WithPassword(testPassword), + testcontainers.WithEnv(map[string]string{ + "MYSQL_ROOT_PASSWORD": testPassword, + }), + testcontainers.WithWaitStrategy( + wait.ForAll( + wait.ForLog("ready for connections").WithOccurrence(2), + wait.ForListeningPort("3306/tcp"), + ).WithDeadline(120*time.Second), + ), + network.WithNetwork(nil, nw), + ) + if err != nil { + t.Fatalf("failed to start dedicated MySQL container: %v", err) + } + + hostDSN, err := container.ConnectionString(ctx, "multiStatements=true") + if err != nil { + container.Terminate(ctx) + t.Fatalf("failed to get MySQL connection string: %v", err) + } + + if err := waitForDB("mysql", hostDSN, 30*time.Second); err != nil { + container.Terminate(ctx) + t.Fatalf("MySQL not ready for connections: %v", err) + } + + name, _ := container.Name(ctx) + host := strings.TrimPrefix(name, "/") + + return hostDSN, host, func() { + container.Terminate(ctx) + } +} + +// GetDedicatedPostgresDSN starts a dedicated PostgreSQL container for migration testing. +// This is needed for isolation when testing migrations with older Memos versions. +// Returns: DSN for host access, container hostname for internal network access, cleanup function. +func GetDedicatedPostgresDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) { + ctx := context.Background() + + nw, err := getTestNetwork(ctx) + if err != nil { + t.Fatalf("failed to create test network: %v", err) + } + + container, err := postgres.Run(ctx, + "postgres:18", + postgres.WithDatabase("memos"), + postgres.WithUsername(testUser), + postgres.WithPassword(testPassword), + testcontainers.WithWaitStrategy( + wait.ForAll( + wait.ForLog("database system is ready to accept connections").WithOccurrence(2), + wait.ForListeningPort("5432/tcp"), + ).WithDeadline(120*time.Second), + ), + network.WithNetwork(nil, nw), + ) + if err != nil { + t.Fatalf("failed to start dedicated PostgreSQL container: %v", err) + } + + hostDSN, err := container.ConnectionString(ctx, "sslmode=disable") + if err != nil { + container.Terminate(ctx) + t.Fatalf("failed to get PostgreSQL connection string: %v", err) + } + + if err := waitForDB("postgres", hostDSN, 30*time.Second); err != nil { + container.Terminate(ctx) + t.Fatalf("PostgreSQL not ready for connections: %v", err) + } + + name, _ := container.Name(ctx) + host := strings.TrimPrefix(name, "/") + + return hostDSN, host, func() { + container.Terminate(ctx) + } +} + +// TerminateContainers cleans up all running containers and network. // This is typically called from TestMain. func TerminateContainers() { ctx := context.Background() @@ -189,4 +332,101 @@ func TerminateContainers() { if postgresContainer != nil { _ = postgresContainer.Terminate(ctx) } + if testDockerNetwork != nil { + _ = testDockerNetwork.Remove(ctx) + } +} + +// GetMySQLContainerHost returns the MySQL container hostname for use within the Docker network. +func GetMySQLContainerHost() string { + if mysqlContainer == nil { + return "" + } + name, _ := mysqlContainer.Name(context.Background()) + // Remove leading slash from container name + return strings.TrimPrefix(name, "/") +} + +// GetPostgresContainerHost returns the PostgreSQL container hostname for use within the Docker network. +func GetPostgresContainerHost() string { + if postgresContainer == nil { + return "" + } + name, _ := postgresContainer.Name(context.Background()) + return strings.TrimPrefix(name, "/") +} + +// MemosContainerConfig holds configuration for starting a Memos container. +type MemosContainerConfig struct { + Version string // Memos version tag (e.g., "0.25") + Driver string // Database driver: sqlite, mysql, postgres + DSN string // Database DSN (for mysql/postgres) + DataDir string // Host directory to mount for SQLite data +} + +// StartMemosContainer starts a Memos container for migration testing. +// For SQLite, it mounts the dataDir to /var/opt/memos. +// For MySQL/PostgreSQL, it connects to the provided DSN via the test network. +// If Version is "local", builds the image from the local Dockerfile. +func StartMemosContainer(ctx context.Context, cfg MemosContainerConfig) (testcontainers.Container, error) { + env := map[string]string{ + "MEMOS_MODE": "prod", + } + + var mounts []testcontainers.ContainerMount + var opts []testcontainers.ContainerCustomizer + + switch cfg.Driver { + case "sqlite": + env["MEMOS_DRIVER"] = "sqlite" + opts = append(opts, testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, fmt.Sprintf("%s:%s", cfg.DataDir, "/var/opt/memos")) + })) + case "mysql": + env["MEMOS_DRIVER"] = "mysql" + env["MEMOS_DSN"] = cfg.DSN + opts = append(opts, network.WithNetwork(nil, testDockerNetwork)) + case "postgres": + env["MEMOS_DRIVER"] = "postgres" + env["MEMOS_DSN"] = cfg.DSN + opts = append(opts, network.WithNetwork(nil, testDockerNetwork)) + default: + return nil, errors.Errorf("unsupported driver: %s", cfg.Driver) + } + + req := testcontainers.ContainerRequest{ + Env: env, + Mounts: testcontainers.Mounts(mounts...), + ExposedPorts: []string{"5230/tcp"}, + WaitingFor: MemosStartupWaitStrategy, + } + + // Use local Dockerfile build or remote image + if cfg.Version == "local" { + req.FromDockerfile = testcontainers.FromDockerfile{ + Context: "../../", + Dockerfile: "store/test/Dockerfile", // Simple Dockerfile without BuildKit requirements + } + } else { + req.Image = fmt.Sprintf("%s:%s", MemosDockerImage, cfg.Version) + } + + genericReq := testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + } + + // Apply network options + for _, opt := range opts { + if err := opt.Customize(&genericReq); err != nil { + return nil, errors.Wrap(err, "failed to apply container option") + } + } + + container, err := testcontainers.GenericContainer(ctx, genericReq) + if err != nil { + return nil, errors.Wrap(err, "failed to start memos container") + } + + return container, nil } diff --git a/store/test/migrator_test.go b/store/test/migrator_test.go index 946d9ce10..87bcb18ee 100644 --- a/store/test/migrator_test.go +++ b/store/test/migrator_test.go @@ -2,7 +2,10 @@ package test import ( "context" + "fmt" + "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -15,3 +18,120 @@ func TestGetCurrentSchemaVersion(t *testing.T) { require.NoError(t, err) require.Equal(t, "0.26.1", currentSchemaVersion) } + +// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database. +// This is essentially what NewTestingStore already does, but we make it explicit. +func TestFreshInstall(t *testing.T) { + ctx := context.Background() + + // NewTestingStore creates a fresh database and runs Migrate() + // which applies LATEST.sql for uninitialized databases + ts := NewTestingStore(ctx, t) + + // Verify migration completed successfully + currentSchemaVersion, err := ts.GetCurrentSchemaVersion() + require.NoError(t, err) + require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install") + + // Verify we can read instance settings (basic sanity check) + instanceSetting, err := ts.GetInstanceBasicSetting(ctx) + require.NoError(t, err) + require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion) +} + +// testMigration is a helper function that orchestrates the migration test flow. +// It starts the stable version, waits for initialization, and then starts the local version. +func testMigration(t *testing.T, driver string, prepareFunc func() (MemosContainerConfig, func())) { + if getDriverFromEnv() != driver { + t.Skipf("skipping %s migration test for non-%s driver", driver, driver) + } + + ctx := context.Background() + + // Prepare resources (temp dir or dedicated container) + cfg, cleanup := prepareFunc() + if cleanup != nil { + defer cleanup() + } + + // 1. Start Old Memos container (Stable) + cfg.Version = StableMemosVersion + t.Logf("Starting Memos %s container to initialize %s database...", cfg.Version, driver) + oldContainer, err := StartMemosContainer(ctx, cfg) + require.NoError(t, err, "failed to start old memos container") + + // Wait for database to be fully initialized + time.Sleep(5 * time.Second) + + // Stop the old container + err = oldContainer.Terminate(ctx) + require.NoError(t, err, "failed to stop old memos container") + + t.Log("Old Memos container stopped, starting new container with local build...") + + // 2. Start New Memos container (Local) + cfg.Version = "local" // Triggers local build in StartMemosContainer + newContainer, err := StartMemosContainer(ctx, cfg) + require.NoError(t, err, "failed to start new memos container - migration may have failed") + defer newContainer.Terminate(ctx) + + t.Logf("Migration successful: %s -> local build", StableMemosVersion) +} + +// TestMigrationFromPreviousVersion_SQLite verifies that migrating from the previous +// Memos version to the current version works correctly for SQLite. +func TestMigrationFromPreviousVersion_SQLite(t *testing.T) { + testMigration(t, "sqlite", func() (MemosContainerConfig, func()) { + // Create a temp directory for SQLite data that persists across container restarts + dataDir := t.TempDir() + return MemosContainerConfig{ + Driver: "sqlite", + DataDir: dataDir, + }, nil + }) +} + +// TestMigrationFromPreviousVersion_MySQL verifies that migrating from the previous +// Memos version to the current version works correctly for MySQL. +func TestMigrationFromPreviousVersion_MySQL(t *testing.T) { + testMigration(t, "mysql", func() (MemosContainerConfig, func()) { + // For migration testing, we need a dedicated MySQL container + dsn, containerHost, cleanup := GetDedicatedMySQLDSN(t) + + // Extract database name from DSN + parts := strings.Split(dsn, "/") + dbNameWithParams := parts[len(parts)-1] + dbName := strings.Split(dbNameWithParams, "?")[0] + + // Container DSN uses internal network hostname + containerDSN := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", testUser, testPassword, containerHost, dbName) + + return MemosContainerConfig{ + Driver: "mysql", + DSN: containerDSN, + }, cleanup + }) +} + +// TestMigrationFromPreviousVersion_Postgres verifies that migrating from the previous +// Memos version to the current version works correctly for PostgreSQL. +func TestMigrationFromPreviousVersion_Postgres(t *testing.T) { + testMigration(t, "postgres", func() (MemosContainerConfig, func()) { + // For migration testing, we need a dedicated PostgreSQL container + dsn, containerHost, cleanup := GetDedicatedPostgresDSN(t) + + // Extract database name from DSN + parts := strings.Split(dsn, "/") + dbNameWithParams := parts[len(parts)-1] + dbName := strings.Split(dbNameWithParams, "?")[0] + + // Container DSN uses internal network hostname + containerDSN := fmt.Sprintf("postgres://%s:%s@%s:5432/%s?sslmode=disable", + testUser, testPassword, containerHost, dbName) + + return MemosContainerConfig{ + Driver: "postgres", + DSN: containerDSN, + }, cleanup + }) +}