fix: add Unicode case-insensitive search for SQLite (#5559)

Add custom memos_unicode_lower() SQLite function to enable proper
case-insensitive text search for non-English languages (Cyrillic,
Greek, CJK, etc.).

Previously, SQLite's LOWER() only worked for ASCII characters due to
modernc.org/sqlite lacking ICU extension. This caused searches for
non-English text to be case-sensitive (e.g., searching 'блины' wouldn't
find 'Блины').

Changes:
- Add store/db/sqlite/functions.go with Unicode case folding function
- Register custom function using golang.org/x/text/cases.Fold()
- Update filter renderer to use custom function for SQLite dialect
- Add test for Unicode case-insensitive search
- Make golang.org/x/text a direct dependency

Fixes #5559
This commit is contained in:
Steven 2026-02-02 21:10:07 +08:00
parent c14843fa62
commit 8770b186e4
5 changed files with 67 additions and 3 deletions

2
go.mod
View File

@ -131,7 +131,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
golang.org/x/sys v0.37.0 // indirect
golang.org/x/text v0.30.0 // indirect
golang.org/x/text v0.30.0
golang.org/x/time v0.12.0 // indirect
google.golang.org/protobuf v1.36.9
gopkg.in/yaml.v3 v3.0.1 // indirect

View File

@ -454,6 +454,11 @@ func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResul
column := field.columnExpr(r.dialect)
arg := fmt.Sprintf("%%%s%%", cond.Value)
switch r.dialect {
case DialectSQLite:
// Use custom Unicode-aware case folding function for case-insensitive comparison.
// This overcomes SQLite's ASCII-only LOWER() limitation.
sql := fmt.Sprintf("memos_unicode_lower(%s) LIKE memos_unicode_lower(%s)", column, r.addArg(arg))
return renderResult{sql: sql}, nil
case DialectPostgres:
sql := fmt.Sprintf("%s ILIKE %s", column, r.addArg(arg))
return renderResult{sql: sql}, nil

View File

@ -0,0 +1,44 @@
// Package sqlite provides SQLite driver implementation with custom functions.
// Custom functions are registered globally on first use to extend SQLite's
// limited ASCII-only text operations with proper Unicode support.
package sqlite
import (
"database/sql/driver"
"sync"
"golang.org/x/text/cases"
msqlite "modernc.org/sqlite"
)
var (
registerUnicodeLowerOnce sync.Once
registerUnicodeLowerErr error
// unicodeFold provides Unicode case folding for case-insensitive comparisons.
// It's safe to use concurrently and reused across all function calls.
unicodeFold = cases.Fold()
)
// ensureUnicodeLowerRegistered registers the memos_unicode_lower custom function
// with SQLite. This function provides proper Unicode case folding for case-insensitive
// text comparisons, overcoming modernc.org/sqlite's lack of ICU extension.
//
// The function is registered once globally and is safe to call multiple times.
func ensureUnicodeLowerRegistered() error {
registerUnicodeLowerOnce.Do(func() {
registerUnicodeLowerErr = msqlite.RegisterScalarFunction("memos_unicode_lower", 1, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
if len(args) == 0 || args[0] == nil {
return nil, nil
}
switch v := args[0].(type) {
case string:
return unicodeFold.String(v), nil
case []byte:
return unicodeFold.String(string(v)), nil
default:
return v, nil
}
})
})
return registerUnicodeLowerErr
}

View File

@ -6,8 +6,8 @@ import (
"github.com/pkg/errors"
// Import the SQLite driver.
_ "modernc.org/sqlite"
// Note: modernc.org/sqlite driver is imported in functions.go where
// RegisterScalarFunction is used. No blank import needed here.
"github.com/usememos/memos/internal/profile"
"github.com/usememos/memos/store"
@ -27,6 +27,10 @@ func NewDB(profile *profile.Profile) (store.Driver, error) {
return nil, errors.New("dsn required")
}
if err := ensureUnicodeLowerRegistered(); err != nil {
return nil, errors.Wrap(err, "failed to register sqlite unicode lower function")
}
// Connect to the database with some sane settings:
// - No shared-cache: it's obsolete; WAL journal mode is a better solution.
// - No foreign key constraints: it's currently disabled by default, but it's a

View File

@ -61,6 +61,17 @@ func TestMemoFilterContentUnicode(t *testing.T) {
require.Len(t, memos, 1)
}
func TestMemoFilterContentUnicodeCaseFold(t *testing.T) {
t.Parallel()
tc := NewMemoFilterTestContext(t)
defer tc.Close()
tc.CreateMemo(NewMemoBuilder("memo-unicode-case", tc.User.ID).Content("Привет Мир"))
memos := tc.ListWithFilter(`content.contains("привет")`)
require.Len(t, memos, 1)
}
func TestMemoFilterContentCaseSensitivity(t *testing.T) {
t.Parallel()
tc := NewMemoFilterTestContext(t)