mirror of https://github.com/usememos/memos.git
fix: add Unicode case-insensitive search for SQLite (#5559)
Add custom memos_unicode_lower() SQLite function to enable proper case-insensitive text search for non-English languages (Cyrillic, Greek, CJK, etc.). Previously, SQLite's LOWER() only worked for ASCII characters due to modernc.org/sqlite lacking ICU extension. This caused searches for non-English text to be case-sensitive (e.g., searching 'блины' wouldn't find 'Блины'). Changes: - Add store/db/sqlite/functions.go with Unicode case folding function - Register custom function using golang.org/x/text/cases.Fold() - Update filter renderer to use custom function for SQLite dialect - Add test for Unicode case-insensitive search - Make golang.org/x/text a direct dependency Fixes #5559
This commit is contained in:
parent
c14843fa62
commit
8770b186e4
2
go.mod
2
go.mod
|
|
@ -131,7 +131,7 @@ require (
|
|||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
golang.org/x/sys v0.37.0 // indirect
|
||||
golang.org/x/text v0.30.0 // indirect
|
||||
golang.org/x/text v0.30.0
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/protobuf v1.36.9
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
|
|
|||
|
|
@ -454,6 +454,11 @@ func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResul
|
|||
column := field.columnExpr(r.dialect)
|
||||
arg := fmt.Sprintf("%%%s%%", cond.Value)
|
||||
switch r.dialect {
|
||||
case DialectSQLite:
|
||||
// Use custom Unicode-aware case folding function for case-insensitive comparison.
|
||||
// This overcomes SQLite's ASCII-only LOWER() limitation.
|
||||
sql := fmt.Sprintf("memos_unicode_lower(%s) LIKE memos_unicode_lower(%s)", column, r.addArg(arg))
|
||||
return renderResult{sql: sql}, nil
|
||||
case DialectPostgres:
|
||||
sql := fmt.Sprintf("%s ILIKE %s", column, r.addArg(arg))
|
||||
return renderResult{sql: sql}, nil
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
// Package sqlite provides SQLite driver implementation with custom functions.
|
||||
// Custom functions are registered globally on first use to extend SQLite's
|
||||
// limited ASCII-only text operations with proper Unicode support.
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/text/cases"
|
||||
msqlite "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
var (
|
||||
registerUnicodeLowerOnce sync.Once
|
||||
registerUnicodeLowerErr error
|
||||
// unicodeFold provides Unicode case folding for case-insensitive comparisons.
|
||||
// It's safe to use concurrently and reused across all function calls.
|
||||
unicodeFold = cases.Fold()
|
||||
)
|
||||
|
||||
// ensureUnicodeLowerRegistered registers the memos_unicode_lower custom function
|
||||
// with SQLite. This function provides proper Unicode case folding for case-insensitive
|
||||
// text comparisons, overcoming modernc.org/sqlite's lack of ICU extension.
|
||||
//
|
||||
// The function is registered once globally and is safe to call multiple times.
|
||||
func ensureUnicodeLowerRegistered() error {
|
||||
registerUnicodeLowerOnce.Do(func() {
|
||||
registerUnicodeLowerErr = msqlite.RegisterScalarFunction("memos_unicode_lower", 1, func(_ *msqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
|
||||
if len(args) == 0 || args[0] == nil {
|
||||
return nil, nil
|
||||
}
|
||||
switch v := args[0].(type) {
|
||||
case string:
|
||||
return unicodeFold.String(v), nil
|
||||
case []byte:
|
||||
return unicodeFold.String(string(v)), nil
|
||||
default:
|
||||
return v, nil
|
||||
}
|
||||
})
|
||||
})
|
||||
return registerUnicodeLowerErr
|
||||
}
|
||||
|
|
@ -6,8 +6,8 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
// Import the SQLite driver.
|
||||
_ "modernc.org/sqlite"
|
||||
// Note: modernc.org/sqlite driver is imported in functions.go where
|
||||
// RegisterScalarFunction is used. No blank import needed here.
|
||||
|
||||
"github.com/usememos/memos/internal/profile"
|
||||
"github.com/usememos/memos/store"
|
||||
|
|
@ -27,6 +27,10 @@ func NewDB(profile *profile.Profile) (store.Driver, error) {
|
|||
return nil, errors.New("dsn required")
|
||||
}
|
||||
|
||||
if err := ensureUnicodeLowerRegistered(); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to register sqlite unicode lower function")
|
||||
}
|
||||
|
||||
// Connect to the database with some sane settings:
|
||||
// - No shared-cache: it's obsolete; WAL journal mode is a better solution.
|
||||
// - No foreign key constraints: it's currently disabled by default, but it's a
|
||||
|
|
|
|||
|
|
@ -61,6 +61,17 @@ func TestMemoFilterContentUnicode(t *testing.T) {
|
|||
require.Len(t, memos, 1)
|
||||
}
|
||||
|
||||
func TestMemoFilterContentUnicodeCaseFold(t *testing.T) {
|
||||
t.Parallel()
|
||||
tc := NewMemoFilterTestContext(t)
|
||||
defer tc.Close()
|
||||
|
||||
tc.CreateMemo(NewMemoBuilder("memo-unicode-case", tc.User.ID).Content("Привет Мир"))
|
||||
|
||||
memos := tc.ListWithFilter(`content.contains("привет")`)
|
||||
require.Len(t, memos, 1)
|
||||
}
|
||||
|
||||
func TestMemoFilterContentCaseSensitivity(t *testing.T) {
|
||||
t.Parallel()
|
||||
tc := NewMemoFilterTestContext(t)
|
||||
|
|
|
|||
Loading…
Reference in New Issue