diff --git a/plugin/filter/engine.go b/plugin/filter/engine.go index 25b5485f9..5ee068a66 100644 --- a/plugin/filter/engine.go +++ b/plugin/filter/engine.go @@ -96,9 +96,12 @@ func (p *Program) Render(opts RenderOptions) (Statement, error) { } var ( - defaultOnce sync.Once - defaultInst *Engine - defaultErr error + defaultOnce sync.Once + defaultInst *Engine + defaultErr error + defaultAttachmentOnce sync.Once + defaultAttachmentInst *Engine + defaultAttachmentErr error ) // DefaultEngine returns the process-wide memo filter engine. @@ -109,6 +112,14 @@ func DefaultEngine() (*Engine, error) { return defaultInst, defaultErr } +// DefaultAttachmentEngine returns the process-wide attachment filter engine. +func DefaultAttachmentEngine() (*Engine, error) { + defaultAttachmentOnce.Do(func() { + defaultAttachmentInst, defaultAttachmentErr = NewEngine(NewAttachmentSchema()) + }) + return defaultAttachmentInst, defaultAttachmentErr +} + func normalizeLegacyFilter(expr string) string { expr = rewriteNumericLogicalOperand(expr, "&&") expr = rewriteNumericLogicalOperand(expr, "||") diff --git a/plugin/filter/schema.go b/plugin/filter/schema.go index 4d5e3b4dc..21fae7423 100644 --- a/plugin/filter/schema.go +++ b/plugin/filter/schema.go @@ -243,6 +243,62 @@ func NewSchema() Schema { } } +// NewAttachmentSchema constructs the attachment filter schema and CEL environment. +func NewAttachmentSchema() Schema { + fields := map[string]Field{ + "filename": { + Name: "filename", + Kind: FieldKindScalar, + Type: FieldTypeString, + Column: Column{Table: "resource", Name: "filename"}, + SupportsContains: true, + Expressions: map[DialectName]string{}, + }, + "mime_type": { + Name: "mime_type", + Kind: FieldKindScalar, + Type: FieldTypeString, + Column: Column{Table: "resource", Name: "type"}, + Expressions: map[DialectName]string{}, + }, + "create_time": { + Name: "create_time", + Kind: FieldKindScalar, + Type: FieldTypeTimestamp, + Column: Column{Table: "resource", Name: "created_ts"}, + Expressions: map[DialectName]string{ + DialectMySQL: "UNIX_TIMESTAMP(%s)", + DialectPostgres: "EXTRACT(EPOCH FROM TO_TIMESTAMP(%s))", + }, + }, + "memo": { + Name: "memo", + Kind: FieldKindScalar, + Type: FieldTypeString, + Column: Column{Table: "resource", Name: "memo_uid"}, + Expressions: map[DialectName]string{}, + AllowedComparisonOps: map[ComparisonOperator]bool{ + CompareEq: true, + CompareNeq: true, + }, + }, + } + + envOptions := []cel.EnvOption{ + cel.Variable("filename", cel.StringType), + cel.Variable("mime_type", cel.StringType), + cel.Variable("create_time", cel.IntType), + cel.Variable("memo", cel.StringType), + nowFunction, + } + + return Schema{ + Name: "attachment", + Fields: fields, + EnvOptions: envOptions, + } +} + // columnExpr returns the field expression for the given dialect, applying // any schema-specific overrides (e.g. UNIX timestamp conversions). func (f Field) columnExpr(d DialectName) string { diff --git a/proto/api/v1/attachment_service.proto b/proto/api/v1/attachment_service.proto index ad2631f45..7385e114b 100644 --- a/proto/api/v1/attachment_service.proto +++ b/proto/api/v1/attachment_service.proto @@ -101,9 +101,9 @@ message ListAttachmentsRequest { string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Filter to apply to the list results. - // Example: "type=image/png" or "filename:*.jpg" - // Supported operators: =, !=, <, <=, >, >=, : - // Supported fields: filename, type, size, create_time, memo + // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")" + // Supported operators: =, !=, <, <=, >, >=, : (contains), in + // Supported fields: filename, mime_type, create_time, memo string filter = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The order to sort results by. diff --git a/proto/gen/api/v1/attachment_service.pb.go b/proto/gen/api/v1/attachment_service.pb.go index 535bbb3bd..ea6525e5c 100644 --- a/proto/gen/api/v1/attachment_service.pb.go +++ b/proto/gen/api/v1/attachment_service.pb.go @@ -201,9 +201,9 @@ type ListAttachmentsRequest struct { // Provide this to retrieve the subsequent page. PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` // Optional. Filter to apply to the list results. - // Example: "type=image/png" or "filename:*.jpg" - // Supported operators: =, !=, <, <=, >, >=, : - // Supported fields: filename, type, size, create_time, memo + // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")" + // Supported operators: =, !=, <, <=, >, >=, : (contains), in + // Supported fields: filename, mime_type, create_time, memo Filter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` // Optional. The order to sort results by. // Example: "create_time desc" or "filename asc" diff --git a/proto/gen/openapi.yaml b/proto/gen/openapi.yaml index 1e95422e6..f4746dbd5 100644 --- a/proto/gen/openapi.yaml +++ b/proto/gen/openapi.yaml @@ -97,9 +97,9 @@ paths: in: query description: |- Optional. Filter to apply to the list results. - Example: "type=image/png" or "filename:*.jpg" - Supported operators: =, !=, <, <=, >, >=, : - Supported fields: filename, type, size, create_time, memo + Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")" + Supported operators: =, !=, <, <=, >, >=, : (contains), in + Supported fields: filename, mime_type, create_time, memo schema: type: string - name: orderBy diff --git a/server/router/api/v1/attachment_service.go b/server/router/api/v1/attachment_service.go index 2e05d4526..642fcb67f 100644 --- a/server/router/api/v1/attachment_service.go +++ b/server/router/api/v1/attachment_service.go @@ -21,6 +21,7 @@ import ( "github.com/usememos/memos/internal/profile" "github.com/usememos/memos/internal/util" + "github.com/usememos/memos/plugin/filter" "github.com/usememos/memos/plugin/storage/s3" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" @@ -156,6 +157,14 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAt Offset: &offset, } + // Parse filter if provided + if request.Filter != "" { + if err := s.validateAttachmentFilter(ctx, request.Filter); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err) + } + findAttachment.Filters = append(findAttachment.Filters, request.Filter) + } + attachments, err := s.Store.ListAttachments(ctx, findAttachment) if err != nil { return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err) @@ -472,3 +481,29 @@ func isValidMimeType(mimeType string) bool { matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType) return matched } + +func (s *APIV1Service) validateAttachmentFilter(ctx context.Context, filterStr string) error { + if filterStr == "" { + return errors.New("filter cannot be empty") + } + + engine, err := filter.DefaultAttachmentEngine() + if err != nil { + return err + } + + var dialect filter.DialectName + switch s.Profile.Driver { + case "mysql": + dialect = filter.DialectMySQL + case "postgres": + dialect = filter.DialectPostgres + default: + dialect = filter.DialectSQLite + } + + if _, err := engine.CompileToStatement(ctx, filterStr, filter.RenderOptions{Dialect: dialect}); err != nil { + return errors.Wrap(err, "failed to compile filter") + } + return nil +} diff --git a/store/attachment.go b/store/attachment.go index 6b4dcba85..cd3ac2a16 100644 --- a/store/attachment.go +++ b/store/attachment.go @@ -51,6 +51,7 @@ type FindAttachment struct { MemoIDList []int32 HasRelatedMemo bool StorageType *storepb.AttachmentStorageType + Filters []string Limit *int Offset *int } diff --git a/store/db/mysql/attachment.go b/store/db/mysql/attachment.go index 6eb59338d..ead254d88 100644 --- a/store/db/mysql/attachment.go +++ b/store/db/mysql/attachment.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/encoding/protojson" + "github.com/usememos/memos/plugin/filter" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store" ) @@ -83,6 +84,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String()) } + if len(find.Filters) > 0 { + engine, err := filter.DefaultAttachmentEngine() + if err != nil { + return nil, errors.Wrap(err, "failed to get filter engine") + } + if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectMySQL, &where, &args); err != nil { + return nil, errors.Wrap(err, "failed to append filter conditions") + } + } + fields := []string{ "`resource`.`id` AS `id`", "`resource`.`uid` AS `uid`", diff --git a/store/db/postgres/attachment.go b/store/db/postgres/attachment.go index 1c17941d6..9ee970fbd 100644 --- a/store/db/postgres/attachment.go +++ b/store/db/postgres/attachment.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/encoding/protojson" + "github.com/usememos/memos/plugin/filter" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store" ) @@ -72,6 +73,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String()) } + if len(find.Filters) > 0 { + engine, err := filter.DefaultAttachmentEngine() + if err != nil { + return nil, errors.Wrap(err, "failed to get filter engine") + } + if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectPostgres, &where, &args); err != nil { + return nil, errors.Wrap(err, "failed to append filter conditions") + } + } + fields := []string{ "resource.id AS id", "resource.uid AS uid", diff --git a/store/db/sqlite/attachment.go b/store/db/sqlite/attachment.go index c35257338..04653b185 100644 --- a/store/db/sqlite/attachment.go +++ b/store/db/sqlite/attachment.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/encoding/protojson" + "github.com/usememos/memos/plugin/filter" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store" ) @@ -76,6 +77,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String()) } + if len(find.Filters) > 0 { + engine, err := filter.DefaultAttachmentEngine() + if err != nil { + return nil, errors.Wrap(err, "failed to get filter engine") + } + if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectSQLite, &where, &args); err != nil { + return nil, errors.Wrap(err, "failed to append filter conditions") + } + } + fields := []string{ "`resource`.`id` AS `id`", "`resource`.`uid` AS `uid`", diff --git a/store/test/attachment_test.go b/store/test/attachment_test.go index 8655029bf..68e080636 100644 --- a/store/test/attachment_test.go +++ b/store/test/attachment_test.go @@ -61,3 +61,62 @@ func TestAttachmentStore(t *testing.T) { require.ErrorContains(t, err, "attachment not found") ts.Close() } + +func TestAttachmentStoreWithFilter(t *testing.T) { + ctx := context.Background() + ts := NewTestingStore(ctx, t) + + _, err := ts.CreateAttachment(ctx, &store.Attachment{ + UID: shortuuid.New(), + CreatorID: 101, + Filename: "test.png", + Blob: []byte("test"), + Type: "image/png", + Size: 1000, + }) + require.NoError(t, err) + + _, err = ts.CreateAttachment(ctx, &store.Attachment{ + UID: shortuuid.New(), + CreatorID: 101, + Filename: "test.jpg", + Blob: []byte("test"), + Type: "image/jpeg", + Size: 2000, + }) + require.NoError(t, err) + + _, err = ts.CreateAttachment(ctx, &store.Attachment{ + UID: shortuuid.New(), + CreatorID: 101, + Filename: "test.pdf", + Blob: []byte("test"), + Type: "application/pdf", + Size: 3000, + }) + require.NoError(t, err) + + attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{ + CreatorID: &[]int32{101}[0], + Filters: []string{`mime_type == "image/png"`}, + }) + require.NoError(t, err) + require.Len(t, attachments, 1) + require.Equal(t, "image/png", attachments[0].Type) + + attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{ + CreatorID: &[]int32{101}[0], + Filters: []string{`mime_type in ["image/png", "image/jpeg"]`}, + }) + require.NoError(t, err) + require.Len(t, attachments, 2) + + attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{ + CreatorID: &[]int32{101}[0], + Filters: []string{`filename.contains("test")`}, + }) + require.NoError(t, err) + require.Len(t, attachments, 3) + + ts.Close() +} diff --git a/web/src/types/proto/api/v1/attachment_service_pb.ts b/web/src/types/proto/api/v1/attachment_service_pb.ts index 31d8cd58f..defda15c9 100644 --- a/web/src/types/proto/api/v1/attachment_service_pb.ts +++ b/web/src/types/proto/api/v1/attachment_service_pb.ts @@ -139,9 +139,9 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques /** * Optional. Filter to apply to the list results. - * Example: "type=image/png" or "filename:*.jpg" - * Supported operators: =, !=, <, <=, >, >=, : - * Supported fields: filename, type, size, create_time, memo + * Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")" + * Supported operators: =, !=, <, <=, >, >=, : (contains), in + * Supported fields: filename, mime_type, create_time, memo * * @generated from field: string filter = 3; */