From 0dfcbe97bc30ca82ef758e80fb64dafe681de161 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:06:37 +0800 Subject: [PATCH] feat: support manual file vectorization --- backend/docs/docs.go | 304 ++++++++++++++++- backend/docs/swagger.json | 304 ++++++++++++++++- backend/docs/swagger.yaml | 198 +++++++++++ backend/internal/app/app.go | 1 + .../application/conversation/service_file.go | 20 ++ .../internal/application/embedding/service.go | 308 +++++++++++++++++- .../application/embedding/service_test.go | 178 +++++++++- .../application/knowledgebase/service.go | 35 ++ .../application/processing/service.go | 288 ++++++++++++++-- .../processing/service_embedding_test.go | 232 +++++++++++++ backend/internal/domain/conversation/types.go | 2 + backend/internal/infra/cache/memory/cache.go | 12 +- .../internal/infra/cache/memory/file_queue.go | 188 ++++++++--- .../infra/cache/memory/file_queue_test.go | 74 ++++- .../infra/cache/redis/conversation_cache.go | 209 +++++++++--- .../infra/cache/redis/file_queue_test.go | 49 +++ .../internal/infra/persistence/models/chat.go | 2 +- .../postgres/conversation/repository.go | 28 +- .../repository_sqlite_vector_test.go | 53 +++ .../postgres/knowledgebase/repository.go | 12 +- .../knowledgebase/repository_sqlite_test.go | 34 +- .../internal/repository/conversation_cache.go | 22 +- .../internal/repository/conversation_file.go | 4 +- .../internal/shared/response/error_code.go | 2 + .../http/conversation/dto_request.go | 5 + .../http/conversation/dto_response.go | 120 ++++--- .../http/conversation/handler_file.go | 59 +++- .../transport/http/conversation/router.go | 1 + .../transport/http/knowledgebase/dto.go | 119 +++++-- .../transport/http/knowledgebase/handler.go | 71 +++- .../transport/http/knowledgebase/router.go | 1 + .../http/settings/handler_embedding_test.go | 8 + .../components/sections/files/admin-files.tsx | 89 ++--- .../features/admin/model/files-settings.ts | 18 +- .../features/files/components/app-files.tsx | 31 +- .../sections/content/content-header.tsx | 33 +- .../sections/sidebar/sidebar-header.tsx | 258 ++++++++------- .../sections/sidebar/sidebar-list.tsx | 44 ++- .../features/files/hooks/use-files-page.ts | 99 +++++- .../components/admin-knowledge-bases.tsx | 23 +- .../components/app-knowledge-bases.tsx | 9 +- .../components/knowledge-base-detail.tsx | 166 ++++++++-- .../hooks/use-knowledge-bases-page.ts | 104 +++++- frontend/i18n/messages/en-US/errors.json | 4 +- frontend/i18n/messages/en-US/files.json | 12 + .../i18n/messages/en-US/knowledge-bases.json | 12 + frontend/i18n/messages/zh-CN/errors.json | 4 +- frontend/i18n/messages/zh-CN/files.json | 12 + .../i18n/messages/zh-CN/knowledge-bases.json | 12 + frontend/shared/api/file.ts | 18 + frontend/shared/api/file.types.ts | 2 + frontend/shared/api/knowledge-bases.ts | 26 +- frontend/shared/api/knowledge-bases.types.ts | 10 +- frontend/shared/lib/file-processing.ts | 61 +++- packages/api-contract/src/types.generated.ts | 90 +++++ 55 files changed, 3547 insertions(+), 533 deletions(-) create mode 100644 backend/internal/application/processing/service_embedding_test.go create mode 100644 backend/internal/infra/cache/redis/file_queue_test.go diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 077d8debc..f757e4469 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -2092,6 +2092,63 @@ const docTemplate = `{ } } }, + "/admin/knowledge-bases/files/embeddings": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "为管理员选中的平台资料提交向量化任务，最多100个；重复提交会幂等跳过", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-knowledge-bases" + ], + "summary": "批量提交平台资料向量化", + "parameters": [ + { + "description": "平台资料ID，最多100个", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/SubmitPlatformFileEmbeddingsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + } + } + } + }, "/admin/knowledge-bases/files/{file_id}": { "delete": { "security": [ @@ -11788,6 +11845,63 @@ const docTemplate = `{ } } }, + "/files/embeddings": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "为当前用户已完成文本提取的文件提交向量化任务，最多100个；重复提交会幂等跳过", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "chat" + ], + "summary": "批量提交指定文件向量化", + "parameters": [ + { + "description": "文件ID，最多100个", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/SubmitFileEmbeddingsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/FileEmbeddingSubmissionResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + } + } + } + }, "/files/processing/statuses": { "post": { "security": [ @@ -18842,6 +18956,57 @@ const docTemplate = `{ } } }, + "FileEmbeddingSkipResponse": { + "type": "object", + "required": [ + "fileID", + "reason" + ], + "properties": { + "fileID": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "FileEmbeddingSubmissionResponse": { + "type": "object", + "required": [ + "skipped", + "submittedFileIDs" + ], + "properties": { + "skipped": { + "type": "array", + "items": { + "$ref": "#/definitions/FileEmbeddingSkipResponse" + } + }, + "submittedFileIDs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "FileEmbeddingSubmissionResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/FileEmbeddingSubmissionResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "FileListResponse": { "type": "object", "required": [ @@ -18882,6 +19047,7 @@ const docTemplate = `{ "FileObjectResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "createdAt", "detectedMIME", @@ -18903,9 +19069,13 @@ const docTemplate = `{ "sha256", "sizeBytes", "status", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -18975,12 +19145,16 @@ const docTemplate = `{ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, "FileProcessingStatusResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "completedAt", "detectedMIME", @@ -19000,9 +19174,13 @@ const docTemplate = `{ "ragReady", "ragReason", "startedAt", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -19066,6 +19244,9 @@ const docTemplate = `{ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, @@ -19726,6 +19907,57 @@ const docTemplate = `{ } } }, + "KnowledgeBaseFileEmbeddingSkipResponse": { + "type": "object", + "required": [ + "fileID", + "reason" + ], + "properties": { + "fileID": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "KnowledgeBaseFileEmbeddingSubmissionResponse": { + "type": "object", + "required": [ + "skipped", + "submittedFileIDs" + ], + "properties": { + "skipped": { + "type": "array", + "items": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSkipResponse" + } + }, + "submittedFileIDs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "KnowledgeBaseFileEmbeddingSubmissionResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "KnowledgeBaseFileMutationDataResponse": { "type": "object", "required": [ @@ -19803,27 +20035,40 @@ const docTemplate = `{ "KnowledgeBaseFileProcessingStatusResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "detectedMIME", + "embedError", "embedStatus", + "extractStatus", "fileCategory", "fileID", "processing", "processingReady", "processingStatus", "ragOptOut", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, "detectedMIME": { "type": "string" }, + "embedError": { + "type": "string" + }, "embedStatus": { "type": "string" }, + "extractStatus": { + "type": "string" + }, "fileCategory": { "type": "string" }, @@ -19844,16 +20089,22 @@ const docTemplate = `{ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, "KnowledgeBaseFileResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "createdAt", "detectedMIME", + "embedError", "embedStatus", + "extractStatus", "fileCategory", "fileID", "fileName", @@ -19863,9 +20114,13 @@ const docTemplate = `{ "processingStatus", "ragOptOut", "sizeBytes", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -19875,9 +20130,15 @@ const docTemplate = `{ "detectedMIME": { "type": "string" }, + "embedError": { + "type": "string" + }, "embedStatus": { "type": "string" }, + "extractStatus": { + "type": "string" + }, "fileCategory": { "type": "string" }, @@ -19907,6 +20168,9 @@ const docTemplate = `{ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, @@ -25003,6 +25267,38 @@ const docTemplate = `{ } } }, + "SubmitFileEmbeddingsRequest": { + "type": "object", + "required": [ + "fileIDs" + ], + "properties": { + "fileIDs": { + "type": "array", + "maxItems": 100, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, + "SubmitPlatformFileEmbeddingsRequest": { + "type": "object", + "required": [ + "fileIDs" + ], + "properties": { + "fileIDs": { + "type": "array", + "maxItems": 100, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "SubscribeRequest": { "type": "object", "required": [ diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index f16caaa8f..6faf0f27a 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -2085,6 +2085,63 @@ } } }, + "/admin/knowledge-bases/files/embeddings": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "为管理员选中的平台资料提交向量化任务，最多100个；重复提交会幂等跳过", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-knowledge-bases" + ], + "summary": "批量提交平台资料向量化", + "parameters": [ + { + "description": "平台资料ID，最多100个", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/SubmitPlatformFileEmbeddingsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/KnowledgebaseErrorDoc" + } + } + } + } + }, "/admin/knowledge-bases/files/{file_id}": { "delete": { "security": [ @@ -11781,6 +11838,63 @@ } } }, + "/files/embeddings": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "为当前用户已完成文本提取的文件提交向量化任务，最多100个；重复提交会幂等跳过", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "chat" + ], + "summary": "批量提交指定文件向量化", + "parameters": [ + { + "description": "文件ID，最多100个", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/SubmitFileEmbeddingsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/FileEmbeddingSubmissionResponseDoc" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/ConversationErrorDoc" + } + } + } + } + }, "/files/processing/statuses": { "post": { "security": [ @@ -18835,6 +18949,57 @@ } } }, + "FileEmbeddingSkipResponse": { + "type": "object", + "required": [ + "fileID", + "reason" + ], + "properties": { + "fileID": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "FileEmbeddingSubmissionResponse": { + "type": "object", + "required": [ + "skipped", + "submittedFileIDs" + ], + "properties": { + "skipped": { + "type": "array", + "items": { + "$ref": "#/definitions/FileEmbeddingSkipResponse" + } + }, + "submittedFileIDs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "FileEmbeddingSubmissionResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/FileEmbeddingSubmissionResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "FileListResponse": { "type": "object", "required": [ @@ -18875,6 +19040,7 @@ "FileObjectResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "createdAt", "detectedMIME", @@ -18896,9 +19062,13 @@ "sha256", "sizeBytes", "status", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -18968,12 +19138,16 @@ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, "FileProcessingStatusResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "completedAt", "detectedMIME", @@ -18993,9 +19167,13 @@ "ragReady", "ragReason", "startedAt", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -19059,6 +19237,9 @@ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, @@ -19719,6 +19900,57 @@ } } }, + "KnowledgeBaseFileEmbeddingSkipResponse": { + "type": "object", + "required": [ + "fileID", + "reason" + ], + "properties": { + "fileID": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "KnowledgeBaseFileEmbeddingSubmissionResponse": { + "type": "object", + "required": [ + "skipped", + "submittedFileIDs" + ], + "properties": { + "skipped": { + "type": "array", + "items": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSkipResponse" + } + }, + "submittedFileIDs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "KnowledgeBaseFileEmbeddingSubmissionResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, "KnowledgeBaseFileMutationDataResponse": { "type": "object", "required": [ @@ -19796,27 +20028,40 @@ "KnowledgeBaseFileProcessingStatusResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "detectedMIME", + "embedError", "embedStatus", + "extractStatus", "fileCategory", "fileID", "processing", "processingReady", "processingStatus", "ragOptOut", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, "detectedMIME": { "type": "string" }, + "embedError": { + "type": "string" + }, "embedStatus": { "type": "string" }, + "extractStatus": { + "type": "string" + }, "fileCategory": { "type": "string" }, @@ -19837,16 +20082,22 @@ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, "KnowledgeBaseFileResponse": { "type": "object", "required": [ + "canVectorize", "chunkCount", "createdAt", "detectedMIME", + "embedError", "embedStatus", + "extractStatus", "fileCategory", "fileID", "fileName", @@ -19856,9 +20107,13 @@ "processingStatus", "ragOptOut", "sizeBytes", - "updatedAt" + "updatedAt", + "vectorizationReason" ], "properties": { + "canVectorize": { + "type": "boolean" + }, "chunkCount": { "type": "integer" }, @@ -19868,9 +20123,15 @@ "detectedMIME": { "type": "string" }, + "embedError": { + "type": "string" + }, "embedStatus": { "type": "string" }, + "extractStatus": { + "type": "string" + }, "fileCategory": { "type": "string" }, @@ -19900,6 +20161,9 @@ }, "updatedAt": { "type": "string" + }, + "vectorizationReason": { + "type": "string" } } }, @@ -24996,6 +25260,38 @@ } } }, + "SubmitFileEmbeddingsRequest": { + "type": "object", + "required": [ + "fileIDs" + ], + "properties": { + "fileIDs": { + "type": "array", + "maxItems": 100, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, + "SubmitPlatformFileEmbeddingsRequest": { + "type": "object", + "required": [ + "fileIDs" + ], + "properties": { + "fileIDs": { + "type": "array", + "maxItems": 100, + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "SubscribeRequest": { "type": "object", "required": [ diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index c41855dfb..864e66f41 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -3139,6 +3139,40 @@ definitions: - data - errorMsg type: object + FileEmbeddingSkipResponse: + properties: + fileID: + type: string + reason: + type: string + required: + - fileID + - reason + type: object + FileEmbeddingSubmissionResponse: + properties: + skipped: + items: + $ref: '#/definitions/FileEmbeddingSkipResponse' + type: array + submittedFileIDs: + items: + type: string + type: array + required: + - skipped + - submittedFileIDs + type: object + FileEmbeddingSubmissionResponseDoc: + properties: + data: + $ref: '#/definitions/FileEmbeddingSubmissionResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object FileListResponse: properties: quota: @@ -3166,6 +3200,8 @@ definitions: type: object FileObjectResponse: properties: + canVectorize: + type: boolean chunkCount: type: integer createdAt: @@ -3214,7 +3250,10 @@ definitions: type: string updatedAt: type: string + vectorizationReason: + type: string required: + - canVectorize - chunkCount - createdAt - detectedMIME @@ -3237,9 +3276,12 @@ definitions: - sizeBytes - status - updatedAt + - vectorizationReason type: object FileProcessingStatusResponse: properties: + canVectorize: + type: boolean chunkCount: type: integer completedAt: @@ -3284,7 +3326,10 @@ definitions: x-omitempty: false updatedAt: type: string + vectorizationReason: + type: string required: + - canVectorize - chunkCount - completedAt - detectedMIME @@ -3305,6 +3350,7 @@ definitions: - ragReason - startedAt - updatedAt + - vectorizationReason type: object FileUpdateResponseDoc: properties: @@ -3758,6 +3804,40 @@ definitions: required: - file type: object + KnowledgeBaseFileEmbeddingSkipResponse: + properties: + fileID: + type: string + reason: + type: string + required: + - fileID + - reason + type: object + KnowledgeBaseFileEmbeddingSubmissionResponse: + properties: + skipped: + items: + $ref: '#/definitions/KnowledgeBaseFileEmbeddingSkipResponse' + type: array + submittedFileIDs: + items: + type: string + type: array + required: + - skipped + - submittedFileIDs + type: object + KnowledgeBaseFileEmbeddingSubmissionResponseDoc: + properties: + data: + $ref: '#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object KnowledgeBaseFileMutationDataResponse: properties: updated: @@ -3809,12 +3889,18 @@ definitions: type: object KnowledgeBaseFileProcessingStatusResponse: properties: + canVectorize: + type: boolean chunkCount: type: integer detectedMIME: type: string + embedError: + type: string embedStatus: type: string + extractStatus: + type: string fileCategory: type: string fileID: @@ -3829,10 +3915,15 @@ definitions: type: boolean updatedAt: type: string + vectorizationReason: + type: string required: + - canVectorize - chunkCount - detectedMIME + - embedError - embedStatus + - extractStatus - fileCategory - fileID - processing @@ -3840,17 +3931,24 @@ definitions: - processingStatus - ragOptOut - updatedAt + - vectorizationReason type: object KnowledgeBaseFileResponse: properties: + canVectorize: + type: boolean chunkCount: type: integer createdAt: type: string detectedMIME: type: string + embedError: + type: string embedStatus: type: string + extractStatus: + type: string fileCategory: type: string fileID: @@ -3871,11 +3969,16 @@ definitions: type: integer updatedAt: type: string + vectorizationReason: + type: string required: + - canVectorize - chunkCount - createdAt - detectedMIME + - embedError - embedStatus + - extractStatus - fileCategory - fileID - fileName @@ -3886,6 +3989,7 @@ definitions: - ragOptOut - sizeBytes - updatedAt + - vectorizationReason type: object KnowledgeBaseFileResponseDoc: properties: @@ -7459,6 +7563,28 @@ definitions: - usedBytes - userID type: object + SubmitFileEmbeddingsRequest: + properties: + fileIDs: + items: + type: string + maxItems: 100 + minItems: 1 + type: array + required: + - fileIDs + type: object + SubmitPlatformFileEmbeddingsRequest: + properties: + fileIDs: + items: + type: string + maxItems: 100 + minItems: 1 + type: array + required: + - fileIDs + type: object SubscribeRequest: properties: cycles: @@ -11387,6 +11513,42 @@ paths: summary: 获取平台资料内容 tags: - admin-knowledge-bases + /admin/knowledge-bases/files/embeddings: + post: + consumes: + - application/json + description: 为管理员选中的平台资料提交向量化任务，最多100个；重复提交会幂等跳过 + parameters: + - description: 平台资料ID，最多100个 + in: body + name: request + required: true + schema: + $ref: '#/definitions/SubmitPlatformFileEmbeddingsRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/KnowledgeBaseFileEmbeddingSubmissionResponseDoc' + "400": + description: Bad Request + schema: + $ref: '#/definitions/KnowledgebaseErrorDoc' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/KnowledgebaseErrorDoc' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/KnowledgebaseErrorDoc' + security: + - BearerAuth: [] + summary: 批量提交平台资料向量化 + tags: + - admin-knowledge-bases /admin/llm/icon-assets: get: description: 分页查询仍在图标库中的已上传图标，按上传时间倒序返回 @@ -17296,6 +17458,42 @@ paths: summary: 获取文件内容 tags: - chat + /files/embeddings: + post: + consumes: + - application/json + description: 为当前用户已完成文本提取的文件提交向量化任务，最多100个；重复提交会幂等跳过 + parameters: + - description: 文件ID，最多100个 + in: body + name: request + required: true + schema: + $ref: '#/definitions/SubmitFileEmbeddingsRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/FileEmbeddingSubmissionResponseDoc' + "400": + description: Bad Request + schema: + $ref: '#/definitions/ConversationErrorDoc' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/ConversationErrorDoc' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/ConversationErrorDoc' + security: + - BearerAuth: [] + summary: 批量提交指定文件向量化 + tags: + - chat /files/processing/statuses: post: consumes: diff --git a/backend/internal/app/app.go b/backend/internal/app/app.go index c693c1d3c..3e78b3784 100644 --- a/backend/internal/app/app.go +++ b/backend/internal/app/app.go @@ -412,6 +412,7 @@ func NewApp() (*App, error) { knowledgeBaseService.SetFileCleaner(conversationService) knowledgeBaseService.SetFileContentOpener(conversationService) knowledgeBaseService.SetFileUploader(conversationService) + knowledgeBaseService.SetFileEmbeddingSubmitter(processingService) knowledgeBaseService.SetLogger(log) conversationService.SetKnowledgeBaseResolver(knowledgeBaseService) knowledgeBaseHandler := knowledgebasehttp.NewHandler(knowledgeBaseService, runtimeCfg) diff --git a/backend/internal/application/conversation/service_file.go b/backend/internal/application/conversation/service_file.go index f03ec2b50..4fd8d0217 100644 --- a/backend/internal/application/conversation/service_file.go +++ b/backend/internal/application/conversation/service_file.go @@ -5,6 +5,7 @@ import ( "errors" "strings" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" @@ -52,6 +53,25 @@ func (s *Service) ListFiles( return s.uploadSvc.ListFiles(ctx, userID, page, pageSize, searchQuery, filterKind, sortBy) } +// SubmitFileEmbeddings 提交当前用户指定文件的向量化任务。 +func (s *Service) SubmitFileEmbeddings(ctx context.Context, userID uint, fileIDs []string) (appembedding.TargetedSubmissionResult, error) { + if s.processingSvc == nil { + return appembedding.TargetedSubmissionResult{}, appembedding.ErrEmbeddingServiceNotConfigured + } + return s.processingSvc.SubmitFileEmbeddings(ctx, userID, fileIDs) +} + +// ResolveFileVectorizationCapabilities 返回文件显式向量化能力的后端事实状态。 +func (s *Service) ResolveFileVectorizationCapabilities( + ctx context.Context, + files []model.FileObject, +) map[string]appembedding.FileVectorizationCapability { + if s.processingSvc == nil { + return map[string]appembedding.FileVectorizationCapability{} + } + return s.processingSvc.ResolveFileVectorizationCapabilities(ctx, files) +} + // UploadFile 上传文件并扣减用户配额。 func (s *Service) UploadFile(ctx context.Context, input appupload.UploadFileInput) (*appupload.UploadFileResult, error) { return s.uploadSvc.UploadFile(ctx, input) diff --git a/backend/internal/application/embedding/service.go b/backend/internal/application/embedding/service.go index 4d8b742f2..92c3c64ce 100644 --- a/backend/internal/application/embedding/service.go +++ b/backend/internal/application/embedding/service.go @@ -18,9 +18,54 @@ import ( "go.uber.org/zap" ) -var ErrEmbeddingServiceNotConfigured = errors.New("embedding service not configured") +var ( + ErrEmbeddingServiceNotConfigured = errors.New("embedding service not configured") + ErrEmbeddingServiceUnavailable = errors.New("embedding service unavailable") + ErrTooManyTargetedFiles = errors.New("too many files for targeted embedding") +) + +const ( + WorkerConcurrency = 4 + MaxTargetedFiles = 100 +) + +const ( + SkipReasonNotFound = "not_found" + SkipReasonNotReady = "not_ready" + SkipReasonUnsupported = "unsupported" + SkipReasonAlreadyReady = "already_ready" + SkipReasonProcessing = "processing" + SkipReasonQueueBusy = "queue_busy" + SkipReasonSubmitFailed = "submit_failed" + ReasonOutdatedIndex = "outdated_index" +) + +type TargetedFileSkip struct { + FileID string + Reason string +} -const embeddingWorkerConcurrency = 4 +type TargetedSubmissionResult struct { + SubmittedFileIDs []string + Skipped []TargetedFileSkip +} + +type TargetedJob struct { + FileID string + UserID uint + EmbeddingSignature string + EmbeddingHost string +} + +type TargetedSubmissionPlan struct { + Jobs []TargetedJob + Skipped []TargetedFileSkip +} + +type FileVectorizationCapability struct { + CanVectorize bool + Reason string +} // Service 封装文件 embedding 执行与状态管理能力。 type Service struct { @@ -33,6 +78,10 @@ type Service struct { reindexJobs chan string reindexMu sync.Mutex reindexing bool + + vectorStoreMu sync.Mutex + vectorStoreChecked bool + vectorStoreAvailable bool } // EmbeddingClient 调用外部服务将文本批量转换为向量。 @@ -56,7 +105,7 @@ func NewServiceWithRuntime(cfg *config.Runtime, repo repository.EmbeddingReposit extractSvc: extractSvc, embedClient: embedClient, logger: logger, - workSlots: make(chan struct{}, embeddingWorkerConcurrency), + workSlots: make(chan struct{}, WorkerConcurrency), reindexJobs: make(chan string, 1), } } @@ -110,7 +159,7 @@ func (s *Service) indexingAvailable(ctx context.Context, cfg config.Config) (boo if s.repo == nil { return false, "vector_store_unavailable", nil } - available, err := s.repo.VectorStoreAvailable(ctx) + available, err := s.cachedVectorStoreAvailable(ctx) if err != nil { if s.logger != nil { s.logger.Warn("embedding vector store availability check failed", zap.Error(err)) @@ -123,6 +172,24 @@ func (s *Service) indexingAvailable(ctx context.Context, cfg config.Config) (boo return true, "available", nil } +// cachedVectorStoreAvailable 缓存随进程启动确定的向量存储结构状态。 +// 配置项仍由 indexingAvailable 每次读取运行时快照，只有昂贵且在运行期间不应变化的 +// 扩展、字段和索引结构检查会被缓存；失败结果不会缓存，避免瞬时数据库错误污染后续请求。 +func (s *Service) cachedVectorStoreAvailable(ctx context.Context) (bool, error) { + s.vectorStoreMu.Lock() + defer s.vectorStoreMu.Unlock() + if s.vectorStoreChecked { + return s.vectorStoreAvailable, nil + } + available, err := s.repo.VectorStoreAvailable(ctx) + if err != nil { + return false, err + } + s.vectorStoreAvailable = available + s.vectorStoreChecked = true + return available, nil +} + // ShouldTrigger 判断当前文件是否应触发 embedding。 func (s *Service) ShouldTrigger(fileObj domainconversation.FileObject) bool { cfg := s.snapshot() @@ -150,6 +217,201 @@ func (s *Service) MaybeTrigger(fileObj domainconversation.FileObject) { s.Trigger(fileObj) } +// PlanFiles 校验当前用户指定文件并生成向量化任务计划。 +// 任务认领与投递由 processing 应用服务逐项完成，避免批量预认领后因中途失败遗留 processing 状态。 +func (s *Service) PlanFiles(ctx context.Context, userID uint, fileIDs []string) (TargetedSubmissionPlan, error) { + plan := TargetedSubmissionPlan{ + Jobs: []TargetedJob{}, + Skipped: []TargetedFileSkip{}, + } + normalizedIDs := normalizeTargetedFileIDs(fileIDs) + if len(normalizedIDs) > MaxTargetedFiles { + return plan, ErrTooManyTargetedFiles + } + if len(normalizedIDs) == 0 { + return plan, nil + } + + cfg := s.snapshot() + available, reason, err := s.indexingAvailable(ctx, cfg) + if !available { + return plan, embeddingAvailabilityError(reason, err) + } + + files, err := s.repo.GetActiveFileObjectsByIDs(ctx, userID, normalizedIDs) + if err != nil { + return plan, err + } + filesByID := make(map[string]domainconversation.FileObject, len(files)) + for i := range files { + filesByID[files[i].FileID] = files[i] + } + + embeddingSignature := configuredModelSignature(cfg) + embeddingHost := strings.TrimRight(strings.TrimSpace(cfg.EmbeddingHost), "/") + for _, fileID := range normalizedIDs { + fileObj, found := filesByID[fileID] + if !found { + plan.Skipped = append(plan.Skipped, TargetedFileSkip{FileID: fileID, Reason: SkipReasonNotFound}) + continue + } + if reason := fileVectorizationSkipReason(cfg, fileObj, embeddingSignature); reason != "" { + plan.Skipped = append(plan.Skipped, TargetedFileSkip{FileID: fileID, Reason: reason}) + continue + } + + plan.Jobs = append(plan.Jobs, TargetedJob{ + FileID: fileID, + UserID: userID, + EmbeddingSignature: embeddingSignature, + EmbeddingHost: embeddingHost, + }) + } + return plan, nil +} + +// QueueTargetedJob 原子登记单个已规划任务，防止并发提交产生重复队列消息。 +// 真正的 processing 状态由 worker 领取消息后再设置。 +func (s *Service) QueueTargetedJob(ctx context.Context, job TargetedJob) (bool, error) { + if s == nil || s.repo == nil || strings.TrimSpace(job.FileID) == "" || strings.TrimSpace(job.EmbeddingSignature) == "" { + return false, nil + } + return s.repo.QueueFileEmbedding(ctx, job.UserID, job.FileID, job.EmbeddingSignature) +} + +// ResolveFileVectorizationCapabilities 返回前端展示所需的后端事实状态。 +func (s *Service) ResolveFileVectorizationCapabilities( + ctx context.Context, + files []domainconversation.FileObject, +) map[string]FileVectorizationCapability { + capabilities := make(map[string]FileVectorizationCapability, len(files)) + cfg := s.snapshot() + signature := configuredModelSignature(cfg) + available, reason, _ := s.indexingAvailable(ctx, cfg) + if !available { + for i := range files { + capabilityReason := reason + if fileVectorIndexOutdated(files[i], signature) { + capabilityReason = ReasonOutdatedIndex + } + capabilities[files[i].FileID] = FileVectorizationCapability{Reason: capabilityReason} + } + return capabilities + } + for i := range files { + skipReason := fileVectorizationSkipReason(cfg, files[i], signature) + reason := skipReason + if reason == "" && fileVectorIndexOutdated(files[i], signature) { + reason = ReasonOutdatedIndex + } + capabilities[files[i].FileID] = FileVectorizationCapability{ + CanVectorize: skipReason == "", + Reason: reason, + } + } + return capabilities +} + +// ProcessTargetedJob 执行从可恢复队列中领取的显式向量化任务。 +func (s *Service) ProcessTargetedJob(ctx context.Context, job TargetedJob) error { + if s == nil || s.repo == nil || strings.TrimSpace(job.FileID) == "" { + return nil + } + releaseSlot, err := s.acquireWorkSlot(ctx) + if err != nil { + return err + } + defer releaseSlot() + + cfg := s.snapshot() + if configuredModelSignature(cfg) != strings.TrimSpace(job.EmbeddingSignature) || + strings.TrimRight(strings.TrimSpace(cfg.EmbeddingHost), "/") != strings.TrimRight(strings.TrimSpace(job.EmbeddingHost), "/") { + _ = s.updateFileObjectEmbedStatus(ctx, job.UserID, job.FileID, job.EmbeddingSignature, "stale", "embedding configuration changed before processing") + return nil + } + available, reason, err := s.indexingAvailable(ctx, cfg) + if !available { + switch reason { + case "embedding_disabled", "embedding_model_missing", "embedding_host_missing": + _ = s.updateFileObjectEmbedStatus(ctx, job.UserID, job.FileID, job.EmbeddingSignature, "stale", "embedding configuration changed before processing") + return nil + default: + return embeddingAvailabilityError(reason, err) + } + } + fileObj, err := s.repo.GetActiveFileObjectByID(ctx, job.UserID, job.FileID) + if err != nil || fileObj == nil { + return err + } + if fileObj.EmbedSignature != job.EmbeddingSignature || strings.ToLower(strings.TrimSpace(fileObj.EmbedStatus)) != "processing" { + claimed, claimErr := s.repo.ClaimFileEmbedding(ctx, job.UserID, job.FileID, job.EmbeddingSignature) + if claimErr != nil || !claimed { + return claimErr + } + } + return s.processClaimedFile(ctx, *fileObj, cfg, job.EmbeddingSignature) +} + +// FailTargetedJob 将投递失败的已领取任务释放为可重试状态。 +func (s *Service) FailTargetedJob(ctx context.Context, job TargetedJob, message string) error { + return s.updateFileObjectEmbedStatus(ctx, job.UserID, job.FileID, job.EmbeddingSignature, "failed", truncateError(message, 255)) +} + +// RequeueTargetedJob 将等待重试的任务恢复为排队状态，避免重试退避期间误显示为执行中或失败。 +func (s *Service) RequeueTargetedJob(ctx context.Context, job TargetedJob, message string) error { + return s.updateFileObjectEmbedStatus(ctx, job.UserID, job.FileID, job.EmbeddingSignature, "queued", truncateError(message, 255)) +} + +func fileVectorizationSkipReason(cfg config.Config, fileObj domainconversation.FileObject, embeddingSignature string) string { + if fileObj.EmbedSignature == embeddingSignature { + switch strings.ToLower(strings.TrimSpace(fileObj.EmbedStatus)) { + case "ready": + return SkipReasonAlreadyReady + case "queued", "processing": + return SkipReasonProcessing + } + } + if !fileObj.ProcessingReady { + return SkipReasonNotReady + } + if !canEmbedFile(cfg, fileObj) { + return SkipReasonUnsupported + } + return "" +} + +func fileVectorIndexOutdated(fileObj domainconversation.FileObject, embeddingSignature string) bool { + status := strings.ToLower(strings.TrimSpace(fileObj.EmbedStatus)) + return status == "stale" || (status == "ready" && strings.TrimSpace(embeddingSignature) != "" && fileObj.EmbedSignature != embeddingSignature) +} + +func normalizeTargetedFileIDs(fileIDs []string) []string { + normalized := make([]string, 0, len(fileIDs)) + seen := make(map[string]struct{}, len(fileIDs)) + for _, value := range fileIDs { + fileID := strings.TrimSpace(value) + if fileID == "" { + continue + } + if _, exists := seen[fileID]; exists { + continue + } + seen[fileID] = struct{}{} + normalized = append(normalized, fileID) + } + return normalized +} + +func embeddingAvailabilityError(reason string, cause error) error { + if cause != nil { + return fmt.Errorf("%w: %v", ErrEmbeddingServiceUnavailable, cause) + } + if reason == "embedding_disabled" || reason == "embedding_model_missing" || reason == "embedding_host_missing" { + return ErrEmbeddingServiceNotConfigured + } + return ErrEmbeddingServiceUnavailable +} + // Trigger 异步触发 embedding。 func (s *Service) Trigger(fileObj domainconversation.FileObject) { background.Go(s.logger, "embedding_process_file", func() { @@ -166,15 +428,6 @@ func (s *Service) Trigger(fileObj domainconversation.FileObject) { // ProcessFile 执行 embedding 完整流程。 func (s *Service) ProcessFile(ctx context.Context, fileObj domainconversation.FileObject) error { - if s != nil && s.workSlots != nil { - select { - case s.workSlots <- struct{}{}: - defer func() { <-s.workSlots }() - case <-ctx.Done(): - return ctx.Err() - } - } - cfg := s.snapshot() embeddingSignature := configuredModelSignature(cfg) if !cfg.EmbeddingEnabled || strings.TrimSpace(cfg.RAGModel) == "" || strings.TrimSpace(cfg.EmbeddingHost) == "" { @@ -183,9 +436,14 @@ func (s *Service) ProcessFile(ctx context.Context, fileObj domainconversation.Fi if s.repo == nil { return nil } - if !supportsEmbeddingSource(fileObj, cfg) { + if !canEmbedFile(cfg, fileObj) { return nil } + releaseSlot, err := s.acquireWorkSlot(ctx) + if err != nil { + return err + } + defer releaseSlot() claimed, err := s.repo.ClaimFileEmbedding(ctx, fileObj.UserID, fileObj.FileID, embeddingSignature) if err != nil { @@ -194,7 +452,10 @@ func (s *Service) ProcessFile(ctx context.Context, fileObj domainconversation.Fi if !claimed { return nil } + return s.processClaimedFile(ctx, fileObj, cfg, embeddingSignature) +} +func (s *Service) processClaimedFile(ctx context.Context, fileObj domainconversation.FileObject, cfg config.Config, embeddingSignature string) error { text, err := s.loadSourceText(ctx, fileObj) if err != nil { _ = s.updateFileObjectEmbedStatus(ctx, fileObj.UserID, fileObj.FileID, embeddingSignature, "failed", "无法提取文本") @@ -247,6 +508,18 @@ func (s *Service) ProcessFile(ctx context.Context, fileObj domainconversation.Fi return s.completeFileEmbedding(ctx, fileObj, embeddingSignature, cfg.EmbeddingHost) } +func (s *Service) acquireWorkSlot(ctx context.Context) (func(), error) { + if s == nil || s.workSlots == nil { + return func() {}, nil + } + select { + case s.workSlots <- struct{}{}: + return func() { <-s.workSlots }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + func (s *Service) completeFileEmbedding(ctx context.Context, fileObj domainconversation.FileObject, expectedSignature string, expectedHost string) error { const configurationChanged = "embedding configuration changed during processing" if !s.embeddingConfigurationCurrent(expectedSignature, expectedHost) { @@ -466,8 +739,9 @@ func (s *Service) GetIndexStatus(ctx context.Context) (EmbeddingIndexStatus, err return status, err } noneCount, _ := s.repo.CountFilesByEmbedStatus(ctx, "none") + queuedCount, _ := s.repo.CountFilesByEmbedStatus(ctx, "queued") processingCount, _ := s.repo.CountFilesByEmbedStatus(ctx, "processing") - status.PendingCount = noneCount + processingCount + status.PendingCount = noneCount + queuedCount + processingCount status.NeedsReindex = status.StaleCount > 0 return status, nil } @@ -559,9 +833,9 @@ func (s *Service) runReindex(ctx context.Context, expectedSignature string) { s.reindexMu.Unlock() }() - jobs := make(chan domainconversation.FileObject, embeddingWorkerConcurrency) + jobs := make(chan domainconversation.FileObject, WorkerConcurrency) var workers sync.WaitGroup - for range embeddingWorkerConcurrency { + for range WorkerConcurrency { workers.Add(1) go func() { defer workers.Done() diff --git a/backend/internal/application/embedding/service_test.go b/backend/internal/application/embedding/service_test.go index 51a000230..46cdbed6c 100644 --- a/backend/internal/application/embedding/service_test.go +++ b/backend/internal/application/embedding/service_test.go @@ -2,6 +2,8 @@ package embedding import ( "context" + "errors" + "fmt" "math" "sync" "testing" @@ -163,6 +165,25 @@ func TestIndexingAvailableDoesNotRequireRAGEnabled(t *testing.T) { } } +func TestIndexingAvailableCachesVectorStoreStructureCheck(t *testing.T) { + repo := &reindexRepo{vectorAvailable: true} + service := NewService(config.Config{ + EmbeddingEnabled: true, + RAGModel: "text-embedding-test", + EmbeddingHost: "http://127.0.0.1:8081", + }, repo, nil, infraembedding.New(security.OutboundPolicy{}), nil) + + for range 2 { + available, reason := service.IndexingAvailable(context.Background()) + if !available { + t.Fatalf("expected indexing to be available, got %s", reason) + } + } + if repo.vectorAvailableCalls != 1 { + t.Fatalf("VectorStoreAvailable() calls = %d, want 1", repo.vectorAvailableCalls) + } +} + func TestReindexStaleFilesDoesNotRequireRAGEnabled(t *testing.T) { repo := &reindexRepo{vectorAvailable: true} service := NewService(config.Config{ @@ -208,6 +229,119 @@ func TestCanEmbedFileDoesNotRequireAutoTrigger(t *testing.T) { } } +func TestPlanFilesReturnsEligibleFilesWithoutAutoTrigger(t *testing.T) { + cfg := config.Config{ + EmbeddingEnabled: true, + EmbedTriggerOnUpload: false, + RAGModel: "text-embedding-test", + EmbeddingHost: "http://127.0.0.1:8081", + EmbeddingOutputDimensions: 1536, + } + signature := configuredModelSignature(cfg) + repo := &reindexRepo{ + vectorAvailable: true, + files: []domainconversation.FileObject{ + {ID: 1, UserID: 7, FileID: "eligible", FileName: "ready.txt", MimeType: "text/plain", StoragePath: "uploads/ready.txt", Status: "active", ProcessingReady: true, ExtractStatus: "none"}, + {ID: 2, UserID: 7, FileID: "complete", FileName: "complete.txt", MimeType: "text/plain", StoragePath: "uploads/complete.txt", Status: "active", ProcessingReady: true, ExtractStatus: "ready", EmbedStatus: "ready", EmbedSignature: signature}, + {ID: 3, UserID: 7, FileID: "video", FileName: "clip.mp4", MimeType: "video/mp4", FileCategory: "video", StoragePath: "uploads/clip.mp4", Status: "active", ProcessingReady: true, ExtractStatus: "ready"}, + {ID: 4, UserID: 7, FileID: "extracting", FileName: "pending.txt", MimeType: "text/plain", StoragePath: "uploads/pending.txt", Status: "active", ProcessingReady: false, ExtractStatus: "processing"}, + {ID: 5, UserID: 7, FileID: "queued", FileName: "queued.txt", MimeType: "text/plain", StoragePath: "uploads/queued.txt", Status: "active", ProcessingReady: true, ExtractStatus: "ready", EmbedStatus: "queued", EmbedSignature: signature}, + }, + } + service := NewService(cfg, repo, nil, infraembedding.New(security.OutboundPolicy{}), nil) + + plan, err := service.PlanFiles(context.Background(), 7, []string{"eligible", "complete", "video", "extracting", "queued", "missing", "eligible"}) + if err != nil { + t.Fatalf("prepare files: %v", err) + } + if len(plan.Jobs) != 1 || plan.Jobs[0].FileID != "eligible" { + t.Fatalf("prepared jobs = %#v, want eligible", plan.Jobs) + } + wantSkipped := map[string]string{ + "complete": SkipReasonAlreadyReady, + "video": SkipReasonUnsupported, + "extracting": SkipReasonNotReady, + "queued": SkipReasonProcessing, + "missing": SkipReasonNotFound, + } + if len(plan.Skipped) != len(wantSkipped) { + t.Fatalf("skipped = %#v", plan.Skipped) + } + for _, item := range plan.Skipped { + if wantSkipped[item.FileID] != item.Reason { + t.Fatalf("skip %s = %s, want %s", item.FileID, item.Reason, wantSkipped[item.FileID]) + } + } + if len(repo.claimedFileIDs) != 0 { + t.Fatalf("planning must not claim files, got %#v", repo.claimedFileIDs) + } + claimed, err := service.QueueTargetedJob(context.Background(), plan.Jobs[0]) + if err != nil || !claimed { + t.Fatalf("claim planned job: claimed=%v err=%v", claimed, err) + } + if len(repo.claimedFileIDs) != 1 || repo.claimedFileIDs[0] != "eligible" { + t.Fatalf("claimed files = %#v, want [eligible]", repo.claimedFileIDs) + } +} + +func TestResolveFileVectorizationCapabilitiesDistinguishesOutdatedIndex(t *testing.T) { + cfg := config.Config{ + EmbeddingEnabled: true, + RAGModel: "text-embedding-test", + EmbeddingHost: "http://127.0.0.1:8081", + EmbeddingOutputDimensions: 1536, + } + signature := configuredModelSignature(cfg) + service := NewService( + cfg, + &reindexRepo{vectorAvailable: true}, + nil, + infraembedding.New(security.OutboundPolicy{}), + nil, + ) + + capabilities := service.ResolveFileVectorizationCapabilities(context.Background(), []domainconversation.FileObject{ + {FileID: "current", FileName: "current.txt", MimeType: "text/plain", StoragePath: "uploads/current.txt", Status: "active", ProcessingReady: true, EmbedStatus: "ready", EmbedSignature: signature}, + {FileID: "outdated", FileName: "outdated.txt", MimeType: "text/plain", StoragePath: "uploads/outdated.txt", Status: "active", ProcessingReady: true, EmbedStatus: "ready", EmbedSignature: "legacy-signature"}, + }) + + if current := capabilities["current"]; current.CanVectorize || current.Reason != SkipReasonAlreadyReady { + t.Fatalf("current capability = %#v, want already ready", current) + } + if outdated := capabilities["outdated"]; !outdated.CanVectorize || outdated.Reason != ReasonOutdatedIndex { + t.Fatalf("outdated capability = %#v, want update available", outdated) + } +} + +func TestPlanFilesDistinguishesConfigurationFromRuntimeAvailability(t *testing.T) { + configured := config.Config{ + EmbeddingEnabled: true, + RAGModel: "text-embedding-test", + EmbeddingHost: "http://127.0.0.1:8081", + EmbeddingOutputDimensions: 1536, + } + service := NewService(configured, &reindexRepo{vectorAvailable: false}, nil, infraembedding.New(security.OutboundPolicy{}), nil) + if _, err := service.PlanFiles(context.Background(), 7, []string{"file_1"}); !errors.Is(err, ErrEmbeddingServiceUnavailable) { + t.Fatalf("runtime error = %v, want ErrEmbeddingServiceUnavailable", err) + } + + disabled := NewService(config.Config{}, &reindexRepo{}, nil, infraembedding.New(security.OutboundPolicy{}), nil) + if _, err := disabled.PlanFiles(context.Background(), 7, []string{"file_1"}); !errors.Is(err, ErrEmbeddingServiceNotConfigured) { + t.Fatalf("configuration error = %v, want ErrEmbeddingServiceNotConfigured", err) + } +} + +func TestPlanFilesRejectsMoreThanBatchLimit(t *testing.T) { + service := NewService(config.Config{}, &reindexRepo{}, nil, nil, nil) + fileIDs := make([]string, MaxTargetedFiles+1) + for i := range fileIDs { + fileIDs[i] = fmt.Sprintf("file_%d", i) + } + if _, err := service.PlanFiles(context.Background(), 1, fileIDs); !errors.Is(err, ErrTooManyTargetedFiles) { + t.Fatalf("error = %v, want ErrTooManyTargetedFiles", err) + } +} + func TestReindexStaleFilesSkipsUnsupportedCandidates(t *testing.T) { repo := &reindexRepo{ vectorAvailable: true, @@ -500,14 +634,16 @@ func TestCompleteFileEmbeddingKeepsChangedEndpointStale(t *testing.T) { } type reindexRepo struct { - vectorAvailable bool - files []domainconversation.FileObject - afterIDs []uint - listCalls int - updateStatusCalls int - statusHistory []string - markedSignature string - onStatus func(status string) + vectorAvailable bool + vectorAvailableCalls int + files []domainconversation.FileObject + afterIDs []uint + listCalls int + updateStatusCalls int + statusHistory []string + markedSignature string + claimedFileIDs []string + onStatus func(status string) } type blockingReindexRepo struct { @@ -538,6 +674,7 @@ func (r *blockingReindexRepo) ListFilesForReindex(ctx context.Context, limit int } func (r *reindexRepo) VectorStoreAvailable(context.Context) (bool, error) { + r.vectorAvailableCalls++ return r.vectorAvailable, nil } @@ -545,11 +682,34 @@ func (r *reindexRepo) GetActiveFileObjectByID(context.Context, uint, string) (*d return nil, nil } +func (r *reindexRepo) GetActiveFileObjectsByIDs(_ context.Context, userID uint, fileIDs []string) ([]domainconversation.FileObject, error) { + wanted := make(map[string]struct{}, len(fileIDs)) + for _, fileID := range fileIDs { + wanted[fileID] = struct{}{} + } + results := make([]domainconversation.FileObject, 0, len(fileIDs)) + for _, file := range r.files { + if file.UserID != userID { + continue + } + if _, ok := wanted[file.FileID]; ok { + results = append(results, file) + } + } + return results, nil +} + func (r *reindexRepo) GetFileObjectProcessingByObjectID(context.Context, uint) (*domainconversation.FileObjectProcessing, error) { return nil, nil } -func (r *reindexRepo) ClaimFileEmbedding(context.Context, uint, string, string) (bool, error) { +func (r *reindexRepo) QueueFileEmbedding(_ context.Context, _ uint, fileID string, _ string) (bool, error) { + r.claimedFileIDs = append(r.claimedFileIDs, fileID) + return true, nil +} + +func (r *reindexRepo) ClaimFileEmbedding(_ context.Context, _ uint, fileID string, _ string) (bool, error) { + r.claimedFileIDs = append(r.claimedFileIDs, fileID) return true, nil } diff --git a/backend/internal/application/knowledgebase/service.go b/backend/internal/application/knowledgebase/service.go index 7e08c821f..47a5f22e0 100644 --- a/backend/internal/application/knowledgebase/service.go +++ b/backend/internal/application/knowledgebase/service.go @@ -6,6 +6,7 @@ import ( "strings" "unicode/utf8" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" domainknowledgebase "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/knowledgebase" @@ -29,6 +30,7 @@ type Service struct { fileCleaner fileCleaner fileOpener fileContentOpener fileUploader fileUploader + fileEmbedder fileEmbeddingSubmitter logger *zap.Logger } @@ -48,6 +50,11 @@ type fileUploader interface { UploadFile(ctx context.Context, input appupload.UploadFileInput) (*appupload.UploadFileResult, error) } +type fileEmbeddingSubmitter interface { + SubmitFileEmbeddings(ctx context.Context, userID uint, fileIDs []string) (appembedding.TargetedSubmissionResult, error) + ResolveFileVectorizationCapabilities(ctx context.Context, files []domainconversation.FileObject) map[string]appembedding.FileVectorizationCapability +} + // DeleteResult 描述知识库删除及其可选文件清理结果。 type DeleteResult struct { DeletedFileCount int @@ -78,11 +85,39 @@ func (s *Service) SetFileUploader(uploader fileUploader) { s.fileUploader = uploader } +// SetFileEmbeddingSubmitter 注入平台资料使用的统一向量化队列。 +func (s *Service) SetFileEmbeddingSubmitter(submitter fileEmbeddingSubmitter) { + s.fileEmbedder = submitter +} + // SetLogger 注入结构化日志记录器。 func (s *Service) SetLogger(logger *zap.Logger) { s.logger = logger } +// SubmitPlatformFileEmbeddings 提交指定平台资料的向量化任务。 +// 管理员路由负责角色校验，owner 固定为 0，避免触碰个人文件。 +func (s *Service) SubmitPlatformFileEmbeddings(ctx context.Context, actorUserID uint, fileIDs []string) (appembedding.TargetedSubmissionResult, error) { + if actorUserID == 0 { + return appembedding.TargetedSubmissionResult{}, ErrInvalidKnowledgeBase + } + if s.fileEmbedder == nil { + return appembedding.TargetedSubmissionResult{}, appembedding.ErrEmbeddingServiceNotConfigured + } + return s.fileEmbedder.SubmitFileEmbeddings(ctx, 0, fileIDs) +} + +// ResolveFileVectorizationCapabilities 返回知识库文件显式向量化能力的后端事实状态。 +func (s *Service) ResolveFileVectorizationCapabilities( + ctx context.Context, + files []domainconversation.FileObject, +) map[string]appembedding.FileVectorizationCapability { + if s.fileEmbedder == nil { + return map[string]appembedding.FileVectorizationCapability{} + } + return s.fileEmbedder.ResolveFileVectorizationCapabilities(ctx, files) +} + // AuditInput 描述知识库审计写入。 type AuditInput struct { UserID uint diff --git a/backend/internal/application/processing/service.go b/backend/internal/application/processing/service.go index f883d0665..7dd54fe5b 100644 --- a/backend/internal/application/processing/service.go +++ b/backend/internal/application/processing/service.go @@ -41,26 +41,28 @@ var ( // FileProcessingStatusDTO 文件处理状态响应数据。 type FileProcessingStatusDTO struct { - FileID string - DetectedMIME string - FileCategory string - ProcessingStatus string - ProcessingReady bool - ExtractStatus string - EmbedStatus string - PreviewText string - OCRUsed bool - RAGReady bool - RAGReason string - ErrorCode string - ErrorMessage string - ExtractChars int - ExtractPages int - ChunkCount int - EmbedError string - StartedAt *time.Time - CompletedAt *time.Time - UpdatedAt time.Time + FileID string + DetectedMIME string + FileCategory string + ProcessingStatus string + ProcessingReady bool + ExtractStatus string + EmbedStatus string + PreviewText string + OCRUsed bool + RAGReady bool + RAGReason string + ErrorCode string + ErrorMessage string + ExtractChars int + ExtractPages int + ChunkCount int + EmbedError string + CanVectorize bool + VectorizationReason string + StartedAt *time.Time + CompletedAt *time.Time + UpdatedAt time.Time } // ReadyFileResult 表示等待文件处理完成后的可消费结果。 @@ -135,6 +137,86 @@ func (s *Service) StartBackgroundWorkers(ctx context.Context) { s.logger.Warn("create_file_processing_group_failed", zap.Error(err)) } go s.runFileProcessingWorker(ctx, consumerName) + if s.embeddingSvc != nil { + for workerIndex := range appembedding.WorkerConcurrency { + go s.runFileEmbeddingWorker(ctx, fmt.Sprintf("%s-embedding-%d", consumerName, workerIndex)) + } + } +} + +// SubmitFileEmbeddings 将显式向量化任务提交到独立的可恢复队列。 +// 自动向量化开关仅控制上传后的自动触发，不影响此显式操作。 +func (s *Service) SubmitFileEmbeddings( + ctx context.Context, + userID uint, + fileIDs []string, +) (appembedding.TargetedSubmissionResult, error) { + result := appembedding.TargetedSubmissionResult{ + SubmittedFileIDs: []string{}, + Skipped: []appembedding.TargetedFileSkip{}, + } + if s == nil || s.embeddingSvc == nil || s.cache == nil { + return result, appembedding.ErrEmbeddingServiceNotConfigured + } + plan, err := s.embeddingSvc.PlanFiles(ctx, userID, fileIDs) + if err != nil { + return result, err + } + result.Skipped = append(result.Skipped, plan.Skipped...) + for _, job := range plan.Jobs { + queued, queueErr := s.embeddingSvc.QueueTargetedJob(ctx, job) + if queueErr != nil { + result.Skipped = append(result.Skipped, appembedding.TargetedFileSkip{ + FileID: job.FileID, + Reason: appembedding.SkipReasonSubmitFailed, + }) + if s.logger != nil { + s.logger.Warn("queue_embedding_state_failed", zap.Uint("user_id", job.UserID), zap.String("file_id", job.FileID), zap.Error(queueErr)) + } + continue + } + if !queued { + result.Skipped = append(result.Skipped, appembedding.TargetedFileSkip{ + FileID: job.FileID, + Reason: appembedding.SkipReasonProcessing, + }) + continue + } + err = s.cache.EnqueueFileEmbedding( + ctx, + job.UserID, + job.FileID, + job.EmbeddingSignature, + job.EmbeddingHost, + ) + if err == nil { + result.SubmittedFileIDs = append(result.SubmittedFileIDs, job.FileID) + continue + } + if releaseErr := s.embeddingSvc.FailTargetedJob(ctx, job, "embedding queue is unavailable"); releaseErr != nil && s.logger != nil { + s.logger.Warn("release_unqueued_embedding_failed", zap.Uint("user_id", job.UserID), zap.String("file_id", job.FileID), zap.Error(errors.Join(err, releaseErr))) + } + result.Skipped = append(result.Skipped, appembedding.TargetedFileSkip{ + FileID: job.FileID, + Reason: appembedding.SkipReasonQueueBusy, + }) + } + return result, nil +} + +// ResolveFileVectorizationCapabilities 批量解析文件是否允许显式向量化。 +func (s *Service) ResolveFileVectorizationCapabilities( + ctx context.Context, + files []domainconversation.FileObject, +) map[string]appembedding.FileVectorizationCapability { + if s == nil || s.embeddingSvc == nil { + capabilities := make(map[string]appembedding.FileVectorizationCapability, len(files)) + for i := range files { + capabilities[files[i].FileID] = appembedding.FileVectorizationCapability{Reason: "embedding_service_unavailable"} + } + return capabilities + } + return s.embeddingSvc.ResolveFileVectorizationCapabilities(ctx, files) } // InitializeUploadedFile 初始化新上传文件的处理状态。 @@ -403,6 +485,10 @@ func (s *Service) GetFileProcessingStatus(ctx context.Context, userID uint, file return nil, err } result := fileProcessingStatusFromFileObject(fileObj) + if capability, ok := s.ResolveFileVectorizationCapabilities(ctx, []domainconversation.FileObject{*fileObj})[fileObj.FileID]; ok { + result.CanVectorize = capability.CanVectorize + result.VectorizationReason = capability.Reason + } return &result, nil } @@ -433,10 +519,16 @@ func (s *Service) GetFileProcessingStatuses(ctx context.Context, userID uint, fi for i := range fileObjects { filesByID[fileObjects[i].FileID] = &fileObjects[i] } + capabilities := s.ResolveFileVectorizationCapabilities(ctx, fileObjects) results := make([]FileProcessingStatusDTO, 0, len(fileObjects)) for _, fileID := range normalizedIDs { if fileObj := filesByID[fileID]; fileObj != nil { - results = append(results, fileProcessingStatusFromFileObject(fileObj)) + dto := fileProcessingStatusFromFileObject(fileObj) + if capability, ok := capabilities[fileID]; ok { + dto.CanVectorize = capability.CanVectorize + dto.VectorizationReason = capability.Reason + } + results = append(results, dto) } } return results, nil @@ -518,7 +610,36 @@ func (s *Service) WaitUntilReady( } } +type fileMessageReader func(context.Context, string) ([]repository.FileProcessingMessage, error) +type fileMessageHandler func(context.Context, string, repository.FileProcessingMessage) + func (s *Service) runFileProcessingWorker(ctx context.Context, consumerName string) { + s.runFileQueueWorker( + ctx, + consumerName, + s.cache.ClaimTimedOutFileProcessingMessages, + s.cache.ReadFileProcessingMessages, + s.handleProcessingMessage, + ) +} + +func (s *Service) runFileEmbeddingWorker(ctx context.Context, consumerName string) { + s.runFileQueueWorker( + ctx, + consumerName, + s.cache.ClaimTimedOutFileEmbeddingMessages, + s.cache.ReadFileEmbeddingMessages, + s.handleEmbeddingMessage, + ) +} + +func (s *Service) runFileQueueWorker( + ctx context.Context, + consumerName string, + claim fileMessageReader, + read fileMessageReader, + handle fileMessageHandler, +) { for { select { case <-ctx.Done(): @@ -526,7 +647,7 @@ func (s *Service) runFileProcessingWorker(ctx context.Context, consumerName stri default: } - claimed, claimErr := s.cache.ClaimTimedOutFileProcessingMessages(ctx, consumerName) + claimed, claimErr := claim(ctx, consumerName) if claimErr != nil { if ctx.Err() != nil { return @@ -536,12 +657,12 @@ func (s *Service) runFileProcessingWorker(ctx context.Context, consumerName stri } } else if len(claimed) > 0 { for _, msg := range claimed { - s.handleProcessingMessage(ctx, consumerName, msg) + handle(ctx, consumerName, msg) } continue } - messages, err := s.cache.ReadFileProcessingMessages(ctx, consumerName) + messages, err := read(ctx, consumerName) if err != nil { if ctx.Err() != nil { return @@ -557,7 +678,7 @@ func (s *Service) runFileProcessingWorker(ctx context.Context, consumerName stri continue } for _, msg := range messages { - s.handleProcessingMessage(ctx, consumerName, msg) + handle(ctx, consumerName, msg) } } } @@ -567,6 +688,10 @@ func (s *Service) handleProcessingMessage(ctx context.Context, consumerName stri s.settleProcessingMessage(ctx, consumerName, msg) return } + if msg.Kind == repository.FileProcessingKindEmbedding { + s.handleEmbeddingMessage(ctx, consumerName, msg) + return + } attemptID := uuid.NewString() processingCtx, cancelProcessing := context.WithCancel(ctx) @@ -593,7 +718,7 @@ func (s *Service) handleProcessingMessage(ctx context.Context, consumerName stri return default: } - owned, ownershipErr := s.cache.RenewFileProcessingMessageLease(ctx, consumerName, msg.ID) + owned, ownershipErr := s.cache.RenewFileProcessingMessageLease(ctx, consumerName, msg) if ownershipErr != nil || !owned { if ownershipErr != nil && s.logger != nil { s.logger.Warn("verify_file_processing_lease_failed", @@ -691,6 +816,113 @@ func (s *Service) handleProcessingMessage(ctx context.Context, consumerName stri s.settleProcessingMessage(ctx, consumerName, msg) } +func (s *Service) handleEmbeddingMessage(ctx context.Context, consumerName string, msg repository.FileProcessingMessage) { + if s.embeddingSvc == nil { + return + } + job := appembedding.TargetedJob{ + FileID: msg.FileID, + UserID: msg.UserID, + EmbeddingSignature: msg.EmbeddingSignature, + EmbeddingHost: msg.EmbeddingHost, + } + + processingCtx, cancelProcessing := context.WithCancel(ctx) + leaseCtx, stopLease := context.WithCancel(ctx) + leaseDone := make(chan struct{}) + ownershipLost := make(chan struct{}) + go s.renewProcessingMessageLease( + leaseCtx, + leaseDone, + ownershipLost, + cancelProcessing, + consumerName, + msg, + ) + err := s.embeddingSvc.ProcessTargetedJob(processingCtx, job) + stopLease() + <-leaseDone + cancelProcessing() + if ctx.Err() != nil { + return + } + select { + case <-ownershipLost: + return + default: + } + owned, ownershipErr := s.cache.RenewFileProcessingMessageLease(ctx, consumerName, msg) + if ownershipErr != nil || !owned { + if ownershipErr != nil && s.logger != nil { + s.logger.Warn("verify_embedding_message_lease_failed", + zap.Uint("user_id", msg.UserID), + zap.String("file_id", msg.FileID), + zap.String("message_id", msg.ID), + zap.Error(ownershipErr), + ) + } + return + } + if err == nil { + s.settleProcessingMessage(ctx, consumerName, msg) + return + } + if msg.Retry < fileProcessingMaxRetries { + if retryStateErr := s.embeddingSvc.RequeueTargetedJob(ctx, job, err.Error()); retryStateErr != nil { + if s.logger != nil { + s.logger.Warn("mark_requeued_embedding_failed", + zap.Uint("user_id", msg.UserID), + zap.String("file_id", msg.FileID), + zap.Int("retry", msg.Retry+1), + zap.Error(retryStateErr), + ) + } + return + } + settled, requeueErr := s.cache.RequeueFileProcessingMessage( + ctx, + consumerName, + msg, + msg.Retry+1, + err.Error(), + ) + if requeueErr != nil || !settled { + if s.logger != nil { + s.logger.Warn("requeue_embedding_message_failed", + zap.Uint("user_id", msg.UserID), + zap.String("file_id", msg.FileID), + zap.Int("retry", msg.Retry), + zap.Error(requeueErr), + ) + } + return + } + } else { + if failErr := s.embeddingSvc.FailTargetedJob(ctx, job, err.Error()); failErr != nil { + if s.logger != nil { + s.logger.Warn("finalize_queued_embedding_failed", + zap.Uint("user_id", msg.UserID), + zap.String("file_id", msg.FileID), + zap.Int("retry", msg.Retry), + zap.Error(failErr), + ) + } + return + } + if !s.deadLetterProcessingMessage(ctx, consumerName, msg, err.Error()) { + return + } + } + if s.logger != nil { + s.logger.Warn("process_queued_embedding_failed", + zap.Uint("user_id", msg.UserID), + zap.String("file_id", msg.FileID), + zap.Int("retry", msg.Retry), + zap.Error(err), + ) + } +} + func (s *Service) renewProcessingMessageLease( ctx context.Context, done chan<- struct{}, @@ -707,7 +939,7 @@ func (s *Service) renewProcessingMessageLease( case <-ctx.Done(): return case <-ticker.C: - owned, err := s.cache.RenewFileProcessingMessageLease(ctx, consumerName, msg.ID) + owned, err := s.cache.RenewFileProcessingMessageLease(ctx, consumerName, msg) if err != nil && ctx.Err() == nil && s.logger != nil { s.logger.Warn("renew_file_processing_lease_failed", zap.Uint("user_id", msg.UserID), @@ -730,7 +962,7 @@ func (s *Service) settleProcessingMessage( consumerName string, msg repository.FileProcessingMessage, ) { - settled, err := s.cache.SettleFileProcessingMessage(ctx, consumerName, msg.ID) + settled, err := s.cache.SettleFileProcessingMessage(ctx, consumerName, msg) if err != nil || !settled { if s.logger != nil { s.logger.Warn("settle_file_processing_message_failed", diff --git a/backend/internal/application/processing/service_embedding_test.go b/backend/internal/application/processing/service_embedding_test.go new file mode 100644 index 000000000..9327862ba --- /dev/null +++ b/backend/internal/application/processing/service_embedding_test.go @@ -0,0 +1,232 @@ +package processing + +import ( + "context" + "errors" + "testing" + + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + memorycache "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/cache/memory" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + infraembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/embedding" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/security" +) + +type targetedEmbeddingRepositoryStub struct { + files []domainconversation.FileObject + queueErrors map[string]error + vectorError error + statusHistory map[string][]string +} + +func (r *targetedEmbeddingRepositoryStub) VectorStoreAvailable(context.Context) (bool, error) { + if r.vectorError != nil { + return false, r.vectorError + } + return true, nil +} + +func (r *targetedEmbeddingRepositoryStub) GetActiveFileObjectByID(_ context.Context, userID uint, fileID string) (*domainconversation.FileObject, error) { + for i := range r.files { + if r.files[i].UserID == userID && r.files[i].FileID == fileID { + file := r.files[i] + return &file, nil + } + } + return nil, nil +} + +func (r *targetedEmbeddingRepositoryStub) GetActiveFileObjectsByIDs(_ context.Context, userID uint, fileIDs []string) ([]domainconversation.FileObject, error) { + wanted := make(map[string]struct{}, len(fileIDs)) + for _, fileID := range fileIDs { + wanted[fileID] = struct{}{} + } + files := make([]domainconversation.FileObject, 0, len(fileIDs)) + for i := range r.files { + if r.files[i].UserID != userID { + continue + } + if _, ok := wanted[r.files[i].FileID]; ok { + files = append(files, r.files[i]) + } + } + return files, nil +} + +func (*targetedEmbeddingRepositoryStub) GetFileObjectProcessingByObjectID(context.Context, uint) (*domainconversation.FileObjectProcessing, error) { + return nil, nil +} + +func (r *targetedEmbeddingRepositoryStub) QueueFileEmbedding(_ context.Context, _ uint, fileID, _ string) (bool, error) { + if err := r.queueErrors[fileID]; err != nil { + return false, err + } + return true, nil +} + +func (*targetedEmbeddingRepositoryStub) ClaimFileEmbedding(context.Context, uint, string, string) (bool, error) { + return true, nil +} + +func (r *targetedEmbeddingRepositoryStub) UpdateFileObjectEmbedStatus(_ context.Context, _ uint, fileID, _ string, status, _ string) (bool, error) { + if r.statusHistory == nil { + r.statusHistory = make(map[string][]string) + } + r.statusHistory[fileID] = append(r.statusHistory[fileID], status) + return true, nil +} + +func (*targetedEmbeddingRepositoryStub) UpdateFileObjectChunkCount(context.Context, uint, string, int) (bool, error) { + return true, nil +} + +func (*targetedEmbeddingRepositoryStub) ReplaceFileChunks(context.Context, uint, string, []domainconversation.FileChunk, [][]float32) (bool, error) { + return true, nil +} + +func (*targetedEmbeddingRepositoryStub) MarkEmbeddedFilesStale(context.Context, string) (int64, error) { + return 0, nil +} + +func (*targetedEmbeddingRepositoryStub) CountFilesByEmbedStatus(context.Context, string) (int64, error) { + return 0, nil +} + +func (*targetedEmbeddingRepositoryStub) ListFilesForReindex(context.Context, int, uint) ([]domainconversation.FileObject, error) { + return nil, nil +} + +type selectiveEmbeddingQueue struct { + *memorycache.Cache + enqueueErrors map[string]error +} + +func (q *selectiveEmbeddingQueue) EnqueueFileEmbedding( + ctx context.Context, + userID uint, + fileID string, + embeddingSignature string, + embeddingHost string, +) error { + if err := q.enqueueErrors[fileID]; err != nil { + return err + } + return q.Cache.EnqueueFileEmbedding(ctx, userID, fileID, embeddingSignature, embeddingHost) +} + +func TestSubmitFileEmbeddingsKeepsPerFileFailuresIsolated(t *testing.T) { + cfg := targetedEmbeddingTestConfig() + repo := &targetedEmbeddingRepositoryStub{ + files: []domainconversation.FileObject{ + targetedEmbeddingTestFile("submitted"), + targetedEmbeddingTestFile("state_failed"), + targetedEmbeddingTestFile("queue_failed"), + }, + queueErrors: map[string]error{"state_failed": errors.New("database unavailable")}, + } + queue := &selectiveEmbeddingQueue{ + Cache: memorycache.New(), + enqueueErrors: map[string]error{"queue_failed": errors.New("queue unavailable")}, + } + embeddingSvc := appembedding.NewService( + cfg, + repo, + nil, + infraembedding.New(security.OutboundPolicy{}), + nil, + ) + service := NewService(cfg, nil, queue, nil, embeddingSvc, nil, DefaultExtractorVersion) + + result, err := service.SubmitFileEmbeddings( + context.Background(), + 7, + []string{"submitted", "state_failed", "queue_failed"}, + ) + if err != nil { + t.Fatalf("submit embeddings: %v", err) + } + if len(result.SubmittedFileIDs) != 1 || result.SubmittedFileIDs[0] != "submitted" { + t.Fatalf("submitted files = %#v", result.SubmittedFileIDs) + } + wantSkipped := map[string]string{ + "state_failed": appembedding.SkipReasonSubmitFailed, + "queue_failed": appembedding.SkipReasonQueueBusy, + } + if len(result.Skipped) != len(wantSkipped) { + t.Fatalf("skipped files = %#v", result.Skipped) + } + for _, skipped := range result.Skipped { + if wantSkipped[skipped.FileID] != skipped.Reason { + t.Fatalf("skip %s = %s", skipped.FileID, skipped.Reason) + } + } + if history := repo.statusHistory["queue_failed"]; len(history) != 1 || history[0] != "failed" { + t.Fatalf("queue failure status history = %#v", history) + } + messages, err := queue.ReadFileEmbeddingMessages(context.Background(), "embedding_worker") + if err != nil || len(messages) != 1 || messages[0].FileID != "submitted" { + t.Fatalf("queued messages = %#v, err=%v", messages, err) + } +} + +func TestEmbeddingDeadLetterFinalizesFileStatus(t *testing.T) { + cfg := targetedEmbeddingTestConfig() + repo := &targetedEmbeddingRepositoryStub{vectorError: errors.New("vector store unavailable")} + queue := memorycache.New() + embeddingSvc := appembedding.NewService( + cfg, + repo, + nil, + infraembedding.New(security.OutboundPolicy{}), + nil, + ) + service := NewService(cfg, nil, queue, nil, embeddingSvc, nil, DefaultExtractorVersion) + signature := appembedding.ComputeModelSignature(cfg.RAGModel, cfg.EmbeddingOutputDimensions) + if err := queue.EnqueueFileEmbedding(context.Background(), 7, "file_1", signature, cfg.EmbeddingHost); err != nil { + t.Fatalf("enqueue embedding: %v", err) + } + messages, err := queue.ReadFileEmbeddingMessages(context.Background(), "embedding_worker") + if err != nil || len(messages) != 1 { + t.Fatalf("read embedding: messages=%#v err=%v", messages, err) + } + message := messages[0] + message.Retry = fileProcessingMaxRetries + service.handleEmbeddingMessage(context.Background(), "embedding_worker", message) + + if history := repo.statusHistory["file_1"]; len(history) != 1 || history[0] != "failed" { + t.Fatalf("terminal embedding status history = %#v", history) + } + if owned, renewErr := queue.RenewFileProcessingMessageLease(context.Background(), "embedding_worker", message); renewErr != nil || owned { + t.Fatalf("dead-lettered message still owned: owned=%v err=%v", owned, renewErr) + } +} + +func targetedEmbeddingTestConfig() config.Config { + return config.Config{ + EmbeddingEnabled: true, + RAGModel: "text-embedding-test", + EmbeddingHost: "https://embedding.example/v1", + EmbeddingOutputDimensions: 1536, + } +} + +func targetedEmbeddingTestFile(fileID string) domainconversation.FileObject { + return domainconversation.FileObject{ + ID: 1, + UserID: 7, + FileID: fileID, + FileName: fileID + ".md", + MimeType: "text/markdown", + FileCategory: "text", + StoragePath: "uploads/" + fileID + ".md", + Status: "active", + ProcessingReady: true, + ExtractStatus: "ready", + EmbedStatus: "none", + } +} + +var _ repository.EmbeddingRepository = (*targetedEmbeddingRepositoryStub)(nil) +var _ repository.FileProcessingQueueRepository = (*selectiveEmbeddingQueue)(nil) diff --git a/backend/internal/domain/conversation/types.go b/backend/internal/domain/conversation/types.go index 3f64e21e4..1ec7b5b20 100644 --- a/backend/internal/domain/conversation/types.go +++ b/backend/internal/domain/conversation/types.go @@ -298,6 +298,7 @@ const ( FileProcessingStatusQueued = "queued" FileProcessingStatusExtracting = "extracting" FileProcessingStatusEmbedding = "embedding" + FileSubprocessStatusQueued = "queued" FileSubprocessStatusProcessing = "processing" ) @@ -311,6 +312,7 @@ func IsFileProcessing(file FileObject) bool { return true default: return file.ExtractStatus == FileSubprocessStatusProcessing || + file.EmbedStatus == FileSubprocessStatusQueued || file.EmbedStatus == FileSubprocessStatusProcessing } } diff --git a/backend/internal/infra/cache/memory/cache.go b/backend/internal/infra/cache/memory/cache.go index b442c2b39..50d3e26df 100644 --- a/backend/internal/infra/cache/memory/cache.go +++ b/backend/internal/infra/cache/memory/cache.go @@ -17,11 +17,9 @@ type Cache struct { userSettings map[string]expiringString userSettingVersions map[string]expiringString - fileSeq int64 - fileQueue []repository.FileProcessingMessage - fileInflight map[string]fileProcessingLease - fileDLQ []repository.FileProcessingMessage - fileNotify chan struct{} + fileSeq int64 + fileProcessingQueue fileQueueState + fileEmbeddingQueue fileQueueState rag map[string]expiringRAG @@ -57,8 +55,8 @@ func New() *Cache { settings: map[string]expiringString{}, userSettings: map[string]expiringString{}, userSettingVersions: map[string]expiringString{}, - fileInflight: map[string]fileProcessingLease{}, - fileNotify: make(chan struct{}), + fileProcessingQueue: newFileQueueState(), + fileEmbeddingQueue: newFileQueueState(), rag: map[string]expiringRAG{}, streams: map[string]*generationStream{}, streamNotify: make(chan struct{}), diff --git a/backend/internal/infra/cache/memory/file_queue.go b/backend/internal/infra/cache/memory/file_queue.go index 9815c8014..8b4511ae2 100644 --- a/backend/internal/infra/cache/memory/file_queue.go +++ b/backend/internal/infra/cache/memory/file_queue.go @@ -23,12 +23,59 @@ type fileProcessingLease struct { message repository.FileProcessingMessage } +type fileQueueState struct { + queue []repository.FileProcessingMessage + inflight map[string]fileProcessingLease + dlq []repository.FileProcessingMessage + notify chan struct{} +} + +func newFileQueueState() fileQueueState { + return fileQueueState{ + inflight: map[string]fileProcessingLease{}, + notify: make(chan struct{}), + } +} + func (c *Cache) InitFileProcessingStream(ctx context.Context) error { return ctx.Err() } func (c *Cache) EnqueueFileProcessing(ctx context.Context, userID uint, fileID string, retry int, lastError string) error { - if c == nil || strings.TrimSpace(fileID) == "" { + return c.enqueueFileMessage(ctx, repository.FileProcessingMessage{ + UserID: userID, + FileID: fileID, + Retry: retry, + LastError: lastError, + Queue: repository.FileProcessingQueueDefault, + }) +} + +func (c *Cache) EnqueueFileEmbedding( + ctx context.Context, + userID uint, + fileID string, + embeddingSignature string, + embeddingHost string, +) error { + fileID = strings.TrimSpace(fileID) + embeddingSignature = strings.TrimSpace(embeddingSignature) + embeddingHost = strings.TrimRight(strings.TrimSpace(embeddingHost), "/") + if fileID == "" || embeddingSignature == "" || embeddingHost == "" { + return repository.ErrInvalidInput + } + return c.enqueueFileMessage(ctx, repository.FileProcessingMessage{ + UserID: userID, + FileID: fileID, + Kind: repository.FileProcessingKindEmbedding, + Queue: repository.FileProcessingQueueEmbedding, + EmbeddingSignature: embeddingSignature, + EmbeddingHost: embeddingHost, + }) +} + +func (c *Cache) enqueueFileMessage(ctx context.Context, message repository.FileProcessingMessage) error { + if c == nil || strings.TrimSpace(message.FileID) == "" { return nil } if err := ctx.Err(); err != nil { @@ -36,27 +83,33 @@ func (c *Cache) EnqueueFileProcessing(ctx context.Context, userID uint, fileID s } c.mu.Lock() now := time.Now() - if len(c.fileQueue) >= maxFileQueueLength { + state := c.fileQueueState(message.Queue) + if len(state.queue) >= maxFileQueueLength { c.maybeSweepLocked(now) c.mu.Unlock() return repository.ErrFileProcessingQueueFull } c.fileSeq++ - msg := repository.FileProcessingMessage{ - ID: strconv.FormatInt(c.fileSeq, 10), - UserID: userID, - FileID: strings.TrimSpace(fileID), - Retry: retry, - LastError: strings.TrimSpace(lastError), - } - c.fileQueue = append(c.fileQueue, msg) - c.notifyFileQueueLocked() + message.ID = strconv.FormatInt(c.fileSeq, 10) + message.FileID = strings.TrimSpace(message.FileID) + message.LastError = strings.TrimSpace(message.LastError) + msg := message + state.queue = append(state.queue, msg) + notifyFileQueueLocked(state) c.maybeSweepLocked(now) c.mu.Unlock() return nil } func (c *Cache) ClaimTimedOutFileProcessingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.claimTimedOutFileMessages(ctx, consumerName, repository.FileProcessingQueueDefault) +} + +func (c *Cache) ClaimTimedOutFileEmbeddingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.claimTimedOutFileMessages(ctx, consumerName, repository.FileProcessingQueueEmbedding) +} + +func (c *Cache) claimTimedOutFileMessages(ctx context.Context, consumerName string, queue repository.FileProcessingQueue) ([]repository.FileProcessingMessage, error) { if c == nil || strings.TrimSpace(consumerName) == "" { return nil, nil } @@ -65,10 +118,11 @@ func (c *Cache) ClaimTimedOutFileProcessingMessages(ctx context.Context, consume } c.mu.Lock() defer c.mu.Unlock() + state := c.fileQueueState(queue) now := time.Now() var candidateID string var candidate fileProcessingLease - for messageID, lease := range c.fileInflight { + for messageID, lease := range state.inflight { if now.Sub(lease.leasedAt) < fileProcessingMinIdle { continue } @@ -83,11 +137,19 @@ func (c *Cache) ClaimTimedOutFileProcessingMessages(ctx context.Context, consume candidate.consumerName = strings.TrimSpace(consumerName) candidate.leasedAt = now candidate.message.Reclaimed = true - c.fileInflight[candidateID] = candidate + state.inflight[candidateID] = candidate return []repository.FileProcessingMessage{candidate.message}, nil } func (c *Cache) ReadFileProcessingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.readFileMessages(ctx, consumerName, repository.FileProcessingQueueDefault) +} + +func (c *Cache) ReadFileEmbeddingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.readFileMessages(ctx, consumerName, repository.FileProcessingQueueEmbedding) +} + +func (c *Cache) readFileMessages(ctx context.Context, consumerName string, queue repository.FileProcessingQueue) ([]repository.FileProcessingMessage, error) { if c == nil || strings.TrimSpace(consumerName) == "" { return nil, nil } @@ -101,10 +163,11 @@ func (c *Cache) ReadFileProcessingMessages(ctx context.Context, consumerName str return nil, err } c.mu.Lock() - if len(c.fileQueue) > 0 { - msg := c.fileQueue[0] - c.fileQueue = c.fileQueue[1:] - c.fileInflight[msg.ID] = fileProcessingLease{ + state := c.fileQueueState(queue) + if len(state.queue) > 0 { + msg := state.queue[0] + state.queue = state.queue[1:] + state.inflight[msg.ID] = fileProcessingLease{ consumerName: strings.TrimSpace(consumerName), leasedAt: time.Now(), message: msg, @@ -112,7 +175,7 @@ func (c *Cache) ReadFileProcessingMessages(ctx context.Context, consumerName str c.mu.Unlock() return []repository.FileProcessingMessage{msg}, nil } - notify := c.fileNotify + notify := state.notify c.mu.Unlock() select { case <-ctx.Done(): @@ -124,7 +187,7 @@ func (c *Cache) ReadFileProcessingMessages(ctx context.Context, consumerName str } } -func (c *Cache) RenewFileProcessingMessageLease(ctx context.Context, consumerName, messageID string) (bool, error) { +func (c *Cache) RenewFileProcessingMessageLease(ctx context.Context, consumerName string, message repository.FileProcessingMessage) (bool, error) { if c == nil { return false, nil } @@ -133,17 +196,18 @@ func (c *Cache) RenewFileProcessingMessageLease(ctx context.Context, consumerNam } c.mu.Lock() defer c.mu.Unlock() - messageID = strings.TrimSpace(messageID) - lease, exists := c.fileInflight[messageID] + state := c.fileQueueState(queueForMessage(message)) + messageID := strings.TrimSpace(message.ID) + lease, exists := state.inflight[messageID] if !exists || lease.consumerName != strings.TrimSpace(consumerName) { return false, nil } lease.leasedAt = time.Now() - c.fileInflight[messageID] = lease + state.inflight[messageID] = lease return true, nil } -func (c *Cache) SettleFileProcessingMessage(ctx context.Context, consumerName, messageID string) (bool, error) { +func (c *Cache) SettleFileProcessingMessage(ctx context.Context, consumerName string, message repository.FileProcessingMessage) (bool, error) { if c == nil { return false, nil } @@ -152,12 +216,13 @@ func (c *Cache) SettleFileProcessingMessage(ctx context.Context, consumerName, m } c.mu.Lock() defer c.mu.Unlock() - messageID = strings.TrimSpace(messageID) - lease, exists := c.fileInflight[messageID] + state := c.fileQueueState(queueForMessage(message)) + messageID := strings.TrimSpace(message.ID) + lease, exists := state.inflight[messageID] if !exists || lease.consumerName != strings.TrimSpace(consumerName) { return false, nil } - delete(c.fileInflight, messageID) + delete(state.inflight, messageID) c.maybeSweepLocked(time.Now()) return true, nil } @@ -177,22 +242,27 @@ func (c *Cache) RequeueFileProcessingMessage( } c.mu.Lock() defer c.mu.Unlock() + state := c.fileQueueState(queueForMessage(message)) messageID := strings.TrimSpace(message.ID) - lease, exists := c.fileInflight[messageID] + lease, exists := state.inflight[messageID] if !exists || lease.consumerName != strings.TrimSpace(consumerName) { return false, nil } // 重入队不受 maxFileQueueLength 限制：消息只是从 inflight 移回队列，总量无净增长。 c.fileSeq++ - c.fileQueue = append(c.fileQueue, repository.FileProcessingMessage{ - ID: strconv.FormatInt(c.fileSeq, 10), - UserID: message.UserID, - FileID: strings.TrimSpace(message.FileID), - Retry: retry, - LastError: strings.TrimSpace(lastError), + state.queue = append(state.queue, repository.FileProcessingMessage{ + ID: strconv.FormatInt(c.fileSeq, 10), + UserID: message.UserID, + FileID: strings.TrimSpace(message.FileID), + Retry: retry, + LastError: strings.TrimSpace(lastError), + Kind: message.Kind, + Queue: queueForMessage(message), + EmbeddingSignature: message.EmbeddingSignature, + EmbeddingHost: message.EmbeddingHost, }) - delete(c.fileInflight, messageID) - c.notifyFileQueueLocked() + delete(state.inflight, messageID) + notifyFileQueueLocked(state) c.maybeSweepLocked(time.Now()) return true, nil } @@ -211,28 +281,50 @@ func (c *Cache) DeadLetterFileProcessingMessage( } c.mu.Lock() defer c.mu.Unlock() + state := c.fileQueueState(queueForMessage(message)) messageID := strings.TrimSpace(message.ID) - lease, exists := c.fileInflight[messageID] + lease, exists := state.inflight[messageID] if !exists || lease.consumerName != strings.TrimSpace(consumerName) { return false, nil } c.fileSeq++ - c.fileDLQ = append(c.fileDLQ, repository.FileProcessingMessage{ - ID: "dlq-" + strconv.FormatInt(c.fileSeq, 10), - UserID: message.UserID, - FileID: strings.TrimSpace(message.FileID), - Retry: message.Retry, - LastError: strings.TrimSpace(lastError), + state.dlq = append(state.dlq, repository.FileProcessingMessage{ + ID: "dlq-" + strconv.FormatInt(c.fileSeq, 10), + UserID: message.UserID, + FileID: strings.TrimSpace(message.FileID), + Retry: message.Retry, + LastError: strings.TrimSpace(lastError), + Kind: message.Kind, + Queue: queueForMessage(message), + EmbeddingSignature: message.EmbeddingSignature, + EmbeddingHost: message.EmbeddingHost, }) - if len(c.fileDLQ) > 10_000 { - c.fileDLQ = append([]repository.FileProcessingMessage(nil), c.fileDLQ[len(c.fileDLQ)-10_000:]...) + if len(state.dlq) > 10_000 { + state.dlq = append([]repository.FileProcessingMessage(nil), state.dlq[len(state.dlq)-10_000:]...) } - delete(c.fileInflight, messageID) + delete(state.inflight, messageID) c.maybeSweepLocked(time.Now()) return true, nil } -func (c *Cache) notifyFileQueueLocked() { - close(c.fileNotify) - c.fileNotify = make(chan struct{}) +func (c *Cache) fileQueueState(queue repository.FileProcessingQueue) *fileQueueState { + if queue == repository.FileProcessingQueueEmbedding { + return &c.fileEmbeddingQueue + } + return &c.fileProcessingQueue +} + +func queueForMessage(message repository.FileProcessingMessage) repository.FileProcessingQueue { + if message.Queue != "" { + return message.Queue + } + if message.Kind == repository.FileProcessingKindEmbedding { + return repository.FileProcessingQueueEmbedding + } + return repository.FileProcessingQueueDefault +} + +func notifyFileQueueLocked(state *fileQueueState) { + close(state.notify) + state.notify = make(chan struct{}) } diff --git a/backend/internal/infra/cache/memory/file_queue_test.go b/backend/internal/infra/cache/memory/file_queue_test.go index e7f8c4826..7c87d524f 100644 --- a/backend/internal/infra/cache/memory/file_queue_test.go +++ b/backend/internal/infra/cache/memory/file_queue_test.go @@ -25,25 +25,25 @@ func TestFileProcessingLeaseTransfersOnlyAfterTimeout(t *testing.T) { } cache.mu.Lock() - lease := cache.fileInflight[message.ID] + lease := cache.fileProcessingQueue.inflight[message.ID] lease.leasedAt = time.Now().Add(-fileProcessingMinIdle - time.Second) - cache.fileInflight[message.ID] = lease + cache.fileProcessingQueue.inflight[message.ID] = lease cache.mu.Unlock() claimed, err := cache.ClaimTimedOutFileProcessingMessages(ctx, "worker_b") if err != nil || len(claimed) != 1 || !claimed[0].Reclaimed { t.Fatalf("claim expired lease: claimed=%#v err=%v", claimed, err) } - if owned, err := cache.RenewFileProcessingMessageLease(ctx, "worker_a", message.ID); err != nil || owned { + if owned, err := cache.RenewFileProcessingMessageLease(ctx, "worker_a", message); err != nil || owned { t.Fatalf("previous owner retained lease: owned=%v err=%v", owned, err) } - if settled, err := cache.SettleFileProcessingMessage(ctx, "worker_a", message.ID); err != nil || settled { + if settled, err := cache.SettleFileProcessingMessage(ctx, "worker_a", message); err != nil || settled { t.Fatalf("previous owner settled transferred message: settled=%v err=%v", settled, err) } - if owned, err := cache.RenewFileProcessingMessageLease(ctx, "worker_b", message.ID); err != nil || !owned { + if owned, err := cache.RenewFileProcessingMessageLease(ctx, "worker_b", message); err != nil || !owned { t.Fatalf("new owner does not own lease: owned=%v err=%v", owned, err) } - if settled, err := cache.SettleFileProcessingMessage(ctx, "worker_b", message.ID); err != nil || !settled { + if settled, err := cache.SettleFileProcessingMessage(ctx, "worker_b", message); err != nil || !settled { t.Fatalf("new owner failed to settle message: settled=%v err=%v", settled, err) } } @@ -62,7 +62,7 @@ func TestEnqueueFileProcessingRejectsWhenQueueFull(t *testing.T) { } cache.mu.Lock() - cache.fileQueue = make([]repository.FileProcessingMessage, maxFileQueueLength) + cache.fileProcessingQueue.queue = make([]repository.FileProcessingMessage, maxFileQueueLength) cache.mu.Unlock() if err := cache.EnqueueFileProcessing(ctx, 2, "file_overflow", 0, ""); !errors.Is(err, repository.ErrFileProcessingQueueFull) { @@ -74,3 +74,63 @@ func TestEnqueueFileProcessingRejectsWhenQueueFull(t *testing.T) { t.Fatalf("requeue must bypass queue length limit: requeued=%v err=%v", requeued, err) } } + +func TestFileEmbeddingMessagePreservesJobMetadataWhenRequeued(t *testing.T) { + cache := New() + ctx := context.Background() + if err := cache.EnqueueFileEmbedding(ctx, 0, "file_platform", "signature@1536", "https://embedding.example/v1/"); err != nil { + t.Fatalf("enqueue embedding: %v", err) + } + + messages, err := cache.ReadFileEmbeddingMessages(ctx, "worker_a") + if err != nil || len(messages) != 1 { + t.Fatalf("read embedding message: messages=%#v err=%v", messages, err) + } + message := messages[0] + if message.Kind != repository.FileProcessingKindEmbedding || message.UserID != 0 || + message.EmbeddingSignature != "signature@1536" || message.EmbeddingHost != "https://embedding.example/v1" { + t.Fatalf("unexpected embedding message: %#v", message) + } + + requeued, err := cache.RequeueFileProcessingMessage(ctx, "worker_a", message, 1, "temporary failure") + if err != nil || !requeued { + t.Fatalf("requeue embedding message: requeued=%v err=%v", requeued, err) + } + messages, err = cache.ReadFileEmbeddingMessages(ctx, "worker_b") + if err != nil || len(messages) != 1 { + t.Fatalf("read requeued embedding message: messages=%#v err=%v", messages, err) + } + retried := messages[0] + if retried.Kind != message.Kind || retried.UserID != message.UserID || retried.FileID != message.FileID || + retried.EmbeddingSignature != message.EmbeddingSignature || retried.EmbeddingHost != message.EmbeddingHost || + retried.Retry != 1 || retried.LastError != "temporary failure" { + t.Fatalf("requeued embedding metadata changed: %#v", retried) + } +} + +func TestFileEmbeddingQueueIsIsolatedFromExtractionQueue(t *testing.T) { + cache := New() + ctx := context.Background() + if err := cache.EnqueueFileProcessing(ctx, 7, "extract_file", 0, ""); err != nil { + t.Fatalf("enqueue extraction: %v", err) + } + if err := cache.EnqueueFileEmbedding(ctx, 7, "embed_file", "signature@1536", "https://embedding.example/v1"); err != nil { + t.Fatalf("enqueue embedding: %v", err) + } + + extractionMessages, err := cache.ReadFileProcessingMessages(ctx, "extract_worker") + if err != nil || len(extractionMessages) != 1 || extractionMessages[0].FileID != "extract_file" { + t.Fatalf("extraction queue returned %#v, err=%v", extractionMessages, err) + } + if extractionMessages[0].Queue != repository.FileProcessingQueueDefault { + t.Fatalf("extraction message queue = %q", extractionMessages[0].Queue) + } + + embeddingMessages, err := cache.ReadFileEmbeddingMessages(ctx, "embedding_worker") + if err != nil || len(embeddingMessages) != 1 || embeddingMessages[0].FileID != "embed_file" { + t.Fatalf("embedding queue returned %#v, err=%v", embeddingMessages, err) + } + if embeddingMessages[0].Queue != repository.FileProcessingQueueEmbedding { + t.Fatalf("embedding message queue = %q", embeddingMessages[0].Queue) + } +} diff --git a/backend/internal/infra/cache/redis/conversation_cache.go b/backend/internal/infra/cache/redis/conversation_cache.go index 78c0a2ea6..69877b844 100644 --- a/backend/internal/infra/cache/redis/conversation_cache.go +++ b/backend/internal/infra/cache/redis/conversation_cache.go @@ -32,6 +32,9 @@ const ( fileProcessingStreamName = "file_processing_v1" fileProcessingDLQName = "file_processing_v1_dlq" fileProcessingGroupName = "file_processing_workers" + fileEmbeddingStreamName = "file_embedding_v1" + fileEmbeddingDLQName = "file_embedding_v1_dlq" + fileEmbeddingGroupName = "file_embedding_workers" fileProcessingMinIdle = 45 * time.Second fileProcessingDLQMaxLen = 10_000 @@ -39,6 +42,13 @@ const ( generationStreamIndexTTL = 2 * time.Hour ) +type fileQueueConfig struct { + stream string + dlq string + group string + queue repository.FileProcessingQueue +} + // appendGenerationStreamEventScript keeps the event sequence, bounded replay // window, and cumulative visible-text checkpoint consistent in one Redis // round trip. Key TTLs are initialized only when a value is first created; @@ -107,7 +117,10 @@ redis.call( "user_id", ARGV[4], "file_id", ARGV[5], "retry", ARGV[6], - "last_error", ARGV[7] + "last_error", ARGV[7], + "kind", ARGV[8], + "embedding_signature", ARGV[9], + "embedding_host", ARGV[10] ) redis.call("XACK", KEYS[1], ARGV[1], ARGV[3]) redis.call("XDEL", KEYS[1], ARGV[3]) @@ -120,11 +133,14 @@ if #pending == 0 or pending[1][2] ~= ARGV[2] then return 0 end redis.call( - "XADD", KEYS[2], "MAXLEN", ARGV[8], "*", + "XADD", KEYS[2], "MAXLEN", ARGV[11], "*", "user_id", ARGV[4], "file_id", ARGV[5], "retry", ARGV[6], - "last_error", ARGV[7] + "last_error", ARGV[7], + "kind", ARGV[8], + "embedding_signature", ARGV[9], + "embedding_host", ARGV[10] ) redis.call("XACK", KEYS[1], ARGV[1], ARGV[3]) redis.call("XDEL", KEYS[1], ARGV[3]) @@ -168,11 +184,13 @@ func (c *conversationCache) InitFileProcessingStream(ctx context.Context) error if c.client == nil { return nil } - err := c.client.XGroupCreateMkStream(ctx, fileProcessingStreamName, fileProcessingGroupName, "0").Err() - if err != nil && strings.Contains(err.Error(), "BUSYGROUP") { - return nil + for _, queue := range []fileQueueConfig{processingQueueConfig(), embeddingQueueConfig()} { + err := c.client.XGroupCreateMkStream(ctx, queue.stream, queue.group, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return err + } } - return err + return nil } // EnqueueFileProcessing 将文件处理任务推入 Stream 队列。 @@ -195,14 +213,57 @@ func (c *conversationCache) EnqueueFileProcessing(ctx context.Context, userID ui return err } +// EnqueueFileEmbedding 将显式向量化任务推入独立的可恢复 Stream。 +func (c *conversationCache) EnqueueFileEmbedding( + ctx context.Context, + userID uint, + fileID string, + embeddingSignature string, + embeddingHost string, +) error { + fileID = strings.TrimSpace(fileID) + embeddingSignature = strings.TrimSpace(embeddingSignature) + embeddingHost = strings.TrimRight(strings.TrimSpace(embeddingHost), "/") + if fileID == "" || embeddingSignature == "" || embeddingHost == "" { + return repository.ErrInvalidInput + } + if c.client == nil { + return nil + } + _, err := c.client.XAdd(ctx, &redis.XAddArgs{ + Stream: fileEmbeddingStreamName, + Values: map[string]interface{}{ + "user_id": userID, + "file_id": fileID, + "retry": 0, + "kind": repository.FileProcessingKindEmbedding, + "embedding_signature": embeddingSignature, + "embedding_host": embeddingHost, + }, + }).Result() + return err +} + // ClaimTimedOutFileProcessingMessages 认领超时未确认的 pending 任务，避免 worker 重启后任务永久卡住。 func (c *conversationCache) ClaimTimedOutFileProcessingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.claimTimedOutFileMessages(ctx, consumerName, processingQueueConfig()) +} + +func (c *conversationCache) ClaimTimedOutFileEmbeddingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.claimTimedOutFileMessages(ctx, consumerName, embeddingQueueConfig()) +} + +func (c *conversationCache) claimTimedOutFileMessages( + ctx context.Context, + consumerName string, + queue fileQueueConfig, +) ([]repository.FileProcessingMessage, error) { if c.client == nil { return nil, nil } pending, err := c.client.XPendingExt(ctx, &redis.XPendingExtArgs{ - Stream: fileProcessingStreamName, - Group: fileProcessingGroupName, + Stream: queue.stream, + Group: queue.group, Idle: fileProcessingMinIdle, Start: "-", End: "+", @@ -228,8 +289,8 @@ func (c *conversationCache) ClaimTimedOutFileProcessingMessages(ctx context.Cont return nil, nil } claimed, err := c.client.XClaim(ctx, &redis.XClaimArgs{ - Stream: fileProcessingStreamName, - Group: fileProcessingGroupName, + Stream: queue.stream, + Group: queue.group, Consumer: consumerName, MinIdle: fileProcessingMinIdle, Messages: messageIDs, @@ -240,18 +301,30 @@ func (c *conversationCache) ClaimTimedOutFileProcessingMessages(ctx context.Cont } return nil, err } - return c.decodeFileProcessingMessages(ctx, consumerName, claimed, true) + return c.decodeFileProcessingMessages(ctx, consumerName, claimed, true, queue) } // ReadFileProcessingMessages 阻塞读取未处理消息（最多 1 条，5s 超时）。 func (c *conversationCache) ReadFileProcessingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.readFileMessages(ctx, consumerName, processingQueueConfig()) +} + +func (c *conversationCache) ReadFileEmbeddingMessages(ctx context.Context, consumerName string) ([]repository.FileProcessingMessage, error) { + return c.readFileMessages(ctx, consumerName, embeddingQueueConfig()) +} + +func (c *conversationCache) readFileMessages( + ctx context.Context, + consumerName string, + queue fileQueueConfig, +) ([]repository.FileProcessingMessage, error) { if c.client == nil { return nil, nil } streams, err := c.client.XReadGroup(ctx, &redis.XReadGroupArgs{ - Group: fileProcessingGroupName, + Group: queue.group, Consumer: consumerName, - Streams: []string{fileProcessingStreamName, ">"}, + Streams: []string{queue.stream, ">"}, Count: 1, Block: 5 * time.Second, }).Result() @@ -263,7 +336,7 @@ func (c *conversationCache) ReadFileProcessingMessages(ctx context.Context, cons } messages := make([]repository.FileProcessingMessage, 0) for _, stream := range streams { - parsed, parseErr := c.decodeFileProcessingMessages(ctx, consumerName, stream.Messages, false) + parsed, parseErr := c.decodeFileProcessingMessages(ctx, consumerName, stream.Messages, false, queue) if parseErr != nil { return nil, parseErr } @@ -277,12 +350,13 @@ func (c *conversationCache) decodeFileProcessingMessages( consumerName string, messages []redis.XMessage, reclaimed bool, + queue fileQueueConfig, ) ([]repository.FileProcessingMessage, error) { parsedMessages := make([]repository.FileProcessingMessage, 0, len(messages)) for _, msg := range messages { parsed, err := parseFileProcessingMessage(msg) if err != nil { - quarantined, quarantineErr := c.deadLetterInvalidFileProcessingMessage(ctx, consumerName, msg, err) + quarantined, quarantineErr := c.deadLetterInvalidFileProcessingMessage(ctx, consumerName, msg, err, queue) if quarantineErr != nil { return nil, fmt.Errorf("dead-letter invalid file processing message %q: %w", msg.ID, quarantineErr) } @@ -292,14 +366,19 @@ func (c *conversationCache) decodeFileProcessingMessages( continue } parsed.Reclaimed = reclaimed + parsed.Queue = queue.queue parsedMessages = append(parsedMessages, parsed) } return parsedMessages, nil } func parseFileProcessingMessage(msg redis.XMessage) (repository.FileProcessingMessage, error) { + kind := strings.TrimSpace(getOptionalStringVal(msg.Values, "kind")) + if kind != "" && kind != repository.FileProcessingKindEmbedding { + return repository.FileProcessingMessage{}, fmt.Errorf("invalid processing kind %q", kind) + } userID, err := strconv.ParseUint(strings.TrimSpace(getStringVal(msg.Values["user_id"])), 10, strconv.IntSize) - if err != nil || userID == 0 { + if err != nil || (userID == 0 && kind != repository.FileProcessingKindEmbedding) { if err == nil { err = errors.New("must be greater than zero") } @@ -318,13 +397,21 @@ func parseFileProcessingMessage(msg redis.XMessage) (repository.FileProcessingMe if rawLastError, ok := msg.Values["last_error"]; ok { lastError = getStringVal(rawLastError) } + embeddingSignature := strings.TrimSpace(getOptionalStringVal(msg.Values, "embedding_signature")) + embeddingHost := strings.TrimRight(strings.TrimSpace(getOptionalStringVal(msg.Values, "embedding_host")), "/") + if kind == repository.FileProcessingKindEmbedding && (embeddingSignature == "" || embeddingHost == "") { + return repository.FileProcessingMessage{}, errors.New("invalid embedding queue metadata") + } return repository.FileProcessingMessage{ - ID: msg.ID, - UserID: uint(userID), - FileID: strings.TrimSpace(getStringVal(msg.Values["file_id"])), - Retry: retry, - LastError: lastError, + ID: msg.ID, + UserID: uint(userID), + FileID: strings.TrimSpace(getStringVal(msg.Values["file_id"])), + Retry: retry, + LastError: lastError, + Kind: kind, + EmbeddingSignature: embeddingSignature, + EmbeddingHost: embeddingHost, }, nil } @@ -333,6 +420,7 @@ func (c *conversationCache) deadLetterInvalidFileProcessingMessage( consumerName string, message redis.XMessage, parseErr error, + queue fileQueueConfig, ) (bool, error) { lastError := "invalid queue message: " + parseErr.Error() if rawLastError, ok := message.Values["last_error"]; ok { @@ -344,44 +432,49 @@ func (c *conversationCache) deadLetterInvalidFileProcessingMessage( return fileProcessingScriptResult(deadLetterFileProcessingMessageScript.Run( ctx, c.client, - []string{fileProcessingStreamName, fileProcessingDLQName}, - fileProcessingGroupName, + []string{queue.stream, queue.dlq}, + queue.group, consumerName, message.ID, getStringVal(message.Values["user_id"]), getStringVal(message.Values["file_id"]), getStringVal(message.Values["retry"]), truncateStr(lastError, 255), + getOptionalStringVal(message.Values, "kind"), + getOptionalStringVal(message.Values, "embedding_signature"), + getOptionalStringVal(message.Values, "embedding_host"), fileProcessingDLQMaxLen, ).Result()) } // RenewFileProcessingMessageLease 刷新执行中消息的空闲时间，避免长任务被其他 worker 重复认领。 -func (c *conversationCache) RenewFileProcessingMessageLease(ctx context.Context, consumerName, messageID string) (bool, error) { - if c.client == nil || strings.TrimSpace(consumerName) == "" || strings.TrimSpace(messageID) == "" { +func (c *conversationCache) RenewFileProcessingMessageLease(ctx context.Context, consumerName string, message repository.FileProcessingMessage) (bool, error) { + if c.client == nil || strings.TrimSpace(consumerName) == "" || strings.TrimSpace(message.ID) == "" { return true, nil } + queue := redisQueueForMessage(message) return fileProcessingScriptResult(renewFileProcessingLeaseScript.Run( ctx, c.client, - []string{fileProcessingStreamName}, - fileProcessingGroupName, + []string{queue.stream}, + queue.group, consumerName, - messageID, + message.ID, ).Result()) } -func (c *conversationCache) SettleFileProcessingMessage(ctx context.Context, consumerName, messageID string) (bool, error) { +func (c *conversationCache) SettleFileProcessingMessage(ctx context.Context, consumerName string, message repository.FileProcessingMessage) (bool, error) { if c.client == nil { return true, nil } + queue := redisQueueForMessage(message) return fileProcessingScriptResult(settleFileProcessingMessageScript.Run( ctx, c.client, - []string{fileProcessingStreamName}, - fileProcessingGroupName, + []string{queue.stream}, + queue.group, consumerName, - messageID, + message.ID, ).Result()) } @@ -395,17 +488,21 @@ func (c *conversationCache) RequeueFileProcessingMessage( if c.client == nil { return true, nil } + queue := redisQueueForMessage(message) return fileProcessingScriptResult(requeueFileProcessingMessageScript.Run( ctx, c.client, - []string{fileProcessingStreamName}, - fileProcessingGroupName, + []string{queue.stream}, + queue.group, consumerName, message.ID, message.UserID, message.FileID, retry, truncateStr(lastError, 255), + message.Kind, + message.EmbeddingSignature, + message.EmbeddingHost, ).Result()) } @@ -418,21 +515,51 @@ func (c *conversationCache) DeadLetterFileProcessingMessage( if c.client == nil { return true, nil } + queue := redisQueueForMessage(message) return fileProcessingScriptResult(deadLetterFileProcessingMessageScript.Run( ctx, c.client, - []string{fileProcessingStreamName, fileProcessingDLQName}, - fileProcessingGroupName, + []string{queue.stream, queue.dlq}, + queue.group, consumerName, message.ID, message.UserID, message.FileID, message.Retry, truncateStr(lastError, 255), + message.Kind, + message.EmbeddingSignature, + message.EmbeddingHost, fileProcessingDLQMaxLen, ).Result()) } +func processingQueueConfig() fileQueueConfig { + return fileQueueConfig{ + stream: fileProcessingStreamName, + dlq: fileProcessingDLQName, + group: fileProcessingGroupName, + queue: repository.FileProcessingQueueDefault, + } +} + +func embeddingQueueConfig() fileQueueConfig { + return fileQueueConfig{ + stream: fileEmbeddingStreamName, + dlq: fileEmbeddingDLQName, + group: fileEmbeddingGroupName, + queue: repository.FileProcessingQueueEmbedding, + } +} + +func redisQueueForMessage(message repository.FileProcessingMessage) fileQueueConfig { + if message.Queue == repository.FileProcessingQueueEmbedding || + (message.Queue == "" && message.Kind == repository.FileProcessingKindEmbedding) { + return embeddingQueueConfig() + } + return processingQueueConfig() +} + func fileProcessingScriptResult(result interface{}, err error) (bool, error) { if errors.Is(err, redis.Nil) { return false, nil @@ -894,6 +1021,14 @@ func getStringVal(raw interface{}) string { } } +func getOptionalStringVal(values map[string]interface{}, key string) string { + raw, ok := values[key] + if !ok || raw == nil { + return "" + } + return getStringVal(raw) +} + func getInt64Val(raw interface{}) int64 { switch v := raw.(type) { case int64: diff --git a/backend/internal/infra/cache/redis/file_queue_test.go b/backend/internal/infra/cache/redis/file_queue_test.go new file mode 100644 index 000000000..6b485fc74 --- /dev/null +++ b/backend/internal/infra/cache/redis/file_queue_test.go @@ -0,0 +1,49 @@ +package cache + +import ( + "testing" + + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "github.com/go-redis/redis/v8" +) + +func TestParseFileEmbeddingMessagePreservesMetadata(t *testing.T) { + message, err := parseFileProcessingMessage(redis.XMessage{ + ID: "1-0", + Values: map[string]interface{}{ + "user_id": "7", + "file_id": "file_1", + "retry": "1", + "kind": repository.FileProcessingKindEmbedding, + "embedding_signature": "model@1536", + "embedding_host": "https://embedding.example/v1/", + }, + }) + if err != nil { + t.Fatalf("parse embedding message: %v", err) + } + if message.UserID != 7 || message.FileID != "file_1" || message.Retry != 1 || + message.Kind != repository.FileProcessingKindEmbedding || + message.EmbeddingSignature != "model@1536" || + message.EmbeddingHost != "https://embedding.example/v1" { + t.Fatalf("unexpected embedding message: %#v", message) + } +} + +func TestRedisQueueForMessagePreservesLegacySourceQueue(t *testing.T) { + legacy := repository.FileProcessingMessage{ + Kind: repository.FileProcessingKindEmbedding, + Queue: repository.FileProcessingQueueDefault, + } + if queue := redisQueueForMessage(legacy); queue.stream != fileProcessingStreamName { + t.Fatalf("legacy message routed to %q, want %q", queue.stream, fileProcessingStreamName) + } + + current := repository.FileProcessingMessage{ + Kind: repository.FileProcessingKindEmbedding, + Queue: repository.FileProcessingQueueEmbedding, + } + if queue := redisQueueForMessage(current); queue.stream != fileEmbeddingStreamName { + t.Fatalf("embedding message routed to %q, want %q", queue.stream, fileEmbeddingStreamName) + } +} diff --git a/backend/internal/infra/persistence/models/chat.go b/backend/internal/infra/persistence/models/chat.go index 53c53e69f..a1fe69d10 100644 --- a/backend/internal/infra/persistence/models/chat.go +++ b/backend/internal/infra/persistence/models/chat.go @@ -217,7 +217,7 @@ type FileObject struct { OCRUsed bool `gorm:"not null;default:false;comment:是否使用OCR"` RAGReady bool `gorm:"not null;default:false;comment:RAG是否就绪"` RAGReason string `gorm:"size:255;not null;default:'';comment:RAG处理说明"` - EmbedStatus string `gorm:"size:16;not null;default:'none';index:idx_file_objects_embed_status;comment:向量嵌入状态(none/processing/ready/stale/failed)"` + EmbedStatus string `gorm:"size:16;not null;default:'none';index:idx_file_objects_embed_status;comment:向量嵌入状态(none/queued/processing/ready/stale/failed)"` EmbedSignature string `gorm:"size:64;not null;default:'';index:idx_file_objects_embed_signature;comment:当前向量任务所属空间签名"` EmbedError string `gorm:"type:text;not null;default:'';comment:嵌入失败原因"` PageCount int `gorm:"not null;default:0;comment:PDF页数"` diff --git a/backend/internal/infra/persistence/postgres/conversation/repository.go b/backend/internal/infra/persistence/postgres/conversation/repository.go index 0059a9687..53f5f7fd7 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository.go @@ -2427,10 +2427,10 @@ func (r *Repo) GetActiveFileProcessingStatusesByIDs(ctx context.Context, userID } if err := r.db.WithContext(ctx). Select( - "file_id", "detected_mime", "file_category", + "file_id", "file_name", "mime_type", "detected_mime", "file_category", "storage_path", "status", "processing_status", "processing_ready", "processing_error_code", "processing_error_message", "extract_status", "extract_chars", "extract_pages", "preview_text", "ocr_used", - "rag_ready", "rag_reason", "embed_status", "embed_error", "chunk_count", + "rag_ready", "rag_reason", "embed_status", "embed_signature", "embed_error", "chunk_count", "processing_started_at", "processing_completed_at", "updated_at", ). Where("user_id = ? AND status = ? AND file_id IN ?", userID, "active", fileIDs). @@ -2881,6 +2881,26 @@ func getOrInitQuotaForUpdate(tx *gorm.DB, userID uint, defaultQuotaBytes int64) return "a, nil } +// QueueFileEmbedding 原子登记指定向量空间的待执行任务。 +// 同一签名已经排队、执行或完成时不会重复登记；失败和失效任务允许重新排队。 +func (r *Repo) QueueFileEmbedding(ctx context.Context, userID uint, fileID string, embeddingSignature string) (bool, error) { + fileID = strings.TrimSpace(fileID) + embeddingSignature = strings.TrimSpace(embeddingSignature) + if fileID == "" || embeddingSignature == "" { + return false, repository.ErrInvalidInput + } + result := r.db.WithContext(ctx). + Model(&models.FileObject{}). + Where("user_id = ? AND file_id = ? AND status = ?", userID, fileID, "active"). + Where("NOT (embed_signature = ? AND embed_status IN ?)", embeddingSignature, []string{"queued", "processing", "ready"}). + Updates(map[string]interface{}{ + "embed_status": "queued", + "embed_signature": embeddingSignature, + "embed_error": "", + }) + return result.RowsAffected > 0, translateError(result.Error) +} + // ClaimFileEmbedding 原子领取指定向量空间的文件任务。 // 同一签名已经处于 processing/ready 时不会重复领取；切换向量空间后允许新任务接管。 func (r *Repo) ClaimFileEmbedding(ctx context.Context, userID uint, fileID string, embeddingSignature string) (bool, error) { @@ -4839,7 +4859,7 @@ func (r *Repo) SearchMessageChunks(ctx context.Context, input repository.Message return results, nil } -// MarkEmbeddedFilesStale 将缺少当前向量空间签名分片的 ready/processing 文件标记为 stale。 +// MarkEmbeddedFilesStale 将缺少当前向量空间签名分片的 queued/processing/ready 文件标记为 stale。 func (r *Repo) MarkEmbeddedFilesStale(ctx context.Context, activeSignature string) (int64, error) { activeSignature = strings.TrimSpace(activeSignature) if activeSignature == "" { @@ -4847,7 +4867,7 @@ func (r *Repo) MarkEmbeddedFilesStale(ctx context.Context, activeSignature strin } result := r.db.WithContext(ctx). Model(&models.FileObject{}). - Where("embed_status IN ? AND status = ?", []string{"ready", "processing"}, "active"). + Where("embed_status IN ? AND status = ?", []string{"queued", "processing", "ready"}, "active"). Where(`NOT EXISTS ( SELECT 1 FROM file_chunks diff --git a/backend/internal/infra/persistence/postgres/conversation/repository_sqlite_vector_test.go b/backend/internal/infra/persistence/postgres/conversation/repository_sqlite_vector_test.go index 6b765e6e0..50d7e30f6 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository_sqlite_vector_test.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository_sqlite_vector_test.go @@ -218,6 +218,59 @@ func TestMarkEmbeddedFilesStaleKeepsCurrentSignatureReady(t *testing.T) { } } +func TestFileEmbeddingQueueStateAndProcessingProjection(t *testing.T) { + db := openConversationSQLiteVectorTestDB(t) + repo := NewRepo(db) + ctx := context.Background() + file := model.FileObject{ + FileID: "file_queue_state", + UserID: 7, + FileName: "manual.md", + MimeType: "text/markdown", + FileCategory: "text", + StoragePath: "uploads/manual.md", + Status: "active", + ProcessingReady: true, + ExtractStatus: "ready", + EmbedStatus: "none", + } + if err := db.Create(&file).Error; err != nil { + t.Fatalf("create file: %v", err) + } + + queued, err := repo.QueueFileEmbedding(ctx, 7, file.FileID, "model@1536") + if err != nil || !queued { + t.Fatalf("queue embedding: queued=%v err=%v", queued, err) + } + if duplicate, duplicateErr := repo.QueueFileEmbedding(ctx, 7, file.FileID, "model@1536"); duplicateErr != nil || duplicate { + t.Fatalf("duplicate queue must be rejected: queued=%v err=%v", duplicate, duplicateErr) + } + + statuses, err := repo.GetActiveFileProcessingStatusesByIDs(ctx, 7, []string{file.FileID}) + if err != nil || len(statuses) != 1 { + t.Fatalf("load processing projection: statuses=%#v err=%v", statuses, err) + } + status := statuses[0] + if status.FileName != file.FileName || status.MimeType != file.MimeType || + status.FileCategory != file.FileCategory || status.StoragePath != file.StoragePath || + status.Status != "active" || !status.ProcessingReady || status.ExtractStatus != "ready" || + status.EmbedStatus != "queued" || status.EmbedSignature != "model@1536" { + t.Fatalf("incomplete processing projection: %#v", status) + } + + claimed, err := repo.ClaimFileEmbedding(ctx, 7, file.FileID, "model@1536") + if err != nil || !claimed { + t.Fatalf("claim queued embedding: claimed=%v err=%v", claimed, err) + } + var stored model.FileObject + if err = db.Where("id = ?", file.ID).Take(&stored).Error; err != nil { + t.Fatalf("load claimed file: %v", err) + } + if stored.EmbedStatus != "processing" || stored.EmbedSignature != "model@1536" { + t.Fatalf("unexpected claimed state: %#v", stored) + } +} + func TestFileEmbeddingGenerationRejectsSupersededPublisher(t *testing.T) { db := openConversationSQLiteVectorTestDB(t) repo := NewRepo(db) diff --git a/backend/internal/infra/persistence/postgres/knowledgebase/repository.go b/backend/internal/infra/persistence/postgres/knowledgebase/repository.go index 1ea3c19b4..e70b35b1e 100644 --- a/backend/internal/infra/persistence/postgres/knowledgebase/repository.go +++ b/backend/internal/infra/persistence/postgres/knowledgebase/repository.go @@ -20,8 +20,8 @@ type Repo struct { const knowledgeBaseFileSelectColumns = ` fo.id, fo.file_id, fo.user_id, fo.purpose, fo.file_name, fo.mime_type, fo.detected_mime, - fo.file_category, fo.size_bytes, fo.sha256, fo.status, fo.processing_status, - fo.processing_ready, fo.extract_status, fo.embed_status, fo.rag_opt_out, + fo.file_category, fo.size_bytes, fo.sha256, fo.storage_path, fo.status, fo.processing_status, + fo.processing_ready, fo.extract_status, fo.embed_status, fo.embed_signature, fo.embed_error, fo.rag_opt_out, fo.chunk_count, fo.page_count, fo.created_at, fo.updated_at` // NewRepo 创建知识库仓储。 @@ -247,8 +247,9 @@ func (r *Repo) listKnowledgeBaseFileProcessingStatuses(ctx context.Context, know if len(fileIDs) > 0 { if err := r.db.WithContext(ctx).Table("knowledge_base_files AS kbf"). Select(` - fo.file_id, fo.detected_mime, fo.file_category, fo.processing_status, - fo.processing_ready, fo.extract_status, fo.embed_status, fo.rag_opt_out, fo.chunk_count, fo.updated_at`). + fo.file_id, fo.file_name, fo.mime_type, fo.detected_mime, fo.file_category, fo.storage_path, fo.status, + fo.processing_status, fo.processing_ready, fo.extract_status, fo.embed_status, fo.embed_signature, + fo.embed_error, fo.rag_opt_out, fo.chunk_count, fo.updated_at`). Joins("JOIN file_objects AS fo ON fo.id = kbf.file_object_id AND fo.status = ? AND fo.deleted_at IS NULL", "active"). Where("kbf.knowledge_base_id = ? AND fo.file_id IN ?", knowledgeBaseID, fileIDs). Scan(&items).Error; err != nil { @@ -622,7 +623,8 @@ func toFileDomain(item model.FileObject) domainconversation.FileObject { SizeBytes: item.SizeBytes, SHA256: item.SHA256, StoragePath: item.StoragePath, Status: item.Status, ProcessingStatus: item.ProcessingStatus, ProcessingReady: item.ProcessingReady, ProcessingErrorCode: item.ProcessingErrorCode, ProcessingErrorMessage: item.ProcessingErrorMessage, - ExtractStatus: item.ExtractStatus, EmbedStatus: item.EmbedStatus, RagOptOut: item.RagOptOut, + ExtractStatus: item.ExtractStatus, EmbedStatus: item.EmbedStatus, EmbedSignature: item.EmbedSignature, + EmbedError: item.EmbedError, RagOptOut: item.RagOptOut, ChunkCount: item.ChunkCount, PageCount: item.PageCount, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt, } } diff --git a/backend/internal/infra/persistence/postgres/knowledgebase/repository_sqlite_test.go b/backend/internal/infra/persistence/postgres/knowledgebase/repository_sqlite_test.go index a0aaf3114..bc0acd745 100644 --- a/backend/internal/infra/persistence/postgres/knowledgebase/repository_sqlite_test.go +++ b/backend/internal/infra/persistence/postgres/knowledgebase/repository_sqlite_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" domainknowledgebase "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/knowledgebase" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" @@ -43,7 +44,11 @@ func TestRepositoryVisibilityResolutionAndDeletion(t *testing.T) { {FileID: "builtin-available", UserID: 0, FileName: "platform-guide.md", Status: "active"}, {FileID: "mine-file", UserID: 11, FileName: "notes.md", Status: "active", ProcessingReady: true, EmbedStatus: "ready", ChunkCount: 2}, {FileID: "other-file", UserID: 22, FileName: "private.md", Status: "active", ProcessingReady: true, EmbedStatus: "ready", ChunkCount: 2}, - {FileID: "mine-pending", UserID: 11, FileName: "pending.md", Status: "active", ProcessingReady: true, EmbedStatus: "processing"}, + { + FileID: "mine-pending", UserID: 11, FileName: "pending.md", MimeType: "text/markdown", + FileCategory: "text", StoragePath: "uploads/pending.md", Status: "active", + ProcessingReady: true, ExtractStatus: "ready", EmbedStatus: "processing", EmbedSignature: "model@1536", + }, {FileID: "mine-available", UserID: 11, FileName: "available.md", Status: "active"}, } if err = db.Create(&files).Error; err != nil { @@ -63,6 +68,33 @@ func TestRepositoryVisibilityResolutionAndDeletion(t *testing.T) { } repo := NewRepo(db) + processingStatuses, err := repo.GetKnowledgeBaseFileProcessingStatuses( + context.Background(), items[2].ID, []string{"mine-pending"}, + ) + if err != nil || len(processingStatuses) != 1 { + t.Fatalf("GetKnowledgeBaseFileProcessingStatuses() = %#v, %v", processingStatuses, err) + } + processingStatus := processingStatuses[0] + if processingStatus.FileName != "pending.md" || processingStatus.MimeType != "text/markdown" || + processingStatus.FileCategory != "text" || processingStatus.StoragePath != "uploads/pending.md" || + processingStatus.Status != "active" || !processingStatus.ProcessingReady || processingStatus.ExtractStatus != "ready" || + processingStatus.EmbedStatus != "processing" || processingStatus.EmbedSignature != "model@1536" { + t.Fatalf("incomplete knowledge base processing projection: %#v", processingStatus) + } + listedFiles, listedTotal, err := repo.ListKnowledgeBaseFiles(context.Background(), items[2].ID, 0, 100) + if err != nil || listedTotal != 2 { + t.Fatalf("ListKnowledgeBaseFiles() = %#v, total=%d, err=%v", listedFiles, listedTotal, err) + } + var listedPending *domainconversation.FileObject + for i := range listedFiles { + if listedFiles[i].FileID == "mine-pending" { + listedPending = &listedFiles[i] + break + } + } + if listedPending == nil || listedPending.StoragePath != "uploads/pending.md" || listedPending.EmbedSignature != "model@1536" { + t.Fatalf("incomplete knowledge base list projection: %#v", listedPending) + } createdDisabled, err := repo.CreateKnowledgeBase(context.Background(), &domainknowledgebase.KnowledgeBase{ PublicID: "created-disabled", Scope: domainknowledgebase.ScopeBuiltin, Name: "Created disabled", Enabled: false, }) diff --git a/backend/internal/repository/conversation_cache.go b/backend/internal/repository/conversation_cache.go index da62e260f..2483d4cc1 100644 --- a/backend/internal/repository/conversation_cache.go +++ b/backend/internal/repository/conversation_cache.go @@ -7,6 +7,15 @@ import ( domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" ) +const FileProcessingKindEmbedding = "embedding" + +type FileProcessingQueue string + +const ( + FileProcessingQueueDefault FileProcessingQueue = "file_processing" + FileProcessingQueueEmbedding FileProcessingQueue = "file_embedding" +) + // FileProcessingMessage 文件处理队列消息。 type FileProcessingMessage struct { // ID 是 Redis Stream 消息 ID。 @@ -16,6 +25,12 @@ type FileProcessingMessage struct { Retry int LastError string Reclaimed bool + Kind string + // Queue 标记消息实际所属队列；旧消息即使携带 embedding kind，也能在原队列完成续租和确认。 + Queue FileProcessingQueue + // EmbeddingSignature 与 EmbeddingHost 将显式向量化任务固定到接收任务时的运行时配置。 + EmbeddingSignature string + EmbeddingHost string } // GenerationStreamMessage 是生成流中的一条可恢复事件。 @@ -42,10 +57,13 @@ type GenerationStreamTextSnapshot struct { type FileProcessingQueueRepository interface { InitFileProcessingStream(ctx context.Context) error EnqueueFileProcessing(ctx context.Context, userID uint, fileID string, retry int, lastError string) error + EnqueueFileEmbedding(ctx context.Context, userID uint, fileID string, embeddingSignature string, embeddingHost string) error ClaimTimedOutFileProcessingMessages(ctx context.Context, consumerName string) ([]FileProcessingMessage, error) + ClaimTimedOutFileEmbeddingMessages(ctx context.Context, consumerName string) ([]FileProcessingMessage, error) ReadFileProcessingMessages(ctx context.Context, consumerName string) ([]FileProcessingMessage, error) - RenewFileProcessingMessageLease(ctx context.Context, consumerName, messageID string) (bool, error) - SettleFileProcessingMessage(ctx context.Context, consumerName, messageID string) (bool, error) + ReadFileEmbeddingMessages(ctx context.Context, consumerName string) ([]FileProcessingMessage, error) + RenewFileProcessingMessageLease(ctx context.Context, consumerName string, message FileProcessingMessage) (bool, error) + SettleFileProcessingMessage(ctx context.Context, consumerName string, message FileProcessingMessage) (bool, error) RequeueFileProcessingMessage(ctx context.Context, consumerName string, message FileProcessingMessage, retry int, lastError string) (bool, error) DeadLetterFileProcessingMessage(ctx context.Context, consumerName string, message FileProcessingMessage, lastError string) (bool, error) } diff --git a/backend/internal/repository/conversation_file.go b/backend/internal/repository/conversation_file.go index 8f4bb1679..5cf918f5d 100644 --- a/backend/internal/repository/conversation_file.go +++ b/backend/internal/repository/conversation_file.go @@ -65,12 +65,14 @@ type FileEmbeddingArtifactsRepository interface { type EmbeddingRepository interface { VectorStoreAvailable(ctx context.Context) (bool, error) GetActiveFileObjectByID(ctx context.Context, userID uint, fileID string) (*domainconversation.FileObject, error) + GetActiveFileObjectsByIDs(ctx context.Context, userID uint, fileIDs []string) ([]domainconversation.FileObject, error) GetFileObjectProcessingByObjectID(ctx context.Context, fileObjID uint) (*domainconversation.FileObjectProcessing, error) + QueueFileEmbedding(ctx context.Context, userID uint, fileID string, embeddingSignature string) (bool, error) ClaimFileEmbedding(ctx context.Context, userID uint, fileID string, embeddingSignature string) (bool, error) UpdateFileObjectEmbedStatus(ctx context.Context, userID uint, fileID string, embeddingSignature string, status string, embedErr string) (bool, error) UpdateFileObjectChunkCount(ctx context.Context, fileObjID uint, embeddingSignature string, chunkCount int) (bool, error) ReplaceFileChunks(ctx context.Context, fileObjID uint, embeddingSignature string, chunks []domainconversation.FileChunk, embeddings [][]float32) (bool, error) - // MarkEmbeddedFilesStale 将缺少当前向量空间签名分片的 ready/processing 文件标记为 stale。 + // MarkEmbeddedFilesStale 将缺少当前向量空间签名分片的 queued/processing/ready 文件标记为 stale。 // 在 Embedding 配置变更及服务启动时调用，使旧向量失效并等待重建。 // 返回被标记的文件数量。 MarkEmbeddedFilesStale(ctx context.Context, activeSignature string) (int64, error) diff --git a/backend/internal/shared/response/error_code.go b/backend/internal/shared/response/error_code.go index 13ede8164..0a143e148 100644 --- a/backend/internal/shared/response/error_code.go +++ b/backend/internal/shared/response/error_code.go @@ -635,6 +635,8 @@ var fallbackMessages = map[string]string{ "settings.extract_invalid": "invalid file extraction settings", "embedding.service_unavailable": "embedding service is not available", "embedding.service_not_configured": "embedding service is not configured", + "embedding.submit_failed": "failed to submit embedding jobs", + "embedding.too_many_files": "too many files for embedding", "user_settings.unknown_key": "unknown setting key", "user_settings.invalid_value": "invalid user setting value", "memory.key_required": "memory_key is required", diff --git a/backend/internal/transport/http/conversation/dto_request.go b/backend/internal/transport/http/conversation/dto_request.go index fd904f3f8..b9be38152 100644 --- a/backend/internal/transport/http/conversation/dto_request.go +++ b/backend/internal/transport/http/conversation/dto_request.go @@ -96,6 +96,11 @@ type GetFileProcessingStatusesRequest struct { FileIDs []string `json:"fileIDs" binding:"required,min=1,max=100,dive,required,max=64"` } +// SubmitFileEmbeddingsRequest 批量提交文件向量化请求。 +type SubmitFileEmbeddingsRequest struct { + FileIDs []string `json:"fileIDs" binding:"required,min=1,max=100,dive,required,max=64"` +} + // GetConversationRunStatusesRequest 批量运行状态查询请求。 type GetConversationRunStatusesRequest struct { RunIDs []string `json:"runIDs" binding:"required,min=1,max=100,dive,required,max=64"` diff --git a/backend/internal/transport/http/conversation/dto_response.go b/backend/internal/transport/http/conversation/dto_response.go index 88804313e..a7ed43ff5 100644 --- a/backend/internal/transport/http/conversation/dto_response.go +++ b/backend/internal/transport/http/conversation/dto_response.go @@ -6,6 +6,7 @@ import ( "time" appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" appprocessing "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/processing" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" @@ -445,13 +446,15 @@ type FileObjectResponse struct { EmbedError string `json:"embedError"` ChunkCount int `json:"chunkCount"` RagOptOut bool `json:"ragOptOut"` + CanVectorize bool `json:"canVectorize"` + VectorizationReason string `json:"vectorizationReason"` LastAccessedAt *time.Time `json:"lastAccessedAt" extensions:"x-nullable,!x-omitempty"` ExpiresAt *time.Time `json:"expiresAt" extensions:"x-nullable,!x-omitempty"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } -func toFileObjectResponse(item *model.FileObject) FileObjectResponse { +func toFileObjectResponse(item *model.FileObject, capability appembedding.FileVectorizationCapability) FileObjectResponse { return FileObjectResponse{ FileID: item.FileID, Purpose: item.Purpose, @@ -471,6 +474,8 @@ func toFileObjectResponse(item *model.FileObject) FileObjectResponse { EmbedError: item.EmbedError, ChunkCount: item.ChunkCount, RagOptOut: item.RagOptOut, + CanVectorize: capability.CanVectorize, + VectorizationReason: capability.Reason, LastAccessedAt: item.LastAccessedAt, ExpiresAt: item.ExpiresAt, CreatedAt: item.CreatedAt, @@ -1328,52 +1333,79 @@ type ConversationRunStatusResponse struct { // ---------- File Processing Status ---------- +// FileEmbeddingSkipResponse 表示未提交向量化的文件及原因。 +type FileEmbeddingSkipResponse struct { + FileID string `json:"fileID"` + Reason string `json:"reason"` +} + +// FileEmbeddingSubmissionResponse 表示定向向量化任务提交结果。 +type FileEmbeddingSubmissionResponse struct { + SubmittedFileIDs []string `json:"submittedFileIDs"` + Skipped []FileEmbeddingSkipResponse `json:"skipped"` +} + +func toFileEmbeddingSubmissionResponse(result appembedding.TargetedSubmissionResult) FileEmbeddingSubmissionResponse { + skipped := make([]FileEmbeddingSkipResponse, 0, len(result.Skipped)) + for _, item := range result.Skipped { + skipped = append(skipped, FileEmbeddingSkipResponse{FileID: item.FileID, Reason: item.Reason}) + } + return FileEmbeddingSubmissionResponse{ + SubmittedFileIDs: result.SubmittedFileIDs, + Skipped: skipped, + } +} + // FileProcessingStatusResponse 文件处理状态响应 DTO。 type FileProcessingStatusResponse struct { - FileID string `json:"fileID"` - DetectedMIME string `json:"detectedMIME"` - FileCategory string `json:"fileCategory"` - ProcessingStatus string `json:"processingStatus"` - ProcessingReady bool `json:"processingReady"` - ExtractStatus string `json:"extractStatus"` - EmbedStatus string `json:"embedStatus"` - PreviewText string `json:"previewText"` - OCRUsed bool `json:"ocrUsed"` - RAGReady bool `json:"ragReady"` - RAGReason string `json:"ragReason"` - ErrorCode string `json:"errorCode"` - ErrorMessage string `json:"errorMessage"` - ExtractChars int `json:"extractChars"` - ExtractPages int `json:"extractPages"` - ChunkCount int `json:"chunkCount"` - EmbedError string `json:"embedError"` - StartedAt *time.Time `json:"startedAt" extensions:"x-nullable,!x-omitempty"` - CompletedAt *time.Time `json:"completedAt" extensions:"x-nullable,!x-omitempty"` - UpdatedAt time.Time `json:"updatedAt"` + FileID string `json:"fileID"` + DetectedMIME string `json:"detectedMIME"` + FileCategory string `json:"fileCategory"` + ProcessingStatus string `json:"processingStatus"` + ProcessingReady bool `json:"processingReady"` + ExtractStatus string `json:"extractStatus"` + EmbedStatus string `json:"embedStatus"` + PreviewText string `json:"previewText"` + OCRUsed bool `json:"ocrUsed"` + RAGReady bool `json:"ragReady"` + RAGReason string `json:"ragReason"` + ErrorCode string `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + ExtractChars int `json:"extractChars"` + ExtractPages int `json:"extractPages"` + ChunkCount int `json:"chunkCount"` + EmbedError string `json:"embedError"` + CanVectorize bool `json:"canVectorize"` + VectorizationReason string `json:"vectorizationReason"` + StartedAt *time.Time `json:"startedAt" extensions:"x-nullable,!x-omitempty"` + CompletedAt *time.Time `json:"completedAt" extensions:"x-nullable,!x-omitempty"` + UpdatedAt time.Time `json:"updatedAt"` } func toFileProcessingStatusResponse(d *appprocessing.FileProcessingStatusDTO) FileProcessingStatusResponse { return FileProcessingStatusResponse{ - FileID: d.FileID, - DetectedMIME: d.DetectedMIME, - FileCategory: d.FileCategory, - ProcessingStatus: d.ProcessingStatus, - ProcessingReady: d.ProcessingReady, - ExtractStatus: d.ExtractStatus, - EmbedStatus: d.EmbedStatus, - PreviewText: d.PreviewText, - OCRUsed: d.OCRUsed, - RAGReady: d.RAGReady, - RAGReason: d.RAGReason, - ErrorCode: d.ErrorCode, - ErrorMessage: appprocessing.HumanizeFileProcessingError(d.FileCategory, d.ErrorCode, d.ErrorMessage), - ExtractChars: d.ExtractChars, - ExtractPages: d.ExtractPages, - ChunkCount: d.ChunkCount, - EmbedError: d.EmbedError, - StartedAt: d.StartedAt, - CompletedAt: d.CompletedAt, - UpdatedAt: d.UpdatedAt, + FileID: d.FileID, + DetectedMIME: d.DetectedMIME, + FileCategory: d.FileCategory, + ProcessingStatus: d.ProcessingStatus, + ProcessingReady: d.ProcessingReady, + ExtractStatus: d.ExtractStatus, + EmbedStatus: d.EmbedStatus, + PreviewText: d.PreviewText, + OCRUsed: d.OCRUsed, + RAGReady: d.RAGReady, + RAGReason: d.RAGReason, + ErrorCode: d.ErrorCode, + ErrorMessage: appprocessing.HumanizeFileProcessingError(d.FileCategory, d.ErrorCode, d.ErrorMessage), + ExtractChars: d.ExtractChars, + ExtractPages: d.ExtractPages, + ChunkCount: d.ChunkCount, + EmbedError: d.EmbedError, + CanVectorize: d.CanVectorize, + VectorizationReason: d.VectorizationReason, + StartedAt: d.StartedAt, + CompletedAt: d.CompletedAt, + UpdatedAt: d.UpdatedAt, } } @@ -1463,6 +1495,12 @@ type FileUpdateResponseDoc struct { Data FileObjectResponse `json:"data"` } +// FileEmbeddingSubmissionResponseDoc 文件向量化提交响应文档。 +type FileEmbeddingSubmissionResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data FileEmbeddingSubmissionResponse `json:"data"` +} + // ConversationCreateResponseDoc 创建会话响应文档。 type ConversationCreateResponseDoc struct { ErrorMsg string `json:"errorMsg"` diff --git a/backend/internal/transport/http/conversation/handler_file.go b/backend/internal/transport/http/conversation/handler_file.go index f9aa0533a..106ea244d 100644 --- a/backend/internal/transport/http/conversation/handler_file.go +++ b/backend/internal/transport/http/conversation/handler_file.go @@ -6,6 +6,7 @@ import ( "strings" appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/response" @@ -92,8 +93,12 @@ func (h *Handler) UploadFile(c *gin.Context) { }, ) + capability := h.service.ResolveFileVectorizationCapabilities( + c.Request.Context(), + []model.FileObject{result.File}, + )[result.File.FileID] response.Success(c, FileUploadResponse{ - File: toFileObjectResponse(&result.File), + File: toFileObjectResponse(&result.File, capability), Quota: toStorageQuotaResponse(result.Quota), Reused: result.Reused, }) @@ -139,8 +144,9 @@ func (h *Handler) ListFiles(c *gin.Context) { return } results := make([]FileObjectResponse, 0, len(result.Items)) + capabilities := h.service.ResolveFileVectorizationCapabilities(c.Request.Context(), result.Items) for i := range result.Items { - results = append(results, toFileObjectResponse(&result.Items[i])) + results = append(results, toFileObjectResponse(&result.Items[i], capabilities[result.Items[i].FileID])) } response.Success(c, FileListResponse{ Total: result.Total, @@ -184,7 +190,7 @@ func (h *Handler) GetFileProcessingStatus(c *gin.Context) { func (h *Handler) GetFileProcessingStatuses(c *gin.Context) { var req GetFileProcessingStatusesRequest if err := c.ShouldBindJSON(&req); err != nil { - response.Error(c, http.StatusBadRequest, "invalid file ids") + response.InvalidRequestBody(c, err) return } @@ -204,6 +210,47 @@ func (h *Handler) GetFileProcessingStatuses(c *gin.Context) { response.Success(c, statuses) } +// SubmitFileEmbeddings godoc +// @Summary 批量提交指定文件向量化 +// @Description 为当前用户已完成文本提取的文件提交向量化任务，最多100个；重复提交会幂等跳过 +// @Tags chat +// @Produce json +// @Security BearerAuth +// @Accept json +// @Param request body SubmitFileEmbeddingsRequest true "文件ID，最多100个" +// @Success 200 {object} FileEmbeddingSubmissionResponseDoc +// @Failure 400 {object} ErrorDoc +// @Failure 500 {object} ErrorDoc +// @Failure 503 {object} ErrorDoc +// @Router /files/embeddings [post] +func (h *Handler) SubmitFileEmbeddings(c *gin.Context) { + var req SubmitFileEmbeddingsRequest + if err := c.ShouldBindJSON(&req); err != nil { + response.InvalidRequestBody(c, err) + return + } + result, err := h.service.SubmitFileEmbeddings(c.Request.Context(), middleware.MustUserID(c), req.FileIDs) + if err != nil { + switch { + case errors.Is(err, appembedding.ErrTooManyTargetedFiles): + response.ErrorWithCode(c, http.StatusBadRequest, "embedding.too_many_files", "too many files") + case errors.Is(err, appembedding.ErrEmbeddingServiceNotConfigured): + response.ErrorWithCode(c, http.StatusServiceUnavailable, "embedding.service_not_configured", "embedding service not configured") + case errors.Is(err, appembedding.ErrEmbeddingServiceUnavailable): + response.ErrorWithCode(c, http.StatusServiceUnavailable, "embedding.service_unavailable", "embedding service unavailable") + default: + response.ErrorWithCode(c, http.StatusInternalServerError, "embedding.submit_failed", "submit embedding jobs failed") + } + return + } + h.recordAudit(c, "submit_file_embeddings", "file", "", map[string]interface{}{ + "requested_file_ids": req.FileIDs, + "submitted_file_ids": result.SubmittedFileIDs, + "skipped_count": len(result.Skipped), + }) + response.Success(c, toFileEmbeddingSubmissionResponse(result)) +} + // GetFileExtract 获取文件提取文本。 func (h *Handler) GetFileExtract(c *gin.Context) { userID := middleware.MustUserID(c) @@ -325,7 +372,11 @@ func (h *Handler) UpdateFile(c *gin.Context) { auditDetail, ) - response.Success(c, toFileObjectResponse(item)) + capability := h.service.ResolveFileVectorizationCapabilities( + c.Request.Context(), + []model.FileObject{*item}, + )[item.FileID] + response.Success(c, toFileObjectResponse(item, capability)) } // DeleteFile godoc diff --git a/backend/internal/transport/http/conversation/router.go b/backend/internal/transport/http/conversation/router.go index 4688a5fb6..f811975dd 100644 --- a/backend/internal/transport/http/conversation/router.go +++ b/backend/internal/transport/http/conversation/router.go @@ -51,6 +51,7 @@ func (m *Module) RegisterRoutes(authRequired *gin.RouterGroup) { authRequired.POST("/files", m.Handler.UploadFile) authRequired.GET("/files", m.Handler.ListFiles) authRequired.POST("/files/processing/statuses", m.Handler.GetFileProcessingStatuses) + authRequired.POST("/files/embeddings", m.Handler.SubmitFileEmbeddings) authRequired.GET("/files/:file_id/processing", m.Handler.GetFileProcessingStatus) authRequired.GET("/files/:file_id/extract", m.Handler.GetFileExtract) authRequired.PATCH("/files/:file_id", m.Handler.UpdateFile) diff --git a/backend/internal/transport/http/knowledgebase/dto.go b/backend/internal/transport/http/knowledgebase/dto.go index f2acffbfc..240009c59 100644 --- a/backend/internal/transport/http/knowledgebase/dto.go +++ b/backend/internal/transport/http/knowledgebase/dto.go @@ -3,6 +3,7 @@ package knowledgebase import ( "time" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" domainknowledgebase "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/knowledgebase" ) @@ -25,20 +26,24 @@ type KnowledgeBaseResponse struct { // KnowledgeBaseFileResponse 表示知识库文件摘要。 type KnowledgeBaseFileResponse struct { - FileID string `json:"fileID"` - FileName string `json:"fileName"` - MimeType string `json:"mimeType"` - DetectedMIME string `json:"detectedMIME"` - FileCategory string `json:"fileCategory"` - SizeBytes int64 `json:"sizeBytes"` - ProcessingStatus string `json:"processingStatus"` - Processing bool `json:"processing"` - ProcessingReady bool `json:"processingReady"` - EmbedStatus string `json:"embedStatus"` - ChunkCount int `json:"chunkCount"` - RagOptOut bool `json:"ragOptOut"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + FileID string `json:"fileID"` + FileName string `json:"fileName"` + MimeType string `json:"mimeType"` + DetectedMIME string `json:"detectedMIME"` + FileCategory string `json:"fileCategory"` + SizeBytes int64 `json:"sizeBytes"` + ProcessingStatus string `json:"processingStatus"` + Processing bool `json:"processing"` + ProcessingReady bool `json:"processingReady"` + ExtractStatus string `json:"extractStatus"` + EmbedStatus string `json:"embedStatus"` + EmbedError string `json:"embedError"` + ChunkCount int `json:"chunkCount"` + RagOptOut bool `json:"ragOptOut"` + CanVectorize bool `json:"canVectorize"` + VectorizationReason string `json:"vectorizationReason"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } // WriteMyKnowledgeBaseRequest 表示创建个人知识库请求。 @@ -84,18 +89,39 @@ type GetKnowledgeBaseFileProcessingSnapshotRequest struct { FileIDs []string `json:"fileIDs" binding:"max=100,dive,required,max=64"` } +// SubmitPlatformFileEmbeddingsRequest 表示批量提交平台资料向量化请求。 +type SubmitPlatformFileEmbeddingsRequest struct { + FileIDs []string `json:"fileIDs" binding:"required,min=1,max=100,dive,required,max=64"` +} + +// KnowledgeBaseFileEmbeddingSkipResponse 表示未提交向量化的平台资料及原因。 +type KnowledgeBaseFileEmbeddingSkipResponse struct { + FileID string `json:"fileID"` + Reason string `json:"reason"` +} + +// KnowledgeBaseFileEmbeddingSubmissionResponse 表示平台资料向量化提交结果。 +type KnowledgeBaseFileEmbeddingSubmissionResponse struct { + SubmittedFileIDs []string `json:"submittedFileIDs"` + Skipped []KnowledgeBaseFileEmbeddingSkipResponse `json:"skipped"` +} + // KnowledgeBaseFileProcessingStatusResponse 表示知识库文件处理状态。 type KnowledgeBaseFileProcessingStatusResponse struct { - FileID string `json:"fileID"` - DetectedMIME string `json:"detectedMIME"` - FileCategory string `json:"fileCategory"` - ProcessingStatus string `json:"processingStatus"` - Processing bool `json:"processing"` - ProcessingReady bool `json:"processingReady"` - EmbedStatus string `json:"embedStatus"` - ChunkCount int `json:"chunkCount"` - RagOptOut bool `json:"ragOptOut"` - UpdatedAt time.Time `json:"updatedAt"` + FileID string `json:"fileID"` + DetectedMIME string `json:"detectedMIME"` + FileCategory string `json:"fileCategory"` + ProcessingStatus string `json:"processingStatus"` + Processing bool `json:"processing"` + ProcessingReady bool `json:"processingReady"` + ExtractStatus string `json:"extractStatus"` + EmbedStatus string `json:"embedStatus"` + EmbedError string `json:"embedError"` + ChunkCount int `json:"chunkCount"` + RagOptOut bool `json:"ragOptOut"` + CanVectorize bool `json:"canVectorize"` + VectorizationReason string `json:"vectorizationReason"` + UpdatedAt time.Time `json:"updatedAt"` } // KnowledgeBaseFileProcessingSnapshotResponse 表示知识库及文件处理状态快照。 @@ -178,6 +204,12 @@ type PlatformFileDeleteResponseDoc struct { Data PlatformFileDeleteDataResponse `json:"data"` } +// KnowledgeBaseFileEmbeddingSubmissionResponseDoc 用于 Swagger 展示平台资料向量化提交响应。 +type KnowledgeBaseFileEmbeddingSubmissionResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data KnowledgeBaseFileEmbeddingSubmissionResponse `json:"data"` +} + // ErrorDoc 表示错误响应。 type ErrorDoc struct { ErrorMsg string `json:"errorMsg"` @@ -200,33 +232,58 @@ func toKnowledgeBaseResponses(items []domainknowledgebase.KnowledgeBase) []Knowl return results } -func toKnowledgeBaseFileResponses(items []domainconversation.FileObject) []KnowledgeBaseFileResponse { +func toKnowledgeBaseFileResponses( + items []domainconversation.FileObject, + capabilities map[string]appembedding.FileVectorizationCapability, +) []KnowledgeBaseFileResponse { results := make([]KnowledgeBaseFileResponse, 0, len(items)) for _, item := range items { - results = append(results, toKnowledgeBaseFileResponse(item)) + results = append(results, toKnowledgeBaseFileResponse(item, capabilities[item.FileID])) } return results } -func toKnowledgeBaseFileProcessingStatusResponses(items []domainconversation.FileObject) []KnowledgeBaseFileProcessingStatusResponse { +func toKnowledgeBaseFileEmbeddingSubmissionResponse(result appembedding.TargetedSubmissionResult) KnowledgeBaseFileEmbeddingSubmissionResponse { + skipped := make([]KnowledgeBaseFileEmbeddingSkipResponse, 0, len(result.Skipped)) + for _, item := range result.Skipped { + skipped = append(skipped, KnowledgeBaseFileEmbeddingSkipResponse{FileID: item.FileID, Reason: item.Reason}) + } + return KnowledgeBaseFileEmbeddingSubmissionResponse{ + SubmittedFileIDs: result.SubmittedFileIDs, + Skipped: skipped, + } +} + +func toKnowledgeBaseFileProcessingStatusResponses( + items []domainconversation.FileObject, + capabilities map[string]appembedding.FileVectorizationCapability, +) []KnowledgeBaseFileProcessingStatusResponse { results := make([]KnowledgeBaseFileProcessingStatusResponse, 0, len(items)) for _, item := range items { results = append(results, KnowledgeBaseFileProcessingStatusResponse{ FileID: item.FileID, DetectedMIME: item.DetectedMIME, FileCategory: item.FileCategory, ProcessingStatus: item.ProcessingStatus, Processing: domainconversation.IsFileProcessing(item), ProcessingReady: item.ProcessingReady, - EmbedStatus: item.EmbedStatus, ChunkCount: item.ChunkCount, RagOptOut: item.RagOptOut, - UpdatedAt: item.UpdatedAt, + ExtractStatus: item.ExtractStatus, EmbedStatus: item.EmbedStatus, EmbedError: item.EmbedError, + ChunkCount: item.ChunkCount, RagOptOut: item.RagOptOut, + CanVectorize: capabilities[item.FileID].CanVectorize, + VectorizationReason: capabilities[item.FileID].Reason, + UpdatedAt: item.UpdatedAt, }) } return results } -func toKnowledgeBaseFileResponse(item domainconversation.FileObject) KnowledgeBaseFileResponse { +func toKnowledgeBaseFileResponse( + item domainconversation.FileObject, + capability appembedding.FileVectorizationCapability, +) KnowledgeBaseFileResponse { return KnowledgeBaseFileResponse{ FileID: item.FileID, FileName: item.FileName, MimeType: item.MimeType, DetectedMIME: item.DetectedMIME, FileCategory: item.FileCategory, SizeBytes: item.SizeBytes, ProcessingStatus: item.ProcessingStatus, Processing: domainconversation.IsFileProcessing(item), ProcessingReady: item.ProcessingReady, - EmbedStatus: item.EmbedStatus, ChunkCount: item.ChunkCount, RagOptOut: item.RagOptOut, + ExtractStatus: item.ExtractStatus, EmbedStatus: item.EmbedStatus, EmbedError: item.EmbedError, + ChunkCount: item.ChunkCount, RagOptOut: item.RagOptOut, + CanVectorize: capability.CanVectorize, VectorizationReason: capability.Reason, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt, } } diff --git a/backend/internal/transport/http/knowledgebase/handler.go b/backend/internal/transport/http/knowledgebase/handler.go index 50c232356..e0a6c2256 100644 --- a/backend/internal/transport/http/knowledgebase/handler.go +++ b/backend/internal/transport/http/knowledgebase/handler.go @@ -7,6 +7,7 @@ import ( "strings" appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" + appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" appknowledgebase "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/knowledgebase" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" @@ -146,7 +147,7 @@ func (h *Handler) DeleteMine(c *gin.Context) { func (h *Handler) ListVisibleFiles(c *gin.Context) { page, pageSize := pageParams(c) items, total, err := h.service.ListVisibleFiles(c.Request.Context(), middleware.MustUserID(c), c.Param("id"), page, pageSize) - writeFileList(c, items, total, err) + h.writeFileList(c, items, total, err) } // GetVisibleFileProcessingStatuses godoc @@ -203,7 +204,7 @@ func (h *Handler) ListAvailableMineFiles(c *gin.Context) { c.Param("id"), listInput(c), ) - writeFileList(c, items, total, err) + h.writeFileList(c, items, total, err) } // GetVisibleFileContent godoc @@ -294,7 +295,7 @@ func (h *Handler) ListPlatformFiles(c *gin.Context) { middleware.MustUserID(c), listInput(c), ) - writeFileList(c, items, total, err) + h.writeFileList(c, items, total, err) } // UploadAdminFile godoc @@ -335,7 +336,52 @@ func (h *Handler) UploadAdminFile(c *gin.Context) { h.audit(c, "knowledge_base.upload_builtin_file", result.File.FileID, map[string]interface{}{ "file_name": result.File.FileName, "size_bytes": result.File.SizeBytes, }) - response.Success(c, KnowledgeBaseFileDataResponse{File: toKnowledgeBaseFileResponse(result.File)}) + capability := h.service.ResolveFileVectorizationCapabilities( + c.Request.Context(), + []domainconversation.FileObject{result.File}, + )[result.File.FileID] + response.Success(c, KnowledgeBaseFileDataResponse{File: toKnowledgeBaseFileResponse(result.File, capability)}) +} + +// SubmitAdminFileEmbeddings godoc +// @Summary 批量提交平台资料向量化 +// @Description 为管理员选中的平台资料提交向量化任务，最多100个；重复提交会幂等跳过 +// @Tags admin-knowledge-bases +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param request body SubmitPlatformFileEmbeddingsRequest true "平台资料ID，最多100个" +// @Success 200 {object} KnowledgeBaseFileEmbeddingSubmissionResponseDoc +// @Failure 400 {object} ErrorDoc +// @Failure 500 {object} ErrorDoc +// @Failure 503 {object} ErrorDoc +// @Router /admin/knowledge-bases/files/embeddings [post] +func (h *Handler) SubmitAdminFileEmbeddings(c *gin.Context) { + var req SubmitPlatformFileEmbeddingsRequest + if err := c.ShouldBindJSON(&req); err != nil { + response.InvalidRequestBody(c, err) + return + } + result, err := h.service.SubmitPlatformFileEmbeddings(c.Request.Context(), middleware.MustUserID(c), req.FileIDs) + if err != nil { + switch { + case errors.Is(err, appembedding.ErrTooManyTargetedFiles): + response.ErrorWithCode(c, http.StatusBadRequest, "embedding.too_many_files", "too many files") + case errors.Is(err, appembedding.ErrEmbeddingServiceNotConfigured): + response.ErrorWithCode(c, http.StatusServiceUnavailable, "embedding.service_not_configured", "embedding service not configured") + case errors.Is(err, appembedding.ErrEmbeddingServiceUnavailable): + response.ErrorWithCode(c, http.StatusServiceUnavailable, "embedding.service_unavailable", "embedding service unavailable") + default: + writeError(c, err) + } + return + } + h.audit(c, "knowledge_base.submit_platform_file_embeddings", "", map[string]interface{}{ + "requested_file_ids": req.FileIDs, + "submitted_file_ids": result.SubmittedFileIDs, + "skipped_count": len(result.Skipped), + }) + response.Success(c, toKnowledgeBaseFileEmbeddingSubmissionResponse(result)) } // DeleteAdminFile godoc @@ -494,7 +540,7 @@ func (h *Handler) DeleteAdmin(c *gin.Context) { h.delete(c, true) } func (h *Handler) ListAdminFiles(c *gin.Context) { page, pageSize := pageParams(c) items, total, err := h.service.ListAdminFiles(c.Request.Context(), c.Param("id"), page, pageSize) - writeFileList(c, items, total, err) + h.writeFileList(c, items, total, err) } // GetAdminFileProcessingStatuses godoc @@ -551,7 +597,7 @@ func (h *Handler) ListAvailableAdminFiles(c *gin.Context) { c.Param("id"), listInput(c), ) - writeFileList(c, items, total, err) + h.writeFileList(c, items, total, err) } // GetAdminFileContent godoc @@ -699,7 +745,8 @@ func (h *Handler) getFileProcessingStatuses(c *gin.Context, admin bool) { writeError(c, err) return } - response.Success(c, toKnowledgeBaseFileProcessingStatusResponses(files)) + capabilities := h.service.ResolveFileVectorizationCapabilities(c.Request.Context(), files) + response.Success(c, toKnowledgeBaseFileProcessingStatusResponses(files, capabilities)) } func (h *Handler) getFileProcessingSnapshot(c *gin.Context, admin bool) { @@ -728,7 +775,10 @@ func (h *Handler) getFileProcessingSnapshot(c *gin.Context, admin bool) { } response.Success(c, KnowledgeBaseFileProcessingSnapshotResponse{ KnowledgeBase: toKnowledgeBaseResponse(*item), - Statuses: toKnowledgeBaseFileProcessingStatusResponses(files), + Statuses: toKnowledgeBaseFileProcessingStatusResponses( + files, + h.service.ResolveFileVectorizationCapabilities(c.Request.Context(), files), + ), }) } @@ -740,12 +790,13 @@ func writeList(c *gin.Context, items []domainknowledgebase.KnowledgeBase, total response.SuccessPage(c, total, toKnowledgeBaseResponses(items)) } -func writeFileList(c *gin.Context, items []domainconversation.FileObject, total int64, err error) { +func (h *Handler) writeFileList(c *gin.Context, items []domainconversation.FileObject, total int64, err error) { if err != nil { writeError(c, err) return } - response.SuccessPage(c, total, toKnowledgeBaseFileResponses(items)) + capabilities := h.service.ResolveFileVectorizationCapabilities(c.Request.Context(), items) + response.SuccessPage(c, total, toKnowledgeBaseFileResponses(items, capabilities)) } func writeInput(req WriteKnowledgeBaseRequest) appknowledgebase.WriteInput { diff --git a/backend/internal/transport/http/knowledgebase/router.go b/backend/internal/transport/http/knowledgebase/router.go index 4f1b74b61..002435e3d 100644 --- a/backend/internal/transport/http/knowledgebase/router.go +++ b/backend/internal/transport/http/knowledgebase/router.go @@ -26,6 +26,7 @@ func (m *Module) RegisterAdminRoutes(adminGroup *gin.RouterGroup) { adminGroup.GET("/knowledge-bases/:id", m.Handler.GetAdmin) adminGroup.GET("/knowledge-bases/files", m.Handler.ListPlatformFiles) adminGroup.POST("/knowledge-bases/files", m.Handler.UploadAdminFile) + adminGroup.POST("/knowledge-bases/files/embeddings", m.Handler.SubmitAdminFileEmbeddings) adminGroup.GET("/knowledge-bases/files/:file_id/content", m.Handler.GetPlatformFileContent) adminGroup.DELETE("/knowledge-bases/files/:file_id", m.Handler.DeleteAdminFile) adminGroup.PATCH("/knowledge-bases/:id", m.Handler.PatchAdmin) diff --git a/backend/internal/transport/http/settings/handler_embedding_test.go b/backend/internal/transport/http/settings/handler_embedding_test.go index 03e58ae58..5612c16f0 100644 --- a/backend/internal/transport/http/settings/handler_embedding_test.go +++ b/backend/internal/transport/http/settings/handler_embedding_test.go @@ -103,10 +103,18 @@ func (testEmbeddingRepo) GetActiveFileObjectByID(context.Context, uint, string) return nil, nil } +func (testEmbeddingRepo) GetActiveFileObjectsByIDs(context.Context, uint, []string) ([]domainconversation.FileObject, error) { + return nil, nil +} + func (testEmbeddingRepo) GetFileObjectProcessingByObjectID(context.Context, uint) (*domainconversation.FileObjectProcessing, error) { return nil, nil } +func (testEmbeddingRepo) QueueFileEmbedding(context.Context, uint, string, string) (bool, error) { + return true, nil +} + func (testEmbeddingRepo) ClaimFileEmbedding(context.Context, uint, string, string) (bool, error) { return true, nil } diff --git a/frontend/features/admin/components/sections/files/admin-files.tsx b/frontend/features/admin/components/sections/files/admin-files.tsx index af6e117d4..ed3523ab4 100644 --- a/frontend/features/admin/components/sections/files/admin-files.tsx +++ b/frontend/features/admin/components/sections/files/admin-files.tsx @@ -1,26 +1,24 @@ "use client"; -import * as React from "react"; import { Save } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; - -import { - SettingsFieldEditor, - type ServiceRuntimeActionName, - type SettingsFieldServiceRuntime, -} from "../shared/settings-runtime-panel"; import { Button } from "@/components/ui/button"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import { - SettingsFieldInset, - SettingsFieldItem, - SettingsFieldList, - SettingsPage, - SettingsSection, - SettingsSectionSeparator, -} from "@/shared/components/settings-layout"; + type AdminEmbeddingIndexStatus, + getAdminDoclingRuntime, + getAdminEmbeddingRuntime, + getAdminEmbeddingStatus, + getAdminMinerURuntime, + getAdminRapidOCRRuntime, + getAdminTesseractRuntime, + getAdminTikaRuntime, + listAdminSettings, + patchAdminSettings, + triggerAdminEmbeddingReindex, +} from "@/features/admin/api"; import { applySettingsDefaults, denormalizeMBValue, @@ -34,42 +32,41 @@ import { mergeAllowedMIMETypes, normalizeMinerUFileTypes, OCR_ENGINES, - resolveMissingMinerUMIMETypes, - resolveActiveServices, resolveFieldID, resolveMinerUFileTypeFormats, resolveMinerUSource, + resolveMissingMinerUMIMETypes, resolveOCREngine, resolveVisibleFieldBlocks, resolveVisibleFields, SERVICE_LABELS, - SERVICE_NAMES, SETTINGS_GROUPS, - TIKA_SERVICE_SOURCES, - usesTika, type ServiceName, type ServiceRuntimeData, type ServiceState, type SettingsField, type SettingsGroup, + TIKA_SERVICE_SOURCES, + usesTika, } from "@/features/admin/model/files-settings"; -import { - type AdminEmbeddingIndexStatus, - getAdminDoclingRuntime, - getAdminEmbeddingRuntime, - getAdminEmbeddingStatus, - getAdminMinerURuntime, - getAdminRapidOCRRuntime, - getAdminTesseractRuntime, - getAdminTikaRuntime, - listAdminSettings, - patchAdminSettings, - triggerAdminEmbeddingReindex, -} from "@/features/admin/api"; import { resolveAdminErrorMessage } from "@/features/admin/utils/admin-error"; import { cn } from "@/lib/utils"; import type { PatchSettingItem } from "@/shared/api/settings.types"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { + SettingsFieldInset, + SettingsFieldItem, + SettingsFieldList, + SettingsPage, + SettingsSection, + SettingsSectionSeparator, +} from "@/shared/components/settings-layout"; import { configuredSettingsMap, settingHasValue } from "@/shared/lib/settings-meta"; +import { + type ServiceRuntimeActionName, + SettingsFieldEditor, + type SettingsFieldServiceRuntime, +} from "../shared/settings-runtime-panel"; const SERVICE_LOADERS: Record Promise> = { tika: getAdminTikaRuntime, @@ -230,20 +227,6 @@ export function AdminFilesSettingsPage() { [loadServiceRuntime, t], ); - const syncServiceRuntimes = React.useCallback( - (flattened: Record) => { - const active = resolveActiveServices(flattened); - for (const name of SERVICE_NAMES) { - if (active.has(name)) { - void loadServiceRuntime(name); - } else { - setServiceStates((prev) => ({ ...prev, [name]: { ...prev[name], data: null } })); - } - } - }, - [loadServiceRuntime], - ); - const loadSettings = React.useCallback(async () => { setLoading(true); try { @@ -257,7 +240,7 @@ export function AdminFilesSettingsPage() { setConfiguredMap(configuredSettingsMap(grouped)); setSettingsMap(flattened); setSavedMap(flattened); - syncServiceRuntimes(flattened); + setServiceStates(INITIAL_SERVICE_STATES); if (flattened["file.embedding_enabled"] === EMBEDDING_MODES.ON) { void loadEmbeddingStatus(); } else { @@ -268,7 +251,7 @@ export function AdminFilesSettingsPage() { } finally { setLoading(false); } - }, [clearEmbeddingStatus, loadEmbeddingStatus, syncServiceRuntimes, t]); + }, [clearEmbeddingStatus, loadEmbeddingStatus, t]); React.useEffect(() => { void loadSettings(); @@ -360,7 +343,7 @@ export function AdminFilesSettingsPage() { "file.allowed_mime_types": savedValue, })); setSavedMap(flattened); - syncServiceRuntimes(flattened); + setServiceStates(INITIAL_SERVICE_STATES); toast.success(t("toast.mimeTypesUpdated")); } catch (error) { toast.error(t("toast.saveFailed"), { description: resolveAdminErrorMessage(error, t("toast.unknownError")) }); @@ -368,7 +351,7 @@ export function AdminFilesSettingsPage() { setSaving(false); } }, - [syncServiceRuntimes, t], + [t], ); const resolveServiceRuntime = React.useCallback( @@ -595,7 +578,7 @@ export function AdminFilesSettingsPage() { setConfiguredMap(configuredSettingsMap(grouped)); setSettingsMap(flattened); setSavedMap(flattened); - syncServiceRuntimes(flattened); + setServiceStates(INITIAL_SERVICE_STATES); if (embeddingModelWillChange) { toast.warning(t("toast.embeddingModelChanged"), { description: t("toast.embeddingModelChangedDescription"), @@ -623,7 +606,7 @@ export function AdminFilesSettingsPage() { setSaving(false); } }, - [clearEmbeddingStatus, configuredMap, dirtyFieldIDs, loadEmbeddingStatus, savedMap, settingsMap, syncServiceRuntimes, t], + [clearEmbeddingStatus, configuredMap, dirtyFieldIDs, loadEmbeddingStatus, savedMap, settingsMap, t], ); const requestSaveGroup = React.useCallback((group: SettingsGroup) => { diff --git a/frontend/features/admin/model/files-settings.ts b/frontend/features/admin/model/files-settings.ts index 27eeb43b4..017885c79 100644 --- a/frontend/features/admin/model/files-settings.ts +++ b/frontend/features/admin/model/files-settings.ts @@ -1,5 +1,5 @@ -import type { SettingsGrouped } from "@/shared/api/settings.types"; import type { AdminServiceRuntimeView } from "@/features/admin/api/admin.types"; +import type { SettingsGrouped } from "@/shared/api/settings.types"; export type SettingsFieldType = "int" | "bool" | "string" | "password" | "textarea" | "select" | "tabs" | "multi-check" | "button"; @@ -163,8 +163,6 @@ export const OCR_ENABLED_RULE: VisibilityRule = { ], }; -export const SERVICE_NAMES: ServiceName[] = ["tika", "docling", "mineru", "tesseract", "rapidocr", "embedding"]; - export const SERVICE_LABELS: Record = { tika: "Tika", docling: "Docling", @@ -997,20 +995,6 @@ export function mergeAllowedMIMETypes(raw: string, items: MinerUMIMERequirement[ return merged.join(","); } -export function resolveActiveServices(settings: Record): Set { - const active = new Set(); - const engine = settings["extract.engine"] ?? ""; - if (engine === EXTRACT_ENGINE_POLICIES.TIKA) active.add("tika"); - if (engine === EXTRACT_ENGINE_POLICIES.DOCLING) active.add("docling"); - if (engine === EXTRACT_ENGINE_POLICIES.MINERU) active.add("mineru"); - const ocrEnabled = matchesVisibilityRule(OCR_ENABLED_RULE, settings); - const ocr = ocrEnabled ? resolveOCREngine(settings["extract.ocr_engine"] ?? "") : ""; - if (ocr === OCR_ENGINES.TESSERACT) active.add("tesseract"); - if (ocr === OCR_ENGINES.RAPIDOCR) active.add("rapidocr"); - if (settings["file.embedding_enabled"] === EMBEDDING_MODES.ON) active.add("embedding"); - return active; -} - export function isServiceDirty( name: ServiceName, settingsMap: Record, diff --git a/frontend/features/files/components/app-files.tsx b/frontend/features/files/components/app-files.tsx index 805a04105..1a71d68ca 100644 --- a/frontend/features/files/components/app-files.tsx +++ b/frontend/features/files/components/app-files.tsx @@ -1,13 +1,6 @@ "use client"; import { useTranslations } from "next-intl"; - -import { ContentHeader } from "@/features/files/components/sections/content/content-header"; -import { ContentPreview } from "@/features/files/components/sections/content/content-preview"; -import { SidebarHeader } from "@/features/files/components/sections/sidebar/sidebar-header"; -import { SidebarList } from "@/features/files/components/sections/sidebar/sidebar-list"; -import { StorageQuotaPanel } from "@/features/files/components/sections/storage/storage-quota-panel"; -import { useFilesPage } from "@/features/files/hooks/use-files-page"; import { AlertDialog, AlertDialogAction, @@ -18,9 +11,16 @@ import { AlertDialogHeader, AlertDialogTitle, } from "@/components/ui/alert-dialog"; +import { ContentHeader } from "@/features/files/components/sections/content/content-header"; +import { ContentPreview } from "@/features/files/components/sections/content/content-preview"; +import { SidebarHeader } from "@/features/files/components/sections/sidebar/sidebar-header"; +import { SidebarList } from "@/features/files/components/sections/sidebar/sidebar-list"; +import { StorageQuotaPanel } from "@/features/files/components/sections/storage/storage-quota-panel"; +import { useFilesPage } from "@/features/files/hooks/use-files-page"; +import { cn } from "@/lib/utils"; import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; import { useIsMobile } from "@/shared/hooks/use-mobile"; -import { cn } from "@/lib/utils"; +import { canManuallyVectorizeFile } from "@/shared/lib/file-processing"; export function AppFiles() { const tCommon = useTranslations("common.actions"); @@ -42,6 +42,8 @@ export function AppFiles() { selectedFileIDs, bulkDeleteOpen, bulkDeleting, + vectorizing, + vectorizingFileIDs, hasMore, query, sortKey, @@ -79,11 +81,17 @@ export function AppFiles() { onBulkDeleteRequest, onClearBulkDelete, onConfirmBulkDelete, + onVectorizeFile, + onVectorizeSelected, onBackToList, onToggleRagOptOut, } = useFilesPage(); const stableDeleteTarget = useDialogSnapshot(deleteTarget); const selectedCount = selectedFileIDs.length; + const selectedFileIDSet = new Set(selectedFileIDs); + const vectorizableSelectedCount = files.filter( + (file) => selectedFileIDSet.has(file.fileID) && canManuallyVectorizeFile(file), + ).length; const sidebarCollapsed = !isMobileViewport && isSidebarCollapsed; const selectAllDisabled = loading || files.length === 0 || bulkDeleting; const contentDeleting = Boolean(selectedFile && deletingFileID === selectedFile.fileID); @@ -112,8 +120,10 @@ export function AppFiles() { sortKey={sortKey} uploading={uploading} selectedCount={selectedCount} + vectorizableSelectedCount={vectorizableSelectedCount} selectAllDisabled={selectAllDisabled} bulkDeleteDisabled={bulkDeleting} + vectorizing={vectorizing} collapsed={sidebarCollapsed} showCollapseButton={!isMobileViewport} onToggleCollapsed={onToggleSidebarCollapsed} @@ -124,6 +134,7 @@ export function AppFiles() { onSelectLoaded={onSelectLoadedFiles} onClearSelection={onClearFileSelection} onBulkDeleteRequest={onBulkDeleteRequest} + onVectorizeSelected={onVectorizeSelected} onUpload={onOpenUploadPicker} /> @@ -136,6 +147,7 @@ export function AppFiles() { loadingMore={loadingMore} hasMore={hasMore} syncing={syncing} + vectorizingFileIDs={vectorizingFileIDs} renamingFileID={renamingFileID} renameValue={renameValue} onSelect={onSelectFile} @@ -145,6 +157,7 @@ export function AppFiles() { onRenameValueChange={onRenameValueChange} onRenameCommit={onRenameCommit} onRenameCancel={onRenameCancel} + onVectorize={(fileID) => void onVectorizeFile(fileID)} onDeleteRequest={onDeleteRequest} /> ) : null} @@ -160,11 +173,13 @@ export function AppFiles() { file={selectedFile} preview={preview} deleting={contentDeleting} + vectorizing={selectedFile ? vectorizingFileIDs.includes(selectedFile.fileID) : false} onBack={mobileView === "detail" ? onBackToList : undefined} onOpen={openPreview} onDownload={downloadPreview} onDeleteRequest={onDeleteRequest} onToggleRagOptOut={onToggleRagOptOut} + onVectorize={onVectorizeFile} /> void; onOpen: () => void; onDownload: () => void; onDeleteRequest: (file: FileObjectDTO) => void; onToggleRagOptOut: (fileID: string, current: boolean) => Promise; + onVectorize: (fileID: string) => Promise; }; function resolveRawFileTypeLabel(file: FileObjectDTO): string { @@ -43,11 +44,13 @@ export function ContentHeader({ file, preview, deleting, + vectorizing, onBack, onOpen, onDownload, onDeleteRequest, onToggleRagOptOut, + onVectorize, }: ContentHeaderProps) { const tCommon = useTranslations("common.actions"); const t = useTranslations("files"); @@ -151,6 +154,20 @@ export function ContentHeader({
+ {canManuallyVectorizeFile(file) ? ( + + ) : null}
) : null} -
- - - - - - - -
- { - event.preventDefault(); - onFilterToggle("all"); - }} - > - - {t("filters.all")} - {!hasActiveFilters ? : null} - - - - - {FILE_FILTER_OPTIONS.filter((item) => item.value !== "all").map((item) => { - const value = item.value as FileFilterValue; - const active = activeFilterSet.has(value); - return ( - { - event.preventDefault(); - onFilterToggle(value); - }} - > - - {t(`filters.${item.value}`)} - {active ? : null} - - ); - })} -
- - - - - - - - -
- {FILE_SORT_OPTIONS.map((item) => { - const active = item.value === sortKey; - return ( - onSortChange(item.value)} - > - - {t(item.value === "last_used" ? "sort.lastUsed" : `sort.${item.value}`)} - {active ? : null} - - ); - })} -
- - - - {selectedCount > 0 ? ( +
+
- ) : null} + + {selectedCount === 0 ? ( + <> + + + + + +
+ { + event.preventDefault(); + onFilterToggle("all"); + }} + > + + {t("filters.all")} + {!hasActiveFilters ? : null} + + + + + {FILE_FILTER_OPTIONS.filter((item) => item.value !== "all").map((item) => { + const value = item.value as FileFilterValue; + const active = activeFilterSet.has(value); + return ( + { + event.preventDefault(); + onFilterToggle(value); + }} + > + + {t(`filters.${item.value}`)} + {active ? : null} + + ); + })} +
+ + + + + + + + +
+ {FILE_SORT_OPTIONS.map((item) => { + const active = item.value === sortKey; + return ( + onSortChange(item.value)} + > + + {t(item.value === "last_used" ? "sort.lastUsed" : `sort.${item.value}`)} + {active ? : null} + + ); + })} +
+ + + + ) : ( + <> + + + + )} +
); diff --git a/frontend/features/files/components/sections/sidebar/sidebar-list.tsx b/frontend/features/files/components/sections/sidebar/sidebar-list.tsx index 788038213..c50e0f1f2 100644 --- a/frontend/features/files/components/sections/sidebar/sidebar-list.tsx +++ b/frontend/features/files/components/sections/sidebar/sidebar-list.tsx @@ -1,15 +1,11 @@ "use client"; -import * as React from "react"; -import { Ellipsis, PencilLine, SquareCheckBig, Trash2, Zap } from "lucide-react"; +import { DatabaseZap, Ellipsis, PencilLine, SquareCheckBig, Trash2, Zap } from "lucide-react"; import { useTranslations } from "next-intl"; - -import { resolveFileIcon } from "@/shared/lib/file-display"; +import * as React from "react"; +import { AnimatedText } from "@/components/ui/animated-text"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; -import { CenteredEmptyState } from "@/components/ui/empty-state"; -import { AnimatedText } from "@/components/ui/animated-text"; -import { Spinner } from "@/components/ui/spinner"; import { DropdownMenu, DropdownMenuContent, @@ -17,10 +13,14 @@ import { DropdownMenuItemIcon, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { CenteredEmptyState } from "@/components/ui/empty-state"; import { Input } from "@/components/ui/input"; -import { useLoadMoreSentinel } from "@/shared/hooks/use-load-more-sentinel"; +import { Spinner } from "@/components/ui/spinner"; import { cn } from "@/lib/utils"; import type { FileObjectDTO } from "@/shared/api/file.types"; +import { useLoadMoreSentinel } from "@/shared/hooks/use-load-more-sentinel"; +import { resolveFileIcon } from "@/shared/lib/file-display"; +import { canManuallyVectorizeFile, isVectorIndexOutdated } from "@/shared/lib/file-processing"; type SidebarListProps = { items: FileObjectDTO[]; @@ -30,6 +30,7 @@ type SidebarListProps = { loadingMore: boolean; hasMore: boolean; syncing: boolean; + vectorizingFileIDs: string[]; renamingFileID: string | null; renameValue: string; onSelect: (fileID: string) => void; @@ -39,6 +40,7 @@ type SidebarListProps = { onRenameValueChange: (value: string) => void; onRenameCommit: (fileID: string, currentFileName: string) => void; onRenameCancel: () => void; + onVectorize: (fileID: string) => void; onDeleteRequest: (item: FileObjectDTO) => void; }; @@ -46,6 +48,8 @@ function SidebarListItem({ item, selected, checked, + vectorizing, + vectorizationBusy, renaming, renameValue, onSelect, @@ -54,11 +58,14 @@ function SidebarListItem({ onRenameValueChange, onRenameCommit, onRenameCancel, + onVectorize, onDeleteRequest, }: { item: FileObjectDTO; selected: boolean; checked: boolean; + vectorizing: boolean; + vectorizationBusy: boolean; renaming: boolean; renameValue: string; onSelect: (fileID: string) => void; @@ -67,11 +74,13 @@ function SidebarListItem({ onRenameValueChange: (value: string) => void; onRenameCommit: (fileID: string, currentFileName: string) => void; onRenameCancel: () => void; + onVectorize: (fileID: string) => void; onDeleteRequest: (item: FileObjectDTO) => void; }) { const t = useTranslations("files"); const fileIcon = resolveFileIcon(item); const showsRetrievalStatus = item.fileCategory !== "image" && item.embedStatus === "ready"; + const vectorizable = canManuallyVectorizeFile(item); const [actionsMenuOpen, setActionsMenuOpen] = React.useState(false); if (renaming) { @@ -192,6 +201,19 @@ function SidebarListItem({ {t("actions.rename")} + {vectorizable ? ( + onVectorize(item.fileID)} + > + {vectorizing ? ( + + ) : ( + + )} + {t(isVectorIndexOutdated(item) ? "actions.updateIndex" : "actions.vectorize")} + + ) : null} @@ -224,6 +246,7 @@ export function SidebarList({ loadingMore, hasMore, syncing, + vectorizingFileIDs, renamingFileID, renameValue, onSelect, @@ -233,11 +256,13 @@ export function SidebarList({ onRenameValueChange, onRenameCommit, onRenameCancel, + onVectorize, onDeleteRequest, }: SidebarListProps) { const t = useTranslations("files"); const scrollAreaRef = React.useRef(null); const selectedFileIDSet = React.useMemo(() => new Set(selectedFileIDs), [selectedFileIDs]); + const vectorizingFileIDSet = React.useMemo(() => new Set(vectorizingFileIDs), [vectorizingFileIDs]); const loadMoreRef = useLoadMoreSentinel({ enabled: hasMore && !loading && !loadingMore, @@ -283,6 +308,8 @@ export function SidebarList({ item={item} selected={isSelected} checked={isChecked} + vectorizing={vectorizingFileIDSet.has(item.fileID)} + vectorizationBusy={vectorizingFileIDs.length > 0} renaming={isRenaming} renameValue={renameValue} onSelect={onSelect} @@ -291,6 +318,7 @@ export function SidebarList({ onRenameValueChange={onRenameValueChange} onRenameCommit={onRenameCommit} onRenameCancel={onRenameCancel} + onVectorize={onVectorize} onDeleteRequest={onDeleteRequest} /> ); diff --git a/frontend/features/files/hooks/use-files-page.ts b/frontend/features/files/hooks/use-files-page.ts index 267c413de..1ff55c82a 100644 --- a/frontend/features/files/hooks/use-files-page.ts +++ b/frontend/features/files/hooks/use-files-page.ts @@ -1,21 +1,19 @@ "use client"; -import * as React from "react"; import { useSearchParams } from "next/navigation"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; - -import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; import { useFileExtract } from "@/features/files/hooks/use-file-extract"; import { useFileInvalidation } from "@/features/files/hooks/use-file-invalidation"; import { useFilePreview } from "@/features/files/hooks/use-file-preview"; import type { FileFilterValue, FileSortKey } from "@/features/files/types/files"; -import { resolveFileFilter } from "@/shared/lib/file-display"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; import { deleteFile, listFiles, renameFile, + submitFileEmbeddings, updateFileRagOptOut, uploadFile, } from "@/shared/api/file"; @@ -25,12 +23,14 @@ import type { UploadFileResult, UserStorageQuotaDTO, } from "@/shared/api/file.types"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import { - useFileProcessingStatusPolling, type FileStatusPollingResult, + useFileProcessingStatusPolling, } from "@/shared/hooks/use-file-processing-status-polling"; import { runBulkActionInChunks, runSettledItemsWithConcurrency } from "@/shared/lib/bulk-action"; -import { isFileProcessing } from "@/shared/lib/file-processing"; +import { resolveFileFilter } from "@/shared/lib/file-display"; +import { canManuallyVectorizeFile, isFileProcessing } from "@/shared/lib/file-processing"; import { patchByID, replaceByID, upsertByID } from "@/shared/lib/optimistic-list"; const FILES_PAGE_SIZE = 100; @@ -63,6 +63,8 @@ type UseFilesPageResult = { selectedFileIDs: string[]; bulkDeleteOpen: boolean; bulkDeleting: boolean; + vectorizing: boolean; + vectorizingFileIDs: string[]; hasMore: boolean; query: string; sortKey: FileSortKey; @@ -100,6 +102,8 @@ type UseFilesPageResult = { onBulkDeleteRequest: () => void; onClearBulkDelete: () => void; onConfirmBulkDelete: () => Promise; + onVectorizeFile: (fileID: string) => Promise; + onVectorizeSelected: () => Promise; onBackToList: () => void; onToggleRagOptOut: (fileID: string, current: boolean) => Promise; }; @@ -136,6 +140,7 @@ export function useFilesPage(): UseFilesPageResult { const [selectedFileIDs, setSelectedFileIDs] = React.useState([]); const [bulkDeleteOpen, setBulkDeleteOpen] = React.useState(false); const [bulkDeleting, setBulkDeleting] = React.useState(false); + const [vectorizingFileIDs, setVectorizingFileIDs] = React.useState([]); const [nextPage, setNextPage] = React.useState(2); const [hasMore, setHasMore] = React.useState(false); const [query, setQuery] = React.useState(""); @@ -374,6 +379,8 @@ export function useFilesPage(): UseFilesPageResult { item.embedStatus === status.embedStatus && item.embedError === status.embedError && item.chunkCount === status.chunkCount && + item.canVectorize === status.canVectorize && + item.vectorizationReason === status.vectorizationReason && item.updatedAt === status.updatedAt )) { nextFiles.push(item); @@ -392,6 +399,8 @@ export function useFilesPage(): UseFilesPageResult { embedStatus: status.embedStatus, embedError: status.embedError, chunkCount: status.chunkCount, + canVectorize: status.canVectorize, + vectorizationReason: status.vectorizationReason, updatedAt: status.updatedAt, }); } @@ -425,6 +434,7 @@ export function useFilesPage(): UseFilesPageResult { () => files.find((item) => item.fileID === selectedFileID) ?? null, [files, selectedFileID], ); + const vectorizing = vectorizingFileIDs.length > 0; const { preview, open, download } = useFilePreview({ file: selectedFile, @@ -675,6 +685,77 @@ export function useFilesPage(): UseFilesPageResult { toast.success(t("toasts.bulkDeleteSucceeded", { count: successCount })); }, [ensureAccessToken, loadFiles, selectedFileID, selectedFileIDs, t]); + const submitVectorization = React.useCallback(async (fileIDs: string[], clearSelection: boolean) => { + if (vectorizing) { + return; + } + const normalizedFileIDs = Array.from(new Set(fileIDs.map((fileID) => fileID.trim()).filter(Boolean))); + if (normalizedFileIDs.length === 0) { + return; + } + if (normalizedFileIDs.length > 100) { + toast.error(t("toasts.vectorizeLimit")); + return; + } + + setVectorizingFileIDs(normalizedFileIDs); + try { + const token = await ensureAccessToken(); + if (!token) { + toast.error(t("toasts.sessionExpired"), { description: t("toasts.operateAfterLogin") }); + return; + } + const result = await submitFileEmbeddings(token, normalizedFileIDs); + const submittedFileIDSet = new Set(result.submittedFileIDs); + const failedCount = result.skipped.filter(({ reason }) => ["queue_busy", "submit_failed"].includes(reason)).length; + if (submittedFileIDSet.size > 0) { + const nextFiles = filesRef.current.map((item) => submittedFileIDSet.has(item.fileID) + ? { + ...item, + embedStatus: "queued", + embedError: "", + canVectorize: false, + vectorizationReason: "processing", + } + : item); + filesRef.current = nextFiles; + setFiles(nextFiles); + } + if (clearSelection) { + setSelectedFileIDs((current) => current.filter((fileID) => !submittedFileIDSet.has(fileID))); + } + if (failedCount > 0) { + toast.warning(t("toasts.vectorizePartial", { submitted: result.submittedFileIDs.length, failed: failedCount })); + } else if (result.submittedFileIDs.length === 0) { + toast.info(t("toasts.vectorizeNoChanges")); + } else { + toast.success(t("toasts.vectorizeSubmitted", { count: result.submittedFileIDs.length }), { + description: result.skipped.length > 0 + ? t("toasts.vectorizeSkipped", { count: result.skipped.length }) + : undefined, + }); + } + } catch (error) { + toast.error(t("toasts.vectorizeFailed"), { + description: resolveErrorMessage(error, t("toasts.vectorizeFailed")), + }); + } finally { + setVectorizingFileIDs([]); + } + }, [ensureAccessToken, resolveErrorMessage, t, vectorizing]); + + const onVectorizeFile = React.useCallback(async (fileID: string) => { + await submitVectorization([fileID], false); + }, [submitVectorization]); + + const onVectorizeSelected = React.useCallback(async () => { + const selectedFileIDSet = new Set(selectedFileIDs); + const fileIDs = filesRef.current + .filter((file) => selectedFileIDSet.has(file.fileID) && canManuallyVectorizeFile(file)) + .map((file) => file.fileID); + await submitVectorization(fileIDs, true); + }, [selectedFileIDs, submitVectorization]); + const onRenameCommit = React.useCallback( async (fileID: string, currentFileName: string) => { const nextFileName = renameValue.trim(); @@ -855,6 +936,8 @@ export function useFilesPage(): UseFilesPageResult { selectedFileIDs, bulkDeleteOpen, bulkDeleting, + vectorizing, + vectorizingFileIDs, hasMore, query, sortKey, @@ -892,6 +975,8 @@ export function useFilesPage(): UseFilesPageResult { onBulkDeleteRequest, onClearBulkDelete, onConfirmBulkDelete, + onVectorizeFile, + onVectorizeSelected, onBackToList, onToggleRagOptOut, }; diff --git a/frontend/features/knowledge-bases/components/admin-knowledge-bases.tsx b/frontend/features/knowledge-bases/components/admin-knowledge-bases.tsx index fb737ebc0..aa1e771d7 100644 --- a/frontend/features/knowledge-bases/components/admin-knowledge-bases.tsx +++ b/frontend/features/knowledge-bases/components/admin-knowledge-bases.tsx @@ -7,14 +7,6 @@ import * as React from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; -import { Spinner } from "@/components/ui/spinner"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; import { Dialog, DialogContent, @@ -22,6 +14,14 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Spinner } from "@/components/ui/spinner"; import { Table, TableBody, @@ -235,12 +235,19 @@ export function AdminKnowledgeBases({ page }: { page: KnowledgeBasesPageModel }) loadingMore={detail.filesLoadingMore} removingFileID={detail.removingFileID} toggling={detail.toggling} + selectedFileIDs={detail.selectedFileIDs} + vectorizingFileIDs={detail.vectorizingFileIDs} onBack={() => setDetailOpen(false)} onAddFiles={detail.addFiles} onLoadMore={detail.loadMoreFiles} onRemoveFile={detail.removeFile} onToggleEnabled={detail.toggleBuiltinEnabled} onPreviewFile={detail.previewFile} + onToggleFileSelection={detail.toggleFileSelection} + onSelectVectorizableFiles={detail.selectVectorizableFiles} + onClearFileSelection={detail.clearFileSelection} + onVectorizeFile={detail.vectorizeFile} + onVectorizeSelectedFiles={detail.vectorizeSelectedFiles} /> diff --git a/frontend/features/knowledge-bases/components/app-knowledge-bases.tsx b/frontend/features/knowledge-bases/components/app-knowledge-bases.tsx index 8b7d8c4fd..c5befc555 100644 --- a/frontend/features/knowledge-bases/components/app-knowledge-bases.tsx +++ b/frontend/features/knowledge-bases/components/app-knowledge-bases.tsx @@ -3,13 +3,13 @@ import dynamic from "next/dynamic"; import { AdminKnowledgeBases } from "@/features/knowledge-bases/components/admin-knowledge-bases"; +import { KnowledgeBaseDetail } from "@/features/knowledge-bases/components/knowledge-base-detail"; import { AddKnowledgeBaseFilesDialog, BulkDeleteKnowledgeBasesDialog, DeleteKnowledgeBaseDialog, KnowledgeBaseEditorDialog, } from "@/features/knowledge-bases/components/knowledge-base-dialogs"; -import { KnowledgeBaseDetail } from "@/features/knowledge-bases/components/knowledge-base-detail"; import { KnowledgeBaseSidebar } from "@/features/knowledge-bases/components/knowledge-base-sidebar"; import { useKnowledgeBasesPage } from "@/features/knowledge-bases/hooks/use-knowledge-bases-page"; import type { KnowledgeBaseMode } from "@/features/knowledge-bases/types/knowledge-bases"; @@ -68,12 +68,19 @@ export function AppKnowledgeBases({ mode = "user" }: { mode?: KnowledgeBaseMode loadingMore={detail.filesLoadingMore} removingFileID={detail.removingFileID} toggling={detail.toggling} + selectedFileIDs={detail.selectedFileIDs} + vectorizingFileIDs={detail.vectorizingFileIDs} onBack={detail.back} onAddFiles={detail.addFiles} onLoadMore={detail.loadMoreFiles} onRemoveFile={detail.removeFile} onToggleEnabled={detail.toggleBuiltinEnabled} onPreviewFile={detail.previewFile} + onToggleFileSelection={detail.toggleFileSelection} + onSelectVectorizableFiles={detail.selectVectorizableFiles} + onClearFileSelection={detail.clearFileSelection} + onVectorizeFile={detail.vectorizeFile} + onVectorizeSelectedFiles={detail.vectorizeSelectedFiles} /> } diff --git a/frontend/features/knowledge-bases/components/knowledge-base-detail.tsx b/frontend/features/knowledge-bases/components/knowledge-base-detail.tsx index 13f5e07ef..2d7613e52 100644 --- a/frontend/features/knowledge-bases/components/knowledge-base-detail.tsx +++ b/frontend/features/knowledge-bases/components/knowledge-base-detail.tsx @@ -1,21 +1,22 @@ "use client"; -import { ArrowLeft, BookOpen, Link2Off, Plus } from "lucide-react"; +import { ArrowLeft, BookOpen, DatabaseZap, Link2Off, Plus } from "lucide-react"; import { useTranslations } from "next-intl"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { CenteredEmptyState } from "@/components/ui/empty-state"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import type { - KnowledgeBaseMode, KnowledgeBaseMobileView, + KnowledgeBaseMode, } from "@/features/knowledge-bases/types/knowledge-bases"; import { cn } from "@/lib/utils"; import type { KnowledgeBaseDTO, KnowledgeBaseFileDTO } from "@/shared/api/knowledge-bases.types"; import { formatBytes, resolveFileIcon } from "@/shared/lib/file-display"; -import { resolveFileRetrievalBadge } from "@/shared/lib/file-processing"; +import { canManuallyVectorizeFile, isVectorIndexOutdated, resolveFileRetrievalBadge } from "@/shared/lib/file-processing"; type KnowledgeBaseDetailProps = { mode: KnowledgeBaseMode; @@ -27,12 +28,19 @@ type KnowledgeBaseDetailProps = { loadingMore: boolean; removingFileID: string; toggling: boolean; + selectedFileIDs: string[]; + vectorizingFileIDs: string[]; onBack: () => void; onAddFiles: () => void; onLoadMore: () => Promise; onRemoveFile: (fileID: string) => Promise; onToggleEnabled: (enabled: boolean) => Promise; onPreviewFile: (file: KnowledgeBaseFileDTO) => void; + onToggleFileSelection: (fileID: string, checked: boolean) => void; + onSelectVectorizableFiles: () => void; + onClearFileSelection: () => void; + onVectorizeFile: (fileID: string) => Promise; + onVectorizeSelectedFiles: () => Promise; }; export function KnowledgeBaseDetail({ @@ -45,12 +53,19 @@ export function KnowledgeBaseDetail({ loadingMore, removingFileID, toggling, + selectedFileIDs, + vectorizingFileIDs, onBack, onAddFiles, onLoadMore, onRemoveFile, onToggleEnabled, onPreviewFile, + onToggleFileSelection, + onSelectVectorizableFiles, + onClearFileSelection, + onVectorizeFile, + onVectorizeSelectedFiles, }: KnowledgeBaseDetailProps) { const t = useTranslations("knowledgeBases"); @@ -79,9 +94,16 @@ export function KnowledgeBaseDetail({ loading={loading} loadingMore={loadingMore} removingFileID={removingFileID} + selectedFileIDs={selectedFileIDs} + vectorizingFileIDs={vectorizingFileIDs} onLoadMore={onLoadMore} onRemoveFile={onRemoveFile} onPreviewFile={onPreviewFile} + onToggleFileSelection={onToggleFileSelection} + onSelectVectorizableFiles={onSelectVectorizableFiles} + onClearFileSelection={onClearFileSelection} + onVectorizeFile={onVectorizeFile} + onVectorizeSelectedFiles={onVectorizeSelectedFiles} /> ) : ( @@ -177,9 +199,16 @@ type KnowledgeBaseFileListProps = Pick< | "loading" | "loadingMore" | "removingFileID" + | "selectedFileIDs" + | "vectorizingFileIDs" | "onLoadMore" | "onRemoveFile" | "onPreviewFile" + | "onToggleFileSelection" + | "onSelectVectorizableFiles" + | "onClearFileSelection" + | "onVectorizeFile" + | "onVectorizeSelectedFiles" > & { selected: KnowledgeBaseDTO; }; @@ -192,15 +221,61 @@ function KnowledgeBaseFileList({ loading, loadingMore, removingFileID, + selectedFileIDs, + vectorizingFileIDs, onLoadMore, onRemoveFile, onPreviewFile, + onToggleFileSelection, + onSelectVectorizableFiles, + onClearFileSelection, + onVectorizeFile, + onVectorizeSelectedFiles, }: KnowledgeBaseFileListProps) { const t = useTranslations("knowledgeBases"); const tStatus = useTranslations("files.status"); + const editable = mode === "admin" || selected.scope === "user"; + const selectedFileIDSet = new Set(selectedFileIDs); + const vectorizingFileIDSet = new Set(vectorizingFileIDs); + const vectorizingFiles = vectorizingFileIDs.length > 0; + const vectorizableFiles = files.filter(canManuallyVectorizeFile); + const allVectorizableSelected = vectorizableFiles.length > 0 + && vectorizableFiles.every((file) => selectedFileIDSet.has(file.fileID)); return ( -
+
+ {editable && selectedFileIDs.length > 0 ? ( +
+ 0 ? "indeterminate" : false} + onCheckedChange={(checked) => checked ? onSelectVectorizableFiles() : onClearFileSelection()} + aria-label={t("selectVectorizableFiles")} + /> + {t("selectedVectorizeFiles", { count: selectedFileIDs.length })} + + +
+ ) : null} +
{loading ? (
@@ -214,18 +289,35 @@ function KnowledgeBaseFileList({ file, (key, values) => tStatus(key, values), ).label; - const editable = mode === "admin" || selected.scope === "user"; + const vectorizable = canManuallyVectorizeFile(file); + const outdatedIndex = isVectorIndexOutdated(file); + const checked = selectedFileIDSet.has(file.fileID); return ( -
+
+ {editable && vectorizable ? ( + event.stopPropagation()} + onCheckedChange={(value) => onToggleFileSelection(file.fileID, value === true)} + aria-label={t("selectFile", { name: file.fileName })} + /> + ) : null} {editable ? ( - +
+ {vectorizable ? ( + + ) : null} + +
) : null}
); @@ -296,6 +407,7 @@ function KnowledgeBaseFileList({ } /> )} +
); } diff --git a/frontend/features/knowledge-bases/hooks/use-knowledge-bases-page.ts b/frontend/features/knowledge-bases/hooks/use-knowledge-bases-page.ts index e2b10e085..97b969c8e 100644 --- a/frontend/features/knowledge-bases/hooks/use-knowledge-bases-page.ts +++ b/frontend/features/knowledge-bases/hooks/use-knowledge-bases-page.ts @@ -6,20 +6,20 @@ import { toast } from "sonner"; import type { KnowledgeBaseDraft, - KnowledgeBaseMode, KnowledgeBaseMobileView, + KnowledgeBaseMode, KnowledgeBasePreviewTarget, KnowledgeBaseSortKey, } from "@/features/knowledge-bases/types/knowledge-bases"; import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; -import { uploadFile } from "@/shared/api/file"; +import { submitFileEmbeddings, uploadFile } from "@/shared/api/file"; import { addAdminKnowledgeBaseFiles, addMyKnowledgeBaseFiles, createAdminKnowledgeBase, createMyKnowledgeBase, - deleteAdminKnowledgeBaseFile, deleteAdminKnowledgeBase, + deleteAdminKnowledgeBaseFile, deleteMyKnowledgeBase, fetchKnowledgeBaseFileContent, getKnowledgeBase, @@ -32,6 +32,7 @@ import { listVisibleKnowledgeBases, removeAdminKnowledgeBaseFile, removeMyKnowledgeBaseFile, + submitAdminPlatformFileEmbeddings, updateAdminKnowledgeBase, updateMyKnowledgeBase, uploadAdminKnowledgeBaseFile, @@ -50,11 +51,11 @@ import { } from "@/shared/events/knowledge-base-events"; import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; import { - useFileStatusPolling, type FileStatusPollingResult, + useFileStatusPolling, } from "@/shared/hooks/use-file-processing-status-polling"; import { runSettledBulkItems, runSettledItemsWithConcurrency } from "@/shared/lib/bulk-action"; -import { isFileProcessing } from "@/shared/lib/file-processing"; +import { canManuallyVectorizeFile, isFileProcessing } from "@/shared/lib/file-processing"; const FILE_ACTION_LIMIT = 100; const FILE_PAGE_SIZE = 100; @@ -78,6 +79,8 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { const [files, setFiles] = React.useState([]); const [filesTotal, setFilesTotal] = React.useState(0); const [filesPage, setFilesPage] = React.useState(1); + const [selectedDetailFileIDs, setSelectedDetailFileIDs] = React.useState([]); + const [vectorizingFileIDs, setVectorizingFileIDs] = React.useState([]); const [loading, setLoading] = React.useState(true); const [itemsTotal, setItemsTotal] = React.useState(0); const [itemsPage, setItemsPage] = React.useState(1); @@ -161,6 +164,17 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { setSelectedKnowledgeBaseIDs((current) => current.filter((id) => selectableIDs.has(id))); }, [selectableItems]); + React.useEffect(() => { + setSelectedDetailFileIDs([]); + }, [selectedID]); + + React.useEffect(() => { + const vectorizableIDs = new Set( + files.filter(canManuallyVectorizeFile).map((file) => file.fileID), + ); + setSelectedDetailFileIDs((current) => current.filter((fileID) => vectorizableIDs.has(fileID))); + }, [files]); + const listFilePage = React.useCallback(( accessToken: string, knowledgeBaseID: string, @@ -488,9 +502,13 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { file.processingStatus === status.processingStatus && file.processing === status.processing && file.processingReady === status.processingReady && + file.extractStatus === status.extractStatus && file.embedStatus === status.embedStatus && + file.embedError === status.embedError && file.chunkCount === status.chunkCount && file.ragOptOut === status.ragOptOut && + file.canVectorize === status.canVectorize && + file.vectorizationReason === status.vectorizationReason && file.updatedAt === status.updatedAt ) { nextFiles.push(file); @@ -792,6 +810,7 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { if (selectedIDRef.current === knowledgeBaseID) { setFiles((current) => current.filter((file) => file.fileID !== fileID)); setFilesTotal((current) => Math.max(0, current - 1)); + setSelectedDetailFileIDs((current) => current.filter((id) => id !== fileID)); } await loadItems(undefined, true); dispatchKnowledgeBaseInvalidated(knowledgeBaseID); @@ -803,6 +822,72 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { } }, [loadItems, mode, removingFileID, selected, t]); + const submitVectorization = React.useCallback(async (fileIDs: string[], clearSelection: boolean) => { + if (!selected || vectorizingFileIDs.length > 0 || (mode === "user" && selected.scope !== "user")) return; + const normalizedFileIDs = Array.from(new Set(fileIDs.map((fileID) => fileID.trim()).filter(Boolean))); + if (normalizedFileIDs.length === 0) return; + if (normalizedFileIDs.length > FILE_ACTION_LIMIT) { + toast.error(t("tooManyFiles", { max: FILE_ACTION_LIMIT })); + return; + } + + const knowledgeBaseID = selected.publicID; + setVectorizingFileIDs(normalizedFileIDs); + try { + const token = await requireAccessToken(); + const result = mode === "admin" + ? await submitAdminPlatformFileEmbeddings(token, normalizedFileIDs) + : await submitFileEmbeddings(token, normalizedFileIDs); + const submittedFileIDs = new Set(result.submittedFileIDs); + const failedCount = result.skipped.filter(({ reason }) => ["queue_busy", "submit_failed"].includes(reason)).length; + if (submittedFileIDs.size > 0 && selectedIDRef.current === knowledgeBaseID) { + const nextFiles = filesRef.current.map((file) => submittedFileIDs.has(file.fileID) + ? { + ...file, + processing: true, + embedStatus: "queued", + embedError: "", + canVectorize: false, + vectorizationReason: "processing", + } + : file); + filesRef.current = nextFiles; + setFiles(nextFiles); + } + if (clearSelection && selectedIDRef.current === knowledgeBaseID) { + setSelectedDetailFileIDs((current) => current.filter((fileID) => !submittedFileIDs.has(fileID))); + } + if (failedCount > 0) { + toast.warning(t("vectorizePartial", { submitted: result.submittedFileIDs.length, failed: failedCount })); + } else if (result.submittedFileIDs.length === 0) { + toast.info(t("vectorizeNoChanges")); + } else { + toast.success(t("vectorizeSubmitted", { count: result.submittedFileIDs.length }), { + description: result.skipped.length > 0 + ? t("vectorizeSkipped", { count: result.skipped.length }) + : undefined, + }); + } + } catch (error) { + toast.error(t("vectorizeFailed"), { description: resolveErrorMessage(error) }); + } finally { + setVectorizingFileIDs([]); + } + }, [mode, resolveErrorMessage, selected, t, vectorizingFileIDs.length]); + + const toggleDetailFileSelection = React.useCallback((fileID: string, checked: boolean) => { + setSelectedDetailFileIDs((current) => { + const next = new Set(current); + if (checked) { + if (next.size >= FILE_ACTION_LIMIT) return current; + next.add(fileID); + } else { + next.delete(fileID); + } + return Array.from(next); + }); + }, []); + const loadMoreFiles = React.useCallback(async () => { if (!selected || filesLoadingMore || files.length >= filesTotal) return; const knowledgeBaseID = selected.publicID; @@ -1000,10 +1085,19 @@ export function useKnowledgeBasesPage(mode: KnowledgeBaseMode) { }, detail: { selected, files, filesTotal, filesLoading, filesLoadingMore, removingFileID, toggling, + selectedFileIDs: selectedDetailFileIDs, + vectorizingFileIDs, back: () => setMobileView("list"), addFiles: () => setAddFilesOpen(true), loadMoreFiles, removeFile, + toggleFileSelection: toggleDetailFileSelection, + selectVectorizableFiles: () => setSelectedDetailFileIDs( + files.filter(canManuallyVectorizeFile).slice(0, FILE_ACTION_LIMIT).map((file) => file.fileID), + ), + clearFileSelection: () => setSelectedDetailFileIDs([]), + vectorizeFile: (fileID: string) => submitVectorization([fileID], false), + vectorizeSelectedFiles: () => submitVectorization(selectedDetailFileIDs, true), toggleBuiltinEnabled, previewFile: (file: KnowledgeBaseFileDTO) => { if (!selected) return; diff --git a/frontend/i18n/messages/en-US/errors.json b/frontend/i18n/messages/en-US/errors.json index aa784dfe0..82e219915 100644 --- a/frontend/i18n/messages/en-US/errors.json +++ b/frontend/i18n/messages/en-US/errors.json @@ -149,7 +149,9 @@ }, "embedding": { "serviceNotConfigured": "Enable and configure the embedding service first.", - "serviceUnavailable": "Embedding service is unavailable." + "serviceUnavailable": "Embedding service is unavailable.", + "submitFailed": "Failed to submit vectorization jobs. Please try again later.", + "tooManyFiles": "You can submit up to 100 files for vectorization at a time." }, "file": { "embeddingUnavailable": "Embedding is unavailable for this file.", diff --git a/frontend/i18n/messages/en-US/files.json b/frontend/i18n/messages/en-US/files.json index 576e26bd9..c99feb0f2 100644 --- a/frontend/i18n/messages/en-US/files.json +++ b/frontend/i18n/messages/en-US/files.json @@ -39,6 +39,8 @@ "cancelSelect": "Cancel selection", "selectFile": "Select file", "selectAll": "Select all", + "vectorize": "Vectorize", + "updateIndex": "Update index", "loadMore": "Load more" }, "selection": { @@ -146,6 +148,12 @@ "renameSucceeded": "File name updated", "renameFailed": "Failed to rename file", "settingFailed": "Failed to update setting", + "vectorizeSubmitted": "Submitted {count} files for vectorization", + "vectorizeSkipped": "{count} other files did not need processing or are not currently supported.", + "vectorizeNoChanges": "The selected files do not need vectorization or are not currently supported", + "vectorizeLimit": "You can submit up to 100 files at a time", + "vectorizeFailed": "Failed to submit vectorization jobs", + "vectorizePartial": "Submitted {submitted} files; {failed} files could not be queued", "previewLoadFailed": "Failed to load file preview", "previewProcessingFailed": "File processing failed. Preview is unavailable.", "previewProcessing": "File is still processing. Preview will be available later.", @@ -191,6 +199,8 @@ "errorCode": "Error code: {code}", "indexFailed": "Index failed", "indexFailedDetail": "The file is available for full-context chat, but smart retrieval indexing failed.", + "indexOutdated": "Index update required", + "indexOutdatedDetail": "This file uses an older vector space and must be vectorized again for the current retrieval configuration.", "waiting": "Waiting", "waitingDetail": "File uploaded and waiting for the processing queue.", "queued": "Queued", @@ -200,6 +210,8 @@ "ocrDetail": "OCR is recognizing and extracting text.", "extractingDetail": "Extracting text content from the file.", "embedding": "Vectorizing", + "embeddingQueuedDetail": "The vectorization job is queued and will start shortly.", + "embedQueued": "Index queued", "embeddingDetail": "Text extraction is complete. Semantic vector index is being generated.", "ready": "Ready", "readyRagDetail": "File is ready and supports full-context injection and smart retrieval (RAG).", diff --git a/frontend/i18n/messages/en-US/knowledge-bases.json b/frontend/i18n/messages/en-US/knowledge-bases.json index e9f9381f6..b3a856de1 100644 --- a/frontend/i18n/messages/en-US/knowledge-bases.json +++ b/frontend/i18n/messages/en-US/knowledge-bases.json @@ -58,6 +58,13 @@ "noFilesDescription": "Add existing files or upload new ones to build this knowledge base.", "loadMore": "Load more", "removeFile": "Remove from knowledge base", + "vectorize": "Vectorize", + "updateVectorIndex": "Update index", + "selectVectorizableFiles": "Select files awaiting vectorization", + "selectedVectorizeFiles": "{count} files selected for vectorization", + "selectFile": "Select file {name}", + "vectorizeFile": "Vectorize file {name}", + "updateVectorIndexFile": "Update the vector index for file {name}", "createTitle": "New knowledge base", "editTitle": "Edit knowledge base", "moreActions": "More actions", @@ -120,6 +127,11 @@ "platformFileDeleteFailed": "Failed to delete platform source", "removeFailed": "Failed to remove file", "removed": "File removed from knowledge base", + "vectorizeSubmitted": "Submitted {count} files for vectorization", + "vectorizeSkipped": "{count} other files did not need processing or are not currently supported.", + "vectorizeNoChanges": "The selected files do not need vectorization or are not currently supported", + "vectorizeFailed": "Failed to submit vectorization jobs", + "vectorizePartial": "Submitted {submitted} files; {failed} files could not be queued", "toggleFailed": "Failed to update knowledge base status", "enabledToast": "Built-in knowledge base enabled", "disabledToast": "Built-in knowledge base disabled" diff --git a/frontend/i18n/messages/zh-CN/errors.json b/frontend/i18n/messages/zh-CN/errors.json index bc2c067f0..9bdfa3ef6 100644 --- a/frontend/i18n/messages/zh-CN/errors.json +++ b/frontend/i18n/messages/zh-CN/errors.json @@ -149,7 +149,9 @@ }, "embedding": { "serviceNotConfigured": "请先启用并配置 Embedding 服务。", - "serviceUnavailable": "向量服务不可用。" + "serviceUnavailable": "向量服务不可用。", + "submitFailed": "向量化任务提交失败，请稍后重试。", + "tooManyFiles": "单次最多提交 100 个文件进行向量化。" }, "file": { "embeddingUnavailable": "当前文件暂不支持向量处理。", diff --git a/frontend/i18n/messages/zh-CN/files.json b/frontend/i18n/messages/zh-CN/files.json index 7b5861ddb..e89ad82cf 100644 --- a/frontend/i18n/messages/zh-CN/files.json +++ b/frontend/i18n/messages/zh-CN/files.json @@ -39,6 +39,8 @@ "cancelSelect": "取消选择", "selectFile": "选择文件", "selectAll": "全选", + "vectorize": "向量化", + "updateIndex": "更新索引", "loadMore": "下滑加载更多" }, "selection": { @@ -146,6 +148,12 @@ "renameSucceeded": "文件名已更新", "renameFailed": "文件重命名失败", "settingFailed": "设置失败", + "vectorizeSubmitted": "已提交 {count} 个文件进行向量化", + "vectorizeSkipped": "另有 {count} 个文件无需处理或暂不支持。", + "vectorizeNoChanges": "所选文件无需向量化或当前暂不支持", + "vectorizeLimit": "单次最多提交 100 个文件", + "vectorizeFailed": "向量化任务提交失败", + "vectorizePartial": "已提交 {submitted} 个文件，另有 {failed} 个文件暂时无法加入队列", "previewLoadFailed": "文件预览加载失败", "previewProcessingFailed": "文件处理失败，暂时无法预览。", "previewProcessing": "文件正在处理中，稍后可预览。", @@ -191,6 +199,8 @@ "errorCode": "错误代码：{code}", "indexFailed": "索引失败", "indexFailedDetail": "文件已可用于全文对话，但智能检索索引生成失败。", + "indexOutdated": "索引待更新", + "indexOutdatedDetail": "该文件使用旧的向量空间，需要重新向量化后才能用于当前检索配置。", "waiting": "等待处理", "waitingDetail": "文件已上传，等待进入处理队列。", "queued": "排队中", @@ -200,6 +210,8 @@ "ocrDetail": "正在使用 OCR 引擎识别并提取文本。", "extractingDetail": "正在提取文件文本内容。", "embedding": "向量化中", + "embeddingQueuedDetail": "向量化任务已进入队列，即将开始处理。", + "embedQueued": "等待向量化", "embeddingDetail": "文本提取完成，正在生成语义向量索引。", "ready": "已就绪", "readyRagDetail": "文件已就绪，支持全文注入与智能检索（RAG）。", diff --git a/frontend/i18n/messages/zh-CN/knowledge-bases.json b/frontend/i18n/messages/zh-CN/knowledge-bases.json index df8fe5698..21b8184d8 100644 --- a/frontend/i18n/messages/zh-CN/knowledge-bases.json +++ b/frontend/i18n/messages/zh-CN/knowledge-bases.json @@ -58,6 +58,13 @@ "noFilesDescription": "添加已有文件或上传新文件，开始构建知识库。", "loadMore": "加载更多", "removeFile": "移出知识库", + "vectorize": "向量化", + "updateVectorIndex": "更新索引", + "selectVectorizableFiles": "选择待向量化文件", + "selectedVectorizeFiles": "已选择 {count} 个待向量化文件", + "selectFile": "选择文件「{name}」", + "vectorizeFile": "向量化文件「{name}」", + "updateVectorIndexFile": "更新文件「{name}」的向量索引", "createTitle": "新建知识库", "editTitle": "编辑知识库", "moreActions": "更多操作", @@ -120,6 +127,11 @@ "platformFileDeleteFailed": "平台资料删除失败", "removeFailed": "移出文件失败", "removed": "文件已移出知识库", + "vectorizeSubmitted": "已提交 {count} 个文件进行向量化", + "vectorizeSkipped": "另有 {count} 个文件无需处理或暂不支持。", + "vectorizeNoChanges": "所选文件无需向量化或当前暂不支持", + "vectorizeFailed": "向量化任务提交失败", + "vectorizePartial": "已提交 {submitted} 个文件，另有 {failed} 个文件暂时无法加入队列", "toggleFailed": "知识库状态更新失败", "enabledToast": "内置知识库已启用", "disabledToast": "内置知识库已停用" diff --git a/frontend/shared/api/file.ts b/frontend/shared/api/file.ts index 16db0adf3..c8b906789 100644 --- a/frontend/shared/api/file.ts +++ b/frontend/shared/api/file.ts @@ -2,6 +2,7 @@ import { authedFetch, authedRequest } from "@/shared/api/authed-client"; import type { ChatFilePolicyDTO, DeleteFileResult, + FileEmbeddingSubmissionDTO, FileExtractDTO, FileListResult, FileObjectDTO, @@ -263,6 +264,23 @@ export async function getFileProcessingStatuses( return (await Promise.all(requests)).flat(); } +export async function submitFileEmbeddings( + accessToken: string, + fileIDs: string[], + signal?: AbortSignal, +): Promise { + return authedRequest( + "/api/v1/files/embeddings", + { + method: "POST", + accessToken, + body: { fileIDs }, + signal, + }, + true, + ); +} + export async function getChatFilePolicy(accessToken: string, signal?: AbortSignal): Promise { return authedRequest( "/api/v1/runtime/chat-file-policy", diff --git a/frontend/shared/api/file.types.ts b/frontend/shared/api/file.types.ts index 90580f823..d651541ff 100644 --- a/frontend/shared/api/file.types.ts +++ b/frontend/shared/api/file.types.ts @@ -1,5 +1,6 @@ import type { DeleteFileResponse, + FileEmbeddingSubmissionResponse, FileListResponse, FileObjectResponse, FileProcessingStatusResponse, @@ -10,6 +11,7 @@ import type { export type FileObjectDTO = FileObjectResponse; export type FileProcessingStatusDTO = FileProcessingStatusResponse; +export type FileEmbeddingSubmissionDTO = FileEmbeddingSubmissionResponse; export type FileExtractDTO = { fileID: string; diff --git a/frontend/shared/api/knowledge-bases.ts b/frontend/shared/api/knowledge-bases.ts index 32c1bae6f..08a777c0e 100644 --- a/frontend/shared/api/knowledge-bases.ts +++ b/frontend/shared/api/knowledge-bases.ts @@ -1,23 +1,24 @@ import { authedFetch, authedRequest } from "@/shared/api/authed-client"; import type { PagePayload } from "@/shared/api/common.types"; -import { readFileContentResponse, type FileContentResult } from "@/shared/api/file"; +import { type FileContentResult, readFileContentResponse } from "@/shared/api/file"; import { pathParam } from "@/shared/api/http-client"; import type { AddKnowledgeBaseFilesRequest, - KnowledgeBaseDTO, KnowledgeBaseData, KnowledgeBaseDeleteData, + KnowledgeBaseDTO, KnowledgeBaseFileData, KnowledgeBaseFileDTO, + KnowledgeBaseFileEmbeddingSubmissionDTO, KnowledgeBaseFileMutationData, KnowledgeBaseFilePage, KnowledgeBaseFileProcessingSnapshotDTO, KnowledgeBaseFileProcessingStatusDTO, KnowledgeBasePage, - PatchMyKnowledgeBaseRequest, PatchKnowledgeBaseRequest, - WriteMyKnowledgeBaseRequest, + PatchMyKnowledgeBaseRequest, WriteKnowledgeBaseRequest, + WriteMyKnowledgeBaseRequest, } from "@/shared/api/knowledge-bases.types"; type KnowledgeBaseListOptions = { @@ -265,6 +266,23 @@ export async function deleteAdminKnowledgeBaseFile(accessToken: string, fileID: ); } +export async function submitAdminPlatformFileEmbeddings( + accessToken: string, + fileIDs: string[], + signal?: AbortSignal, +): Promise { + return authedRequest( + "/api/v1/admin/knowledge-bases/files/embeddings", + { + method: "POST", + accessToken, + body: { fileIDs }, + signal, + }, + true, + ); +} + export async function updateAdminKnowledgeBase(accessToken: string, id: string, payload: PatchKnowledgeBaseRequest): Promise { return authedRequest(`/api/v1/admin/knowledge-bases/${pathParam(id)}`, { method: "PATCH", accessToken, body: payload }, true); } diff --git a/frontend/shared/api/knowledge-bases.types.ts b/frontend/shared/api/knowledge-bases.types.ts index 16bca3869..30f53c64a 100644 --- a/frontend/shared/api/knowledge-bases.types.ts +++ b/frontend/shared/api/knowledge-bases.types.ts @@ -1,18 +1,19 @@ import type { AddKnowledgeBaseFilesRequest as ContractAddKnowledgeBaseFilesRequest, + PatchKnowledgeBaseRequest as ContractPatchKnowledgeBaseRequest, + PatchMyKnowledgeBaseRequest as ContractPatchMyKnowledgeBaseRequest, + WriteKnowledgeBaseRequest as ContractWriteKnowledgeBaseRequest, + WriteMyKnowledgeBaseRequest as ContractWriteMyKnowledgeBaseRequest, KnowledgeBaseDataResponse, KnowledgeBaseDeleteDataResponse, KnowledgeBaseFileDataResponse, + KnowledgeBaseFileEmbeddingSubmissionResponse, KnowledgeBaseFileMutationDataResponse, KnowledgeBaseFilePageResponseDoc, KnowledgeBaseFileProcessingSnapshotResponse, KnowledgeBaseFileProcessingStatusResponse, KnowledgeBasePageResponseDoc, KnowledgeBaseResponse, - PatchMyKnowledgeBaseRequest as ContractPatchMyKnowledgeBaseRequest, - PatchKnowledgeBaseRequest as ContractPatchKnowledgeBaseRequest, - WriteMyKnowledgeBaseRequest as ContractWriteMyKnowledgeBaseRequest, - WriteKnowledgeBaseRequest as ContractWriteKnowledgeBaseRequest, } from "@deeix/api-contract"; export type KnowledgeBaseScope = "builtin" | "user"; @@ -27,6 +28,7 @@ export type KnowledgeBasePage = Omit & { knowledgeBase: KnowledgeBaseDTO; }; diff --git a/frontend/shared/lib/file-processing.ts b/frontend/shared/lib/file-processing.ts index 8a2d52151..0bdb117cf 100644 --- a/frontend/shared/lib/file-processing.ts +++ b/frontend/shared/lib/file-processing.ts @@ -12,6 +12,8 @@ type FileProcessingView = { ragOptOut?: boolean; chunkCount?: number; ocrUsed?: boolean; + canVectorize?: boolean; + vectorizationReason?: string; }; type FileProcessingBadge = { @@ -23,7 +25,20 @@ type FileProcessingBadge = { type FileProcessingTranslator = (key: string, values?: Record) => string; export function isFileProcessing(file: FileProcessingView): boolean { - return file.processing === true; + if (typeof file.processing === "boolean") { + return file.processing; + } + return ["queued", "processing"].includes(file.embedStatus ?? "") + || ["uploaded", "queued", "extracting", "embedding"].includes(file.processingStatus ?? ""); +} + +export function canManuallyVectorizeFile(file: FileProcessingView): boolean { + return file.canVectorize === true; +} + +export function isVectorIndexOutdated(file: FileProcessingView): boolean { + return file.embedStatus === "stale" + || (file.canVectorize === true && file.vectorizationReason === "outdated_index"); } function translateFileProcessing( @@ -65,6 +80,34 @@ export function resolveFileProcessingBadge( }; } + if (file.embedStatus === "processing") { + return { + label: translateFileProcessing(translate, "embedding", "Vectorizing"), + tone: "info", + detail: translateFileProcessing(translate, "embeddingDetail", "Text extraction is complete. Semantic vector index is being generated."), + }; + } + + if (file.embedStatus === "queued") { + return { + label: translateFileProcessing(translate, "queued", "Queued"), + tone: "info", + detail: translateFileProcessing(translate, "embeddingQueuedDetail", "The vectorization job is queued and will start shortly."), + }; + } + + if (isVectorIndexOutdated(file)) { + return { + label: translateFileProcessing(translate, "indexOutdated", "Index update required"), + tone: "warning", + detail: translateFileProcessing( + translate, + "indexOutdatedDetail", + "This file uses an older vector space and must be vectorized again for the current retrieval configuration.", + ), + }; + } + switch (file.processingStatus) { case "uploaded": return { @@ -144,6 +187,18 @@ export function resolveFileRetrievalBadge( }; } + if (isVectorIndexOutdated(file)) { + return { + label: translateFileProcessing(translate, "indexOutdated", "Index update required"), + tone: "warning", + detail: translateFileProcessing( + translate, + "indexOutdatedDetail", + "This file uses an older vector space and must be vectorized again for the current retrieval configuration.", + ), + }; + } + if (file.processingReady && file.embedStatus === "ready" && (file.chunkCount ?? 0) > 0) { return { label: translateFileProcessing(translate, "searchable", "Searchable"), @@ -151,7 +206,7 @@ export function resolveFileRetrievalBadge( }; } - if (file.embedStatus === "processing") { + if (["queued", "processing"].includes(file.embedStatus ?? "")) { return { label: translateFileProcessing(translate, "embedding", "Vectorizing"), tone: "info", @@ -179,6 +234,8 @@ export function resolveEmbedStatusLabel(embedStatus: string | null | undefined, return translateFileProcessing(translate, "embedReady", "Smart retrieval ready ✓"); case "processing": return translateFileProcessing(translate, "embedProcessing", "Indexing…"); + case "queued": + return translateFileProcessing(translate, "embedQueued", "Index queued"); case "failed": return translateFileProcessing(translate, "embedFailed", "Index failed"); case "none": diff --git a/packages/api-contract/src/types.generated.ts b/packages/api-contract/src/types.generated.ts index 9b59a6753..bf27dc284 100644 --- a/packages/api-contract/src/types.generated.ts +++ b/packages/api-contract/src/types.generated.ts @@ -1390,6 +1390,21 @@ export interface Envelope { requestId?: string; } +export interface FileEmbeddingSkipResponse { + fileID: string; + reason: string; +} + +export interface FileEmbeddingSubmissionResponse { + skipped: FileEmbeddingSkipResponse[]; + submittedFileIDs: string[]; +} + +export interface FileEmbeddingSubmissionResponseDoc { + data: FileEmbeddingSubmissionResponse; + errorMsg: string; +} + export interface FileListResponse { quota: StorageQuotaResponse; results: FileObjectResponse[]; @@ -1403,6 +1418,7 @@ export interface FileListResponseDoc { export interface FileObjectResponse { sha256: string; + canVectorize: boolean; chunkCount: number; createdAt: string; detectedMIME: string; @@ -1424,9 +1440,11 @@ export interface FileObjectResponse { sizeBytes: number; status: string; updatedAt: string; + vectorizationReason: string; } export interface FileProcessingStatusResponse { + canVectorize: boolean; chunkCount: number; completedAt: string | null; detectedMIME: string; @@ -1447,6 +1465,7 @@ export interface FileProcessingStatusResponse { ragReason: string; startedAt: string | null; updatedAt: string; + vectorizationReason: string; } export interface FileUpdateResponseDoc { @@ -1663,6 +1682,21 @@ export interface KnowledgeBaseFileDataResponse { file: KnowledgeBaseFileResponse; } +export interface KnowledgeBaseFileEmbeddingSkipResponse { + fileID: string; + reason: string; +} + +export interface KnowledgeBaseFileEmbeddingSubmissionResponse { + skipped: KnowledgeBaseFileEmbeddingSkipResponse[]; + submittedFileIDs: string[]; +} + +export interface KnowledgeBaseFileEmbeddingSubmissionResponseDoc { + data: KnowledgeBaseFileEmbeddingSubmissionResponse; + errorMsg: string; +} + export interface KnowledgeBaseFileMutationDataResponse { updated: boolean; } @@ -1686,9 +1720,12 @@ export interface KnowledgeBaseFileProcessingSnapshotResponse { } export interface KnowledgeBaseFileProcessingStatusResponse { + canVectorize: boolean; chunkCount: number; detectedMIME: string; + embedError: string; embedStatus: string; + extractStatus: string; fileCategory: string; fileID: string; processing: boolean; @@ -1696,13 +1733,17 @@ export interface KnowledgeBaseFileProcessingStatusResponse { processingStatus: string; ragOptOut: boolean; updatedAt: string; + vectorizationReason: string; } export interface KnowledgeBaseFileResponse { + canVectorize: boolean; chunkCount: number; createdAt: string; detectedMIME: string; + embedError: string; embedStatus: string; + extractStatus: string; fileCategory: string; fileID: string; fileName: string; @@ -1713,6 +1754,7 @@ export interface KnowledgeBaseFileResponse { ragOptOut: boolean; sizeBytes: number; updatedAt: string; + vectorizationReason: string; } export interface KnowledgeBaseFileResponseDoc { @@ -3303,6 +3345,22 @@ export interface StorageQuotaResponse { userID: number; } +export interface SubmitFileEmbeddingsRequest { + /** + * @maxItems 100 + * @minItems 1 + */ + fileIDs: string[]; +} + +export interface SubmitPlatformFileEmbeddingsRequest { + /** + * @maxItems 100 + * @minItems 1 + */ + fileIDs: string[]; +} + export interface SubscribeRequest { /** * @min 1 @@ -5124,6 +5182,22 @@ export namespace Admin { export type ResponseBody = KnowledgeBaseFileResponseDoc; } + /** + * @description 为管理员选中的平台资料提交向量化任务，最多100个；重复提交会幂等跳过 + * @tags admin-knowledge-bases + * @name KnowledgeBasesFilesEmbeddingsCreate + * @summary 批量提交平台资料向量化 + * @request POST:/admin/knowledge-bases/files/embeddings + * @secure + */ + export namespace KnowledgeBasesFilesEmbeddingsCreate { + export type RequestParams = {}; + export type RequestQuery = {}; + export type RequestBody = SubmitPlatformFileEmbeddingsRequest; + export type RequestHeaders = {}; + export type ResponseBody = KnowledgeBaseFileEmbeddingSubmissionResponseDoc; + } + /** * @description 仅允许删除未被任何知识库、会话或账户资料引用的平台资料 * @tags admin-knowledge-bases @@ -8699,6 +8773,22 @@ export namespace Files { export type ResponseBody = UploadFileResponseDoc; } + /** + * @description 为当前用户已完成文本提取的文件提交向量化任务，最多100个；重复提交会幂等跳过 + * @tags chat + * @name EmbeddingsCreate + * @summary 批量提交指定文件向量化 + * @request POST:/files/embeddings + * @secure + */ + export namespace EmbeddingsCreate { + export type RequestParams = {}; + export type RequestQuery = {}; + export type RequestBody = SubmitFileEmbeddingsRequest; + export type RequestHeaders = {}; + export type ResponseBody = FileEmbeddingSubmissionResponseDoc; + } + /** * @description 一次查询当前用户多个文件的处理状态 * @tags chat