diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 1e68b4ab8..077d8debc 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -14225,9 +14225,10 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "由浏览器提交完整纯文本上下文；服务端不创建会话、消息、运行或断线续传记录", + "description": "由浏览器提交完整上下文和可选请求级附件；服务端不创建会话、消息、运行、文件或断线续传记录", "consumes": [ - "application/json" + "application/json", + "multipart/form-data" ], "produces": [ "application/x-ndjson" diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 01c6044a5..f16caaa8f 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -14218,9 +14218,10 @@ "BearerAuth": [] } ], - "description": "由浏览器提交完整纯文本上下文；服务端不创建会话、消息、运行或断线续传记录", + "description": "由浏览器提交完整上下文和可选请求级附件；服务端不创建会话、消息、运行、文件或断线续传记录", "consumes": [ - "application/json" + "application/json", + "multipart/form-data" ], "produces": [ "application/x-ndjson" diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index a9afc1bdf..c41855dfb 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -18729,7 +18729,8 @@ paths: post: consumes: - application/json - description: 由浏览器提交完整纯文本上下文；服务端不创建会话、消息、运行或断线续传记录 + - multipart/form-data + description: 由浏览器提交完整上下文和可选请求级附件；服务端不创建会话、消息、运行、文件或断线续传记录 parameters: - description: 临时对话参数 in: body diff --git a/backend/internal/application/contentmoderation/coordinator.go b/backend/internal/application/contentmoderation/coordinator.go index 823d2f198..26b5f40d3 100644 --- a/backend/internal/application/contentmoderation/coordinator.go +++ b/backend/internal/application/contentmoderation/coordinator.go @@ -154,12 +154,57 @@ func (c *RunCoordinator) EnqueueInputImages(ctx context.Context, fileIDs []strin if len(raw) == 0 { return } + c.enqueueInputImageSources(raw, keptFiles, selected) +} + +// EnqueueInputImageSources queues request-scoped image bytes without requiring +// persisted file records. Ephemeral chat uses this path so its images follow the +// same moderation policy while remaining outside the user file library. +func (c *RunCoordinator) EnqueueInputImageSources(images []OutputImageSource) { + if c == nil { + return + } + selected := c.cfg.Policy.CategoriesFor(domaincm.DirectionInput, domaincm.ModalityImage) + if len(selected) == 0 || len(images) == 0 { + return + } + seenSHA := make(map[string]struct{}) + raw := make([]OutputImageSource, 0, len(images)) + fileIDs := make([]string, 0, len(images)) + for _, image := range images { + if len(image.Data) == 0 { + c.recordSurfaceFailure(domaincm.DirectionInput, domaincm.ModalityImage, image.FileID, ErrModerationInvalidResp) + continue + } + sha := strings.TrimSpace(image.SHA256) + if sha != "" { + if _, exists := seenSHA[sha]; exists { + continue + } + seenSHA[sha] = struct{}{} + } + fileID := strings.TrimSpace(image.FileID) + raw = append(raw, OutputImageSource{ + FileID: fileID, + Data: append([]byte(nil), image.Data...), + MimeType: strings.TrimSpace(image.MimeType), + SHA256: sha, + }) + fileIDs = append(fileIDs, fileID) + } + if len(raw) == 0 { + return + } + c.enqueueInputImageSources(raw, fileIDs, selected) +} + +func (c *RunCoordinator) enqueueInputImageSources(raw []OutputImageSource, fileIDs []string, selected []string) { c.startTask(&moderationTask{ Coord: c, Direction: domaincm.DirectionInput, Modality: domaincm.ModalityImage, RawImages: raw, - FileIDs: keptFiles, + FileIDs: fileIDs, Selected: selected, Location: domaincm.ContentLocation{Field: "user_attachments"}, // Input hits must isolate but not revoke user library files. diff --git a/backend/internal/application/contentmoderation/coordinator_test.go b/backend/internal/application/contentmoderation/coordinator_test.go index fed556fdf..ef9e7cd88 100644 --- a/backend/internal/application/contentmoderation/coordinator_test.go +++ b/backend/internal/application/contentmoderation/coordinator_test.go @@ -343,6 +343,29 @@ func TestInputImageModerationSkipsNonImageAttachments(t *testing.T) { } } +func TestEphemeralInputImageModerationQueuesRequestScopedBytes(t *testing.T) { + service := NewService(nil, &coordinatorTestRepo{}, "", nil) + cfg := runtimeConfig{Policy: Policy{InputImageCategories: []string{"violence"}}} + coord := newRunCoordinator(service, RunMeta{RunID: "run_ephemeral_image", Ephemeral: true}, cfg) + + coord.EnqueueInputImageSources([]OutputImageSource{ + {FileID: "temporary_image", Data: []byte("image-bytes"), MimeType: "image/png", SHA256: "sha"}, + {FileID: "temporary_duplicate", Data: []byte("duplicate"), MimeType: "image/png", SHA256: "sha"}, + }) + + select { + case task := <-service.taskQueue: + if task == nil || task.Modality != domaincm.ModalityImage || !task.IsolateOnly { + t.Fatalf("unexpected request-scoped moderation task: %#v", task) + } + if len(task.RawImages) != 1 || string(task.RawImages[0].Data) != "image-bytes" { + t.Fatalf("request-scoped image bytes were not queued safely: %#v", task.RawImages) + } + default: + t.Fatal("request-scoped image moderation task was not queued") + } +} + func TestRecordHitRollsBackIsolatedImagesWhenEventCreateFails(t *testing.T) { repo := &coordinatorTestRepo{createErr: errors.New("database unavailable")} store := &coordinatorTestObjectStore{} diff --git a/backend/internal/application/contentmoderation/service.go b/backend/internal/application/contentmoderation/service.go index e46444986..df868e060 100644 --- a/backend/internal/application/contentmoderation/service.go +++ b/backend/internal/application/contentmoderation/service.go @@ -44,7 +44,7 @@ type PreparedImage struct { // ImageLoader loads and prepares an image for moderation. type ImageLoader func(ctx context.Context, userID uint, fileID string) (PreparedImage, error) -// OutputImageSource provides final generated image bytes for isolation. +// OutputImageSource provides request-scoped image bytes and optional source metadata for moderation. type OutputImageSource struct { FileID string Data []byte diff --git a/backend/internal/application/conversation/service_temporary_chat.go b/backend/internal/application/conversation/service_temporary_chat.go index 51c488271..684eb7f17 100644 --- a/backend/internal/application/conversation/service_temporary_chat.go +++ b/backend/internal/application/conversation/service_temporary_chat.go @@ -14,7 +14,7 @@ import ( const temporaryChatMaxContentChars = 1_000_000 -// TemporaryChatMessage 是浏览器内临时对话的一条纯文本消息。 +// TemporaryChatMessage 是浏览器内临时对话的一条消息。 type TemporaryChatMessage struct { Role string Content string @@ -22,18 +22,20 @@ type TemporaryChatMessage struct { // TemporaryChatInput 描述不创建会话、消息和运行记录的临时推理请求。 type TemporaryChatInput struct { - UserID uint - RequestID string - SessionID string - ClientRunID string - Model string - Options map[string]interface{} - SelectedToolIDs []uint - SkillIDs []uint - KnowledgeBaseIDs []string - HTMLVisualPromptEnabled bool - Messages []TemporaryChatMessage - OnEvent func(eventType string, payload map[string]interface{}) error + UserID uint + RequestID string + SessionID string + ClientRunID string + Model string + Options map[string]interface{} + SelectedToolIDs []uint + SkillIDs []uint + KnowledgeBaseIDs []string + HTMLVisualPromptEnabled bool + Messages []TemporaryChatMessage + Attachments []TemporaryChatAttachment + ReleaseAttachmentSources func() + OnEvent func(eventType string, payload map[string]interface{}) error } // StreamTemporaryChat 直接以请求上下文调用上游。调用方断开连接时生成随即取消， @@ -82,6 +84,14 @@ func (s *Service) StreamTemporaryChat( for _, item := range input.Messages { messages = append(messages, llm.Message{Role: item.Role, Content: item.Content}) } + attachmentContext, err := s.prepareTemporaryAttachmentContext(ctx, input, messages) + if input.ReleaseAttachmentSources != nil { + input.ReleaseAttachmentSources() + } + if err != nil { + return nil, err + } + messages = attachmentContext.messages systemPrompt := resolveMessageSystemPromptInjection(cfg, route, "", input.HTMLVisualPromptEnabled) if systemPrompt.Content != "" { if systemPrompt.InlineToUser { @@ -97,8 +107,8 @@ func (s *Service) StreamTemporaryChat( if err != nil { return nil, err } - // Attachment processors require persisted file objects and are therefore not - // exposed in a text-only temporary request. + // Attachment processors depend on persisted file IDs. Request-scoped temporary + // attachments are injected directly into the model context instead. toolRuntime = toolRuntime.withoutAttachmentProcessor() skillPrompts, err := s.resolveSkillPrompts(ctx, SendMessageInput{ UserID: input.UserID, @@ -115,11 +125,12 @@ func (s *Service) StreamTemporaryChat( return nil, err } promptPlan := buildPromptPlan(ctx, promptPlanInput{ - BaseMessages: messages, - DynamicContext: knowledgeContext, - SkillPrompts: skillPrompts, - ToolRuntime: toolRuntime, - Config: cfg, + BaseMessages: messages, + StableAttachments: attachmentContext.stableAttachments, + DynamicContext: knowledgeContext, + SkillPrompts: skillPrompts, + ToolRuntime: toolRuntime, + Config: cfg, }) messages = stripTemporaryMessageCacheControls(promptPlan.Messages) fullMessages := cloneLLMMessages(messages) @@ -145,15 +156,15 @@ func (s *Service) StreamTemporaryChat( }) } moderationCoord.EnqueueInputText(lastUser.Content) + moderationCoord.EnqueueInputImageSources(attachmentContext.moderationImages) } } generateInput := llm.GenerateInput{ - RequestID: strings.TrimSpace(input.RequestID), - Messages: messages, - Tools: toolRuntime.definitions, - DisableTools: len(toolRuntime.definitions) == 0, - Options: filteredOptions, - Ephemeral: true, + RequestID: strings.TrimSpace(input.RequestID), + Messages: messages, + Tools: toolRuntime.definitions, + Options: filteredOptions, + Ephemeral: true, } var budgetFit promptBudgetFit generateInput, budgetFit = fitGenerateInputToModelBudget( @@ -345,6 +356,9 @@ func ValidateTemporaryChatInput(input TemporaryChatInput) error { if len(input.Messages) == 0 || len(input.Messages) > 100 { return ErrInvalidMessageContent } + if len(input.Attachments) > TemporaryChatMaxAttachments { + return ErrTooManyMessageFiles + } if len(input.KnowledgeBaseIDs) > 8 { return ErrInvalidMessageContent } @@ -359,12 +373,22 @@ func ValidateTemporaryChatInput(input TemporaryChatInput) error { } seenKnowledgeBases[normalized] = struct{}{} } + attachmentCounts := make(map[int]int) + for _, attachment := range input.Attachments { + if attachment.MessageIndex < 0 || attachment.MessageIndex >= len(input.Messages) || attachment.Reader == nil { + return ErrInvalidFileReference + } + attachmentCounts[attachment.MessageIndex]++ + } totalChars := 0 previousRole := "" - for _, item := range input.Messages { + for index, item := range input.Messages { role := strings.TrimSpace(item.Role) content := strings.TrimSpace(item.Content) - if (role != "user" && role != "assistant") || content == "" || role == previousRole { + if (role != "user" && role != "assistant") || role == previousRole { + return ErrInvalidMessageContent + } + if content == "" && (role != "user" || attachmentCounts[index] == 0) { return ErrInvalidMessageContent } totalChars += len([]rune(item.Content)) diff --git a/backend/internal/application/conversation/service_temporary_chat_attachment.go b/backend/internal/application/conversation/service_temporary_chat_attachment.go new file mode 100644 index 000000000..d916276db --- /dev/null +++ b/backend/internal/application/conversation/service_temporary_chat_attachment.go @@ -0,0 +1,165 @@ +package conversation + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/extraction" + appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/llm" +) + +// TemporaryChatMaxAttachments 限制单次临时请求携带的历史附件总数。 +const TemporaryChatMaxAttachments = 20 + +// TemporaryChatAttachment 是随临时会话请求传入、且不进入持久化文件链路的附件。 +type TemporaryChatAttachment struct { + MessageIndex int + FileName string + MimeType string + DeclaredSize int64 + Reader io.Reader +} + +type temporaryAttachmentContext struct { + messages []llm.Message + stableAttachments []AttachmentInput + moderationImages []appcm.OutputImageSource +} + +func (s *Service) prepareTemporaryAttachmentContext( + ctx context.Context, + input TemporaryChatInput, + messages []llm.Message, +) (temporaryAttachmentContext, error) { + result := temporaryAttachmentContext{ + messages: append([]llm.Message(nil), messages...), + } + if len(input.Attachments) == 0 { + return result, nil + } + if s.uploadSvc == nil { + return result, ErrInvalidFileReference + } + + cfg := s.cfg.Snapshot() + imageCount := 0 + imageBytes := 0 + for _, item := range input.Attachments { + file, err := s.uploadSvc.PrepareTemporaryFile(ctx, appupload.TemporaryFileInput{ + FileName: item.FileName, + MimeType: item.MimeType, + DeclaredSize: item.DeclaredSize, + Reader: item.Reader, + }) + if err != nil { + return result, err + } + processErr := func() error { + defer file.Cleanup() + switch file.FileCategory { + case "image": + imageCount++ + if imageCount > maxConversationImageContextCount { + return ErrTooManyMessageFiles + } + data, err := os.ReadFile(file.AbsolutePath) + if err != nil { + return fmt.Errorf("%w: read temporary image", ErrFileNotFound) + } + if len(data) > maxConversationImageSourceBytes { + return ErrFileTooLarge + } + resized, mimeType := resizeImageIfNeeded(data, resolveImageMimeType(file.DetectedMIME), cfg.ImageMaxDimension) + imageBytes += len(resized) + if imageBytes > maxConversationImageContextBytes { + return ErrFileTooLarge + } + message := result.messages[item.MessageIndex] + parts := append([]llm.ContentPart(nil), message.Parts...) + if len(parts) == 0 && strings.TrimSpace(message.Content) != "" { + parts = append(parts, llm.ContentPart{Kind: llm.ContentPartText, Text: message.Content}) + } + parts = append(parts, llm.ContentPart{ + Kind: llm.ContentPartImage, + MimeType: mimeType, + Data: resized, + FileName: file.FileName, + }) + message.Content = "" + message.Parts = parts + result.messages[item.MessageIndex] = message + result.moderationImages = append(result.moderationImages, appcm.OutputImageSource{ + FileID: temporaryAttachmentContextID(file.SHA256), + Data: resized, + MimeType: mimeType, + SHA256: file.SHA256, + }) + case "pdf", "word", "presentation", "excel", "text": + if s.extractSvc == nil { + return ErrInvalidFileReference + } + extracted, err := s.extractSvc.ExtractTemporaryFile(ctx, extraction.ExtractInput{ + File: domainconversation.FileObject{ + FileID: file.FileID, + FileName: file.FileName, + MimeType: file.MimeType, + DetectedMIME: file.DetectedMIME, + FileCategory: file.FileCategory, + SizeBytes: file.SizeBytes, + SHA256: file.SHA256, + StoragePath: file.AbsolutePath, + }, + PDFMaxPages: cfg.FileFullContextPDFMaxPages, + OCREngine: cfg.ExtractOCREngine, + ImageOCREnabled: false, + PDFOCRFallbackEnabled: cfg.ExtractPDFOCRFallbackEnabled, + }) + if err != nil || strings.TrimSpace(extracted.Text) == "" { + return fmt.Errorf("%w: temporary attachment extraction failed", ErrFileProcessingNotReady) + } + attachment := AttachmentInput{ + FileID: temporaryAttachmentContextID(file.SHA256), + Kind: "file", + FileName: file.FileName, + MimeType: file.MimeType, + DetectedMIME: file.DetectedMIME, + FileCategory: file.FileCategory, + FileSize: file.SizeBytes, + SHA256: file.SHA256, + PageCount: extracted.PageCount, + ExtractedText: extracted.Text, + Current: item.MessageIndex == len(input.Messages)-1, + MessageRole: "user", + ContextMode: fileContextModeFull, + ProcessingReady: true, + ExtractStatus: "success", + } + if !canUseAttachmentFullContext(attachment, cfg) { + return ErrFileTooLargeForFullContext + } + result.stableAttachments = append(result.stableAttachments, attachment) + default: + return ErrInvalidFileReference + } + return nil + }() + if processErr != nil { + return result, processErr + } + } + return result, nil +} + +func temporaryAttachmentContextID(sha string) string { + normalized := strings.ToLower(strings.TrimSpace(sha)) + if len(normalized) > 32 { + normalized = normalized[:32] + } + return "temporary_" + normalized +} diff --git a/backend/internal/application/conversation/service_temporary_chat_test.go b/backend/internal/application/conversation/service_temporary_chat_test.go index 63b5a83b1..4a0e259f8 100644 --- a/backend/internal/application/conversation/service_temporary_chat_test.go +++ b/backend/internal/application/conversation/service_temporary_chat_test.go @@ -1,15 +1,78 @@ package conversation import ( + "bytes" "context" + "image" + "image/color" + "image/png" + "strings" "testing" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/extraction" + appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + extractport "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/extract" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/ports/llm" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" ) +type temporaryLLMGatewayStub struct { + inputs []llm.GenerateInput + onGenerateStream func(llm.GenerateInput) +} + +type temporaryBuiltinParserStub struct{} + +func (temporaryBuiltinParserStub) ExtractText(data []byte) string { return string(data) } + +func (temporaryBuiltinParserStub) ExtractWordText( + context.Context, + string, + []byte, + string, + string, +) extractport.WordTextResult { + return extractport.WordTextResult{} +} + +func (temporaryBuiltinParserStub) ExtractExcelText([]byte, string, string) string { return "" } + +func (temporaryBuiltinParserStub) ExtractPDFText(string, int) (string, error) { return "", nil } + +func (temporaryBuiltinParserStub) ExtractPDFPages(string, int) (extractport.PDFTextResult, error) { + return extractport.PDFTextResult{}, nil +} + +func (temporaryBuiltinParserStub) DetectPDFPageCount(string) int { return 0 } + +func (s *temporaryLLMGatewayStub) Generate(context.Context, llm.RouteConfig, llm.GenerateInput) (*llm.GenerateOutput, error) { + return nil, nil +} + +func (s *temporaryLLMGatewayStub) GenerateStream( + _ context.Context, + _ llm.RouteConfig, + input llm.GenerateInput, + _ func(llm.GenerateStreamEvent) error, +) (*llm.GenerateOutput, error) { + if s.onGenerateStream != nil { + s.onGenerateStream(input) + } + s.inputs = append(s.inputs, input) + return &llm.GenerateOutput{Text: "ok"}, nil +} + +func (s *temporaryLLMGatewayStub) RetrieveOpenAIResponse(context.Context, llm.RouteConfig, string) (*llm.GenerateOutput, error) { + return nil, nil +} + +func (s *temporaryLLMGatewayStub) CancelOpenAIResponse(context.Context, llm.RouteConfig, string) (*llm.GenerateOutput, error) { + return nil, nil +} + type temporaryPersistenceRepositoryStub struct { repository.ConversationRepository traceWrites int @@ -47,6 +110,17 @@ func TestValidateTemporaryChatInput(t *testing.T) { if err := ValidateTemporaryChatInput(valid); err != nil { t.Fatalf("valid input rejected: %v", err) } + attachmentOnly := valid + attachmentOnly.Messages = []TemporaryChatMessage{{Role: "user"}} + attachmentOnly.Attachments = []TemporaryChatAttachment{{ + MessageIndex: 0, + FileName: "notes.txt", + MimeType: "text/plain", + Reader: strings.NewReader("notes"), + }} + if err := ValidateTemporaryChatInput(attachmentOnly); err != nil { + t.Fatalf("attachment-only input rejected: %v", err) + } tests := map[string]TemporaryChatInput{ "assistant last": { @@ -76,6 +150,134 @@ func TestValidateTemporaryChatInput(t *testing.T) { } } +func TestStreamTemporaryChatSendsRequestScopedImageWithoutPersistence(t *testing.T) { + gateway := &temporaryLLMGatewayStub{} + runtimeCfg := config.NewRuntime(config.Config{ + MaxUploadFileBytes: 1024 * 1024, + MaxMessageFiles: 10, + FileAllowedMIMETypes: "image/png", + ImageMaxDimension: 1024, + ModelOptionPolicyMode: modelOptionPolicyAllowlist, + ModelOptionAllowedPaths: config.DefaultModelOptionAllowedPathsJSON(), + ModelOptionDeniedPaths: config.DefaultModelOptionDeniedPathsJSON(), + FileFullContextMaxBytes: config.DefaultFileFullContextMaxBytes, + FileFullContextMaxTokens: 65536, + }) + service := &Service{ + cfg: runtimeCfg, + routeResolver: &textTaskRouteResolverStub{routes: map[string]*channel.ResolvedRoute{ + "vision": { + PlatformModelName: "vision", + UpstreamModel: "vision", + Protocol: llm.AdapterOpenAIChatCompletions, + }, + }}, + llmClient: gateway, + uploadSvc: appupload.NewServiceWithRuntime(runtimeCfg, nil, nil, appupload.Hooks{}, appupload.ErrorSet{}, ""), + extractSvc: extraction.NewServiceWithRuntime(runtimeCfg), + } + var imageData bytes.Buffer + sourceImage := image.NewNRGBA(image.Rect(0, 0, 1, 1)) + sourceImage.Set(0, 0, color.NRGBA{R: 1, G: 2, B: 3, A: 255}) + if err := png.Encode(&imageData, sourceImage); err != nil { + t.Fatalf("encode image: %v", err) + } + + if _, err := service.StreamTemporaryChat(t.Context(), TemporaryChatInput{ + UserID: 1, + SessionID: "temporary-session", + ClientRunID: "temporary-run", + Model: "vision", + Messages: []TemporaryChatMessage{{Role: "user", Content: "describe this image"}}, + Attachments: []TemporaryChatAttachment{{ + MessageIndex: 0, + FileName: "image.png", + MimeType: "image/png", + DeclaredSize: int64(imageData.Len()), + Reader: bytes.NewReader(imageData.Bytes()), + }}, + }, nil); err != nil { + t.Fatalf("stream temporary chat: %v", err) + } + if len(gateway.inputs) != 1 || len(gateway.inputs[0].Messages) != 1 { + t.Fatalf("unexpected upstream input: %#v", gateway.inputs) + } + message := gateway.inputs[0].Messages[0] + if len(message.Parts) != 2 || message.Parts[0].Kind != llm.ContentPartText || message.Parts[1].Kind != llm.ContentPartImage { + t.Fatalf("temporary image was not attached to its user message: %#v", message) + } +} + +func TestStreamTemporaryChatExtractsDocumentAndReleasesUploadSourceBeforeGeneration(t *testing.T) { + extraction.RegisterEngineFactories(extraction.EngineFactories{Builtin: temporaryBuiltinParserStub{}}) + t.Cleanup(func() { extraction.RegisterEngineFactories(extraction.EngineFactories{}) }) + released := false + gateway := &temporaryLLMGatewayStub{ + onGenerateStream: func(input llm.GenerateInput) { + if !released { + t.Error("temporary upload source remained open during upstream generation") + } + }, + } + runtimeCfg := config.NewRuntime(config.Config{ + MaxUploadFileBytes: 1024 * 1024, + MaxMessageFiles: 10, + FileAllowedMIMETypes: "text/plain", + ModelOptionPolicyMode: modelOptionPolicyAllowlist, + ModelOptionAllowedPaths: config.DefaultModelOptionAllowedPathsJSON(), + ModelOptionDeniedPaths: config.DefaultModelOptionDeniedPathsJSON(), + FileFullContextMaxBytes: config.DefaultFileFullContextMaxBytes, + FileFullContextMaxTokens: 65536, + }) + service := &Service{ + cfg: runtimeCfg, + routeResolver: &textTaskRouteResolverStub{routes: map[string]*channel.ResolvedRoute{ + "text": { + PlatformModelName: "text", + UpstreamModel: "text", + Protocol: llm.AdapterOpenAIChatCompletions, + }, + }}, + llmClient: gateway, + uploadSvc: appupload.NewServiceWithRuntime(runtimeCfg, nil, nil, appupload.Hooks{}, appupload.ErrorSet{}, ""), + extractSvc: extraction.NewServiceWithRuntime(runtimeCfg), + } + + if _, err := service.StreamTemporaryChat(t.Context(), TemporaryChatInput{ + UserID: 1, + SessionID: "temporary-session", + ClientRunID: "temporary-run", + Model: "text", + Messages: []TemporaryChatMessage{{Role: "user", Content: "summarize the attachment"}}, + Attachments: []TemporaryChatAttachment{{ + MessageIndex: 0, + FileName: "notes.txt", + MimeType: "text/plain", + DeclaredSize: int64(len("request-scoped document content")), + Reader: strings.NewReader("request-scoped document content"), + }}, + ReleaseAttachmentSources: func() { released = true }, + }, nil); err != nil { + t.Fatalf("stream temporary chat: %v", err) + } + if !released { + t.Fatal("temporary upload source was not released") + } + if len(gateway.inputs) != 1 { + t.Fatalf("expected one upstream call, got %d", len(gateway.inputs)) + } + var contextText strings.Builder + for _, message := range gateway.inputs[0].Messages { + contextText.WriteString(message.Content) + for _, part := range message.Parts { + contextText.WriteString(part.Text) + } + } + if !strings.Contains(contextText.String(), "request-scoped document content") { + t.Fatalf("temporary document was not injected into model context: %q", contextText.String()) + } +} + func TestStripTemporaryChatProviderStateOptions(t *testing.T) { input := map[string]interface{}{ "temperature": 0.5, @@ -114,6 +316,92 @@ func TestEnforceTemporaryGenerateInput(t *testing.T) { } } +func TestStreamTemporaryChatPreservesProviderNativeToolsWithoutMCPTools(t *testing.T) { + tests := []struct { + name string + capabilitiesJSON string + options map[string]interface{} + expectedType string + }{ + { + name: "model default", + capabilitiesJSON: `{ + "defaultOptions": { + "tools": [{"type": "web_search", "enable_image_understanding": true}] + } + }`, + expectedType: "web_search", + }, + { + name: "user option", + capabilitiesJSON: `{"nativeToolKeys":["xai.x_search"]}`, + options: map[string]interface{}{ + "tools": []interface{}{ + map[string]interface{}{ + "type": "x_search", + "enable_image_understanding": true, + "allowed_domains": []interface{}{"x.com"}, + }, + }, + }, + expectedType: "x_search", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gateway := &temporaryLLMGatewayStub{} + service := &Service{ + cfg: config.NewRuntime(config.Config{ + ModelOptionPolicyMode: modelOptionPolicyAllowlist, + ModelOptionAllowedPaths: config.DefaultModelOptionAllowedPathsJSON(), + ModelOptionDeniedPaths: config.DefaultModelOptionDeniedPathsJSON(), + }), + routeResolver: &textTaskRouteResolverStub{routes: map[string]*channel.ResolvedRoute{ + "grok": { + PlatformModelName: "grok", + UpstreamModel: "grok-4.20-multi-agent-0309", + Protocol: llm.AdapterXAIResponses, + ModelCapabilitiesJSON: test.capabilitiesJSON, + }, + }}, + llmClient: gateway, + } + + if _, err := service.StreamTemporaryChat(t.Context(), TemporaryChatInput{ + UserID: 1, + SessionID: "temporary-session", + ClientRunID: "temporary-run", + Model: "grok", + Options: test.options, + Messages: []TemporaryChatMessage{ + {Role: "user", Content: "search the web"}, + }, + }, nil); err != nil { + t.Fatalf("stream temporary chat: %v", err) + } + + if len(gateway.inputs) != 1 { + t.Fatalf("expected one upstream call, got %d", len(gateway.inputs)) + } + generateInput := gateway.inputs[0] + if generateInput.DisableTools { + t.Fatal("temporary chat disabled provider-native tools when no MCP tool was selected") + } + if len(generateInput.Tools) != 0 { + t.Fatalf("expected no MCP tool declarations, got %#v", generateInput.Tools) + } + tools, ok := generateInput.Options["tools"].([]map[string]interface{}) + if !ok || len(tools) != 1 || tools[0]["type"] != test.expectedType { + t.Fatalf("expected %s provider-native tool, got %#v", test.expectedType, generateInput.Options["tools"]) + } + if tools[0]["enable_image_understanding"] != true { + t.Fatalf("expected provider-native tool parameters to remain, got %#v", tools[0]) + } + }) + } +} + func TestStripTemporaryMessageCacheControls(t *testing.T) { cacheControl := &llm.CacheControl{Type: "ephemeral", TTL: "1h"} input := []llm.Message{{ diff --git a/backend/internal/application/extraction/service.go b/backend/internal/application/extraction/service.go index d625d7f00..27dde3970 100644 --- a/backend/internal/application/extraction/service.go +++ b/backend/internal/application/extraction/service.go @@ -148,6 +148,29 @@ func (s *Service) ExtractStoredFile(ctx context.Context, input ExtractInput) (Re return Result{}, err } defer cleanup() + return s.extractLocalFile(ctx, input, absPath) +} + +// ExtractTemporaryFile 从系统临时目录内、由调用方管理生命周期的普通文件中提取文本。 +// 该入口不读取或写入对象存储，也不接受任意本地文件路径。 +func (s *Service) ExtractTemporaryFile(ctx context.Context, input ExtractInput) (Result, error) { + absPath := filepath.Clean(strings.TrimSpace(input.File.StoragePath)) + if absPath == "" || !filepath.IsAbs(absPath) { + return Result{}, ErrInvalidStoredFilePath + } + temporaryRoot := filepath.Clean(os.TempDir()) + relativePath, err := filepath.Rel(temporaryRoot, absPath) + if err != nil || relativePath == "." || relativePath == ".." || strings.HasPrefix(relativePath, ".."+string(os.PathSeparator)) { + return Result{}, ErrInvalidStoredFilePath + } + info, err := os.Stat(absPath) + if err != nil || !info.Mode().IsRegular() { + return Result{}, ErrInvalidStoredFilePath + } + return s.extractLocalFile(ctx, input, absPath) +} + +func (s *Service) extractLocalFile(ctx context.Context, input ExtractInput, absPath string) (Result, error) { file := input.File file.StoragePath = absPath input.File = file diff --git a/backend/internal/application/upload/service.go b/backend/internal/application/upload/service.go index 425ba6378..8ec533cf5 100644 --- a/backend/internal/application/upload/service.go +++ b/backend/internal/application/upload/service.go @@ -1,7 +1,6 @@ package upload import ( - "bufio" "context" "crypto/sha256" "encoding/hex" @@ -989,28 +988,11 @@ func saveUploadedFile( maxUploadBytes = 20 * 1024 * 1024 } - tmpFile, err := os.CreateTemp("", fileID+"_*.upload") + staged, err := stageUploadedFile(reader, fileID, fileName, maxUploadBytes, declaredMIME) if err != nil { return "", "", "", 0, err } - tmpName := tmpFile.Name() - defer func() { - _ = tmpFile.Close() - _ = os.Remove(tmpName) - }() - - bufferedReader := bufio.NewReader(reader) - header, _ := bufferedReader.Peek(512) - detectedMIME := detectContentMIME(header, declaredMIME, fileName) - - hasher := sha256.New() - written, err := io.Copy(io.MultiWriter(tmpFile, hasher), io.LimitReader(bufferedReader, maxUploadBytes+1)) - if err != nil { - return "", "", "", 0, err - } - if written > maxUploadBytes { - return "", "", "", 0, errLocalFileTooLarge - } + defer os.Remove(staged.absolutePath) //nolint:errcheck now := time.Now() relativePath := filepath.Join( @@ -1020,15 +1002,17 @@ func saveUploadedFile( fileID+"_"+sanitizeFileName(fileName), ) relativePath = filepath.ToSlash(relativePath) - if _, err = tmpFile.Seek(0, io.SeekStart); err != nil { + tmpFile, err := os.Open(staged.absolutePath) + if err != nil { return "", "", "", 0, err } + defer tmpFile.Close() //nolint:errcheck if _, err = store.Put(ctx, relativePath, tmpFile, objectstore.PutOptions{ - SizeBytes: written, - ContentType: detectedMIME, + SizeBytes: staged.sizeBytes, + ContentType: staged.detectedMIME, }); err != nil { return "", "", "", 0, err } - return relativePath, detectedMIME, hex.EncodeToString(hasher.Sum(nil)), written, nil + return relativePath, staged.detectedMIME, staged.sha256, staged.sizeBytes, nil } diff --git a/backend/internal/application/upload/service_test.go b/backend/internal/application/upload/service_test.go index ca9a2c976..42d1a98b9 100644 --- a/backend/internal/application/upload/service_test.go +++ b/backend/internal/application/upload/service_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "os" "strings" "sync" "testing" @@ -16,6 +17,32 @@ import ( "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" ) +func TestPrepareTemporaryFileUsesUploadPolicyWithoutPersistence(t *testing.T) { + service := NewService(config.Config{ + MaxUploadFileBytes: 1024, + FileAllowedMIMETypes: "text/plain", + }, nil, nil, Hooks{}, ErrorSet{}, "") + prepared, err := service.PrepareTemporaryFile(t.Context(), TemporaryFileInput{ + FileName: "notes.txt", + MimeType: "text/plain", + DeclaredSize: int64(len("temporary content")), + Reader: strings.NewReader("temporary content"), + }) + if err != nil { + t.Fatalf("prepare temporary file: %v", err) + } + if prepared.FileCategory != "text" || prepared.DetectedMIME != "text/plain" { + t.Fatalf("unexpected prepared metadata: %#v", prepared) + } + if _, err = os.Stat(prepared.AbsolutePath); err != nil { + t.Fatalf("temporary file missing before cleanup: %v", err) + } + prepared.Cleanup() + if _, err = os.Stat(prepared.AbsolutePath); !os.IsNotExist(err) { + t.Fatalf("temporary file still exists after cleanup: %v", err) + } +} + func TestUploadFileReturnsExistingActiveDuplicate(t *testing.T) { ctx := context.Background() repo := newUploadTestRepo() diff --git a/backend/internal/application/upload/temporary_file.go b/backend/internal/application/upload/temporary_file.go new file mode 100644 index 000000000..8ba0546a0 --- /dev/null +++ b/backend/internal/application/upload/temporary_file.go @@ -0,0 +1,149 @@ +package upload + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "strings" + + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/conv" + "github.com/google/uuid" +) + +// TemporaryFileInput 描述只在当前请求生命周期内使用的文件。 +type TemporaryFileInput struct { + FileName string + MimeType string + DeclaredSize int64 + Reader io.Reader +} + +// PreparedTemporaryFile 是通过统一上传策略校验后的请求级临时文件。 +// 调用方必须调用 Cleanup；该文件不会写入对象存储、文件表或用户配额。 +type PreparedTemporaryFile struct { + FileID string + FileName string + MimeType string + DetectedMIME string + FileCategory string + SizeBytes int64 + SHA256 string + AbsolutePath string + Cleanup func() +} + +type stagedUpload struct { + absolutePath string + detectedMIME string + sha256 string + sizeBytes int64 +} + +// PrepareTemporaryFile 复用普通上传的文件名、MIME、大小和危险类型策略， +// 但只生成请求级临时文件，不产生任何持久化记录。 +func (s *Service) PrepareTemporaryFile(_ context.Context, input TemporaryFileInput) (*PreparedTemporaryFile, error) { + if input.Reader == nil { + return nil, s.errInvalidFileReference() + } + fileName := sanitizeFileName(input.FileName) + if fileName == "" { + return nil, s.errInvalidFileReference() + } + mimeType := strings.TrimSpace(input.MimeType) + if mimeType == "" { + mimeType = "application/octet-stream" + } + cfg := s.snapshot() + maxUploadBytes := cfg.MaxUploadFileBytes + if maxUploadBytes <= 0 { + maxUploadBytes = 20 * 1024 * 1024 + } + if input.DeclaredSize > maxUploadBytes { + return nil, s.errFileTooLarge() + } + + fileID := "temporary_" + conv.NormalizePublicID(uuid.NewString()) + staged, err := stageUploadedFile(input.Reader, fileID, fileName, maxUploadBytes, mimeType) + if err != nil { + if errors.Is(err, errLocalFileTooLarge) { + return nil, s.errFileTooLarge() + } + return nil, err + } + cleanup := func() { _ = os.Remove(staged.absolutePath) } + category := inferFileCategory(staged.detectedMIME, fileName) + if isDangerousMIME(staged.detectedMIME) { + cleanup() + return nil, s.errDangerousMIMEType() + } + if !isAllowedMIME(staged.detectedMIME, cfg) { + cleanup() + return nil, s.errMIMEBlocked() + } + if typeLimit := maxBytesForCategory(category, cfg); typeLimit > 0 && staged.sizeBytes > typeLimit { + cleanup() + return nil, s.errFileTooLarge() + } + + return &PreparedTemporaryFile{ + FileID: fileID, + FileName: fileName, + MimeType: mimeType, + DetectedMIME: staged.detectedMIME, + FileCategory: category, + SizeBytes: staged.sizeBytes, + SHA256: staged.sha256, + AbsolutePath: staged.absolutePath, + Cleanup: cleanup, + }, nil +} + +func stageUploadedFile( + reader io.Reader, + fileID string, + fileName string, + maxUploadBytes int64, + declaredMIME string, +) (stagedUpload, error) { + if maxUploadBytes <= 0 { + maxUploadBytes = 20 * 1024 * 1024 + } + tmpFile, err := os.CreateTemp("", fileID+"_*.upload") + if err != nil { + return stagedUpload{}, err + } + tmpName := tmpFile.Name() + remove := true + defer func() { + _ = tmpFile.Close() + if remove { + _ = os.Remove(tmpName) + } + }() + + bufferedReader := bufio.NewReader(reader) + header, _ := bufferedReader.Peek(512) + detectedMIME := detectContentMIME(header, declaredMIME, fileName) + hasher := sha256.New() + written, err := io.Copy(io.MultiWriter(tmpFile, hasher), io.LimitReader(bufferedReader, maxUploadBytes+1)) + if err != nil { + return stagedUpload{}, err + } + if written > maxUploadBytes { + return stagedUpload{}, errLocalFileTooLarge + } + if err = tmpFile.Close(); err != nil { + return stagedUpload{}, err + } + remove = false + return stagedUpload{ + absolutePath: tmpName, + detectedMIME: detectedMIME, + sha256: hex.EncodeToString(hasher.Sum(nil)), + sizeBytes: written, + }, nil +} diff --git a/backend/internal/transport/http/conversation/dto_request.go b/backend/internal/transport/http/conversation/dto_request.go index fdd3a2bb5..fd904f3f8 100644 --- a/backend/internal/transport/http/conversation/dto_request.go +++ b/backend/internal/transport/http/conversation/dto_request.go @@ -119,7 +119,7 @@ type SendMessageRequest struct { } // TemporaryChatMessageRequest 是仅在当前页面内维护的临时对话请求。 -// 历史正文由浏览器逐轮提交，服务端不创建会话或消息记录。 +// 历史正文和请求级附件由浏览器逐轮提交，服务端不创建会话、消息或文件记录。 type TemporaryChatMessageRequest struct { SessionID string `json:"sessionID" binding:"required,max=64"` ClientRunID string `json:"clientRunID" binding:"required,max=64"` @@ -132,10 +132,10 @@ type TemporaryChatMessageRequest struct { Messages []TemporaryChatHistoryMessage `json:"messages" binding:"required,min=1,max=100,dive"` } -// TemporaryChatHistoryMessage 是临时对话可提交的纯文本消息。 +// TemporaryChatHistoryMessage 是临时对话可提交的消息。 type TemporaryChatHistoryMessage struct { Role string `json:"role" binding:"required,oneof=user assistant"` - Content string `json:"content" binding:"required,max=200000"` + Content string `json:"content" binding:"max=200000"` } // MediaImageRequest 图片生成/编辑请求。 diff --git a/backend/internal/transport/http/conversation/handler_temporary_chat.go b/backend/internal/transport/http/conversation/handler_temporary_chat.go index fe9b81337..8ffd45496 100644 --- a/backend/internal/transport/http/conversation/handler_temporary_chat.go +++ b/backend/internal/transport/http/conversation/handler_temporary_chat.go @@ -6,23 +6,27 @@ import ( "encoding/hex" "encoding/json" "errors" + "math" + "mime/multipart" "net/http" "strings" + "sync" "time" appconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/response" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/middleware" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" ) const temporaryChatMaxRequestBytes = 8 << 20 // StreamTemporaryChatMessage godoc // @Summary 流式发送临时对话消息 -// @Description 由浏览器提交完整纯文本上下文；服务端不创建会话、消息、运行或断线续传记录 +// @Description 由浏览器提交完整上下文和可选请求级附件；服务端不创建会话、消息、运行、文件或断线续传记录 // @Tags chat -// @Accept json +// @Accept json,multipart/form-data // @Produce application/x-ndjson // @Security BearerAuth // @Param body body TemporaryChatMessageRequest true "临时对话参数" @@ -31,30 +35,26 @@ const temporaryChatMaxRequestBytes = 8 << 20 // @Failure 500 {object} ErrorDoc // @Router /temporary-chat/messages/stream [post] func (h *Handler) StreamTemporaryChatMessage(c *gin.Context) { - c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, temporaryChatMaxRequestBytes) - var req TemporaryChatMessageRequest - if err := c.ShouldBindJSON(&req); err != nil { - var maxBytesErr *http.MaxBytesError - if errors.As(err, &maxBytesErr) { - response.Error(c, http.StatusRequestEntityTooLarge, "temporary chat context is too large") - return - } - response.InvalidRequestBody(c, err) + req, attachments, closeAttachments, ok := h.bindTemporaryChatRequest(c) + if !ok { return } + defer closeAttachments() req.Options = sanitizeMessageOptions(req.Options) input := appconversation.TemporaryChatInput{ - UserID: middleware.MustUserID(c), - RequestID: middleware.MustRequestID(c), - SessionID: strings.TrimSpace(req.SessionID), - ClientRunID: strings.TrimSpace(req.ClientRunID), - Model: strings.TrimSpace(req.Model), - Options: req.Options, - SelectedToolIDs: append([]uint(nil), req.SelectedToolIDs...), - SkillIDs: append([]uint(nil), req.SkillIDs...), - KnowledgeBaseIDs: append([]string(nil), req.KnowledgeBaseIDs...), - HTMLVisualPromptEnabled: req.HTMLVisualPrompt, - Messages: make([]appconversation.TemporaryChatMessage, 0, len(req.Messages)), + UserID: middleware.MustUserID(c), + RequestID: middleware.MustRequestID(c), + SessionID: strings.TrimSpace(req.SessionID), + ClientRunID: strings.TrimSpace(req.ClientRunID), + Model: strings.TrimSpace(req.Model), + Options: req.Options, + SelectedToolIDs: append([]uint(nil), req.SelectedToolIDs...), + SkillIDs: append([]uint(nil), req.SkillIDs...), + KnowledgeBaseIDs: append([]string(nil), req.KnowledgeBaseIDs...), + HTMLVisualPromptEnabled: req.HTMLVisualPrompt, + Messages: make([]appconversation.TemporaryChatMessage, 0, len(req.Messages)), + Attachments: attachments, + ReleaseAttachmentSources: closeAttachments, } for _, item := range req.Messages { input.Messages = append(input.Messages, appconversation.TemporaryChatMessage{ @@ -120,13 +120,13 @@ func (h *Handler) StreamTemporaryChatMessage(c *gin.Context) { if !result.ModerationTerminalEmitted() && c.Request.Context().Err() == nil { _ = writeEvent(moderationBlockedStreamPayload(result)) } - h.recordTemporaryChatAuditAsync(c, req, "blocked") + h.recordTemporaryChatAuditAsync(c, req, len(input.Attachments), "blocked") return } if c.Request.Context().Err() == nil { _ = writeEvent(streamErrorPayload(streamErr)) } - h.recordTemporaryChatAuditAsync(c, req, "failed") + h.recordTemporaryChatAuditAsync(c, req, len(input.Attachments), "failed") return } @@ -136,24 +136,147 @@ func (h *Handler) StreamTemporaryChatMessage(c *gin.Context) { billingCancel() if billingErr != nil { _ = writeEvent(billingStreamErrorPayload(billingErr)) - h.recordTemporaryChatAuditAsync(c, req, "billing_failed") + h.recordTemporaryChatAuditAsync(c, req, len(input.Attachments), "billing_failed") return } if result.IsModerationBlocked() { if !result.ModerationTerminalEmitted() { _ = writeEvent(moderationBlockedStreamPayload(result)) } - h.recordTemporaryChatAuditAsync(c, req, "blocked") + h.recordTemporaryChatAuditAsync(c, req, len(input.Attachments), "blocked") return } _ = writeEvent(map[string]interface{}{ "type": "completed", "data": toSendMessageResponse(result), }) - h.recordTemporaryChatAuditAsync(c, req, "completed") + h.recordTemporaryChatAuditAsync(c, req, len(input.Attachments), "completed") +} + +func (h *Handler) bindTemporaryChatRequest(c *gin.Context) ( + TemporaryChatMessageRequest, + []appconversation.TemporaryChatAttachment, + func(), + bool, +) { + noop := func() {} + contentType := strings.ToLower(strings.TrimSpace(c.GetHeader("Content-Type"))) + if !strings.HasPrefix(contentType, "multipart/form-data") { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, temporaryChatMaxRequestBytes) + var req TemporaryChatMessageRequest + if err := c.ShouldBindJSON(&req); err != nil { + writeTemporaryChatBindError(c, err) + return TemporaryChatMessageRequest{}, nil, noop, false + } + return req, nil, noop, true + } + + policy, err := h.service.GetChatFilePolicy(c.Request.Context(), middleware.MustUserID(c)) + if err != nil { + response.Error(c, http.StatusInternalServerError, "failed to resolve temporary attachment policy") + return TemporaryChatMessageRequest{}, nil, noop, false + } + maxUploadBytes := policy.MaxUploadFileBytes + if maxUploadBytes <= 0 { + maxUploadBytes = 20 * 1024 * 1024 + } + requestLimit := int64(temporaryChatMaxRequestBytes) + if maxUploadBytes > (math.MaxInt64-requestLimit)/appconversation.TemporaryChatMaxAttachments { + requestLimit = math.MaxInt64 + } else { + requestLimit += maxUploadBytes * appconversation.TemporaryChatMaxAttachments + } + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, requestLimit) + if err = c.Request.ParseMultipartForm(1 << 20); err != nil { + writeTemporaryChatBindError(c, err) + return TemporaryChatMessageRequest{}, nil, noop, false + } + removeMultipartFiles := func() { + if c.Request.MultipartForm != nil { + _ = c.Request.MultipartForm.RemoveAll() + } + } + + var req TemporaryChatMessageRequest + if err = json.Unmarshal([]byte(c.PostForm("payload")), &req); err != nil { + removeMultipartFiles() + response.InvalidRequestBody(c, err) + return TemporaryChatMessageRequest{}, nil, noop, false + } + if err = binding.Validator.ValidateStruct(req); err != nil { + removeMultipartFiles() + response.InvalidRequestBody(c, err) + return TemporaryChatMessageRequest{}, nil, noop, false + } + var messageIndexes []int + if err = json.Unmarshal([]byte(c.PostForm("attachmentMessageIndexes")), &messageIndexes); err != nil { + removeMultipartFiles() + response.InvalidRequestBody(c, err) + return TemporaryChatMessageRequest{}, nil, noop, false + } + fileHeaders := c.Request.MultipartForm.File["attachments"] + if len(fileHeaders) == 0 || len(fileHeaders) != len(messageIndexes) || len(fileHeaders) > appconversation.TemporaryChatMaxAttachments { + removeMultipartFiles() + response.Error(c, http.StatusBadRequest, "invalid file reference") + return TemporaryChatMessageRequest{}, nil, noop, false + } + maxFilesPerMessage := policy.MaxMessageFiles + if maxFilesPerMessage <= 0 { + maxFilesPerMessage = 10 + } + counts := make(map[int]int) + opened := make([]multipart.File, 0, len(fileHeaders)) + var closeOnce sync.Once + closeAll := func() { + closeOnce.Do(func() { + for _, file := range opened { + _ = file.Close() + } + removeMultipartFiles() + }) + } + attachments := make([]appconversation.TemporaryChatAttachment, 0, len(fileHeaders)) + for index, header := range fileHeaders { + messageIndex := messageIndexes[index] + if messageIndex < 0 || messageIndex >= len(req.Messages) || strings.TrimSpace(req.Messages[messageIndex].Role) != "user" { + closeAll() + response.Error(c, http.StatusBadRequest, "invalid file reference") + return TemporaryChatMessageRequest{}, nil, noop, false + } + counts[messageIndex]++ + if counts[messageIndex] > maxFilesPerMessage { + closeAll() + response.Error(c, http.StatusBadRequest, "too many files in one message") + return TemporaryChatMessageRequest{}, nil, noop, false + } + file, openErr := header.Open() + if openErr != nil { + closeAll() + response.Error(c, http.StatusBadRequest, "invalid file reference") + return TemporaryChatMessageRequest{}, nil, noop, false + } + opened = append(opened, file) + attachments = append(attachments, appconversation.TemporaryChatAttachment{ + MessageIndex: messageIndex, + FileName: header.Filename, + MimeType: header.Header.Get("Content-Type"), + DeclaredSize: header.Size, + Reader: file, + }) + } + return req, attachments, closeAll, true +} + +func writeTemporaryChatBindError(c *gin.Context, err error) { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) || strings.Contains(strings.ToLower(err.Error()), "request body too large") { + response.Error(c, http.StatusRequestEntityTooLarge, "temporary chat context is too large") + return + } + response.InvalidRequestBody(c, err) } -func (h *Handler) recordTemporaryChatAuditAsync(c *gin.Context, req TemporaryChatMessageRequest, status string) { +func (h *Handler) recordTemporaryChatAuditAsync(c *gin.Context, req TemporaryChatMessageRequest, attachmentCount int, status string) { userID := middleware.MustUserID(c) requestID := middleware.MustRequestID(c) clientIP := c.ClientIP() @@ -179,6 +302,7 @@ func (h *Handler) recordTemporaryChatAuditAsync(c *gin.Context, req TemporaryCha "selected_tool_count": len(req.SelectedToolIDs), "selected_skill_count": len(req.SkillIDs), "knowledge_base_count": len(req.KnowledgeBaseIDs), + "attachment_count": attachmentCount, "content_stored": false, }, }) diff --git a/frontend/features/chat/components/app-chat-area.tsx b/frontend/features/chat/components/app-chat-area.tsx index 2ac733ac3..f15e5706c 100644 --- a/frontend/features/chat/components/app-chat-area.tsx +++ b/frontend/features/chat/components/app-chat-area.tsx @@ -390,8 +390,15 @@ export function AppChatArea() { attachments, setAttachments, appendAttachmentsForKey, + temporary: temporaryMode, }); + const onTemporaryAttachmentsConsumed = React.useCallback((items: typeof attachments) => { + transferAttachments(items); + const consumedIDs = new Set(items.map((item) => item.fileID)); + setAttachments((current) => current.filter((item) => !consumedIDs.has(item.fileID))); + }, [setAttachments, transferAttachments]); + const { currentLeafMessage, onCycleMessageBranch, @@ -458,7 +465,7 @@ export function AppChatArea() { return () => detachConversationRun(normalizedRunID); }, [detachConversationRun, registerConversationRun, resumingConversationID, resumingRunID]); const generating = sending; - const uploadDropDisabled = temporaryMode || loading || uploading; + const uploadDropDisabled = loading || uploading; const onStopActiveMessage = React.useCallback(() => { const visibleRunID = currentLeafMessage?.runID?.trim() || ""; if (resumingRunID && visibleRunID === resumingRunID) { @@ -635,7 +642,10 @@ export function AppChatArea() { selectedSkillIDs: temporarySelectedSkillIDs, selectedKnowledgeBaseIDs, htmlVisualPromptEnabled: htmlVisualPrompt.enabled, + attachments, onDraftChange: setDraft, + onAttachmentsConsumed: onTemporaryAttachmentsConsumed, + releaseAttachments, }); const displayMessages = temporaryMode ? temporaryRuntime.messages : messagesWithInlineError; const artifactWorkspace = useChatArtifacts({ @@ -676,8 +686,8 @@ export function AppChatArea() { ragAvailabilityReason, sendShortcut, inputHeight, - attachments: temporaryMode ? EMPTY_LIST : attachments, - uploadingAttachments: temporaryMode ? EMPTY_LIST : uploadingAttachments, + attachments, + uploadingAttachments, modelOptions, billingDisplayCurrency, billingDisplayUsdToCnyRate, @@ -695,7 +705,7 @@ export function AppChatArea() { defaultOptions: selectedModelDefaultOptions, modelOptionPolicy, modelLoading: modelsLoading, - dropActive: temporaryMode ? false : fileDragActive, + dropActive: fileDragActive, temporaryMode, onDraftChange: setDraft, onModelChange: setSelectedPlatformModelName, @@ -781,15 +791,16 @@ export function AppChatArea() { starred={activeConversationStarred} canOperateConversation={temporaryMode ? false : canOperateConversation} messages={displayMessages} - messagesReadOnly={temporaryMode} + attachmentContentLoader={temporaryMode ? temporaryRuntime.loadAttachmentContent : undefined} + persistMessageFeedback={!temporaryMode} busy={composerSending} messageContentRef={messageContentRef} onScroll={onScroll} - onRetryUserMessage={onRetryUserMessage} - onRetryAssistantMessage={onRetryAssistantMessage} - onContinueAssistantMessage={onContinueAssistantMessage} - onEditAssistantMessage={onEditAssistantMessage} - onEditUserMessage={onEditUserMessage} + onRetryUserMessage={temporaryMode ? temporaryRuntime.onRetryUserMessage : onRetryUserMessage} + onRetryAssistantMessage={temporaryMode ? temporaryRuntime.onRetryAssistantMessage : onRetryAssistantMessage} + onContinueAssistantMessage={temporaryMode ? undefined : onContinueAssistantMessage} + onEditAssistantMessage={temporaryMode ? temporaryRuntime.onEditAssistantMessage : onEditAssistantMessage} + onEditUserMessage={temporaryMode ? temporaryRuntime.onEditUserMessage : onEditUserMessage} onForkMessage={temporaryMode ? undefined : onForkMessage} modelOptions={modelOptions} selectedPlatformModelName={selectedPlatformModelName} diff --git a/frontend/features/chat/components/sections/chat-area.tsx b/frontend/features/chat/components/sections/chat-area.tsx index 8889f93fe..811f8f860 100644 --- a/frontend/features/chat/components/sections/chat-area.tsx +++ b/frontend/features/chat/components/sections/chat-area.tsx @@ -152,6 +152,7 @@ type ChatAreaProps = { canOperateConversation: boolean; messages: ChatAreaMessage[]; messagesReadOnly?: boolean; + persistMessageFeedback?: boolean; busy: boolean; messageContentRef: React.RefObject; onScroll: (event: React.UIEvent) => void; @@ -535,6 +536,7 @@ export function ChatArea({ canOperateConversation, messages, messagesReadOnly = false, + persistMessageFeedback = true, busy, messageContentRef, onScroll, @@ -579,7 +581,9 @@ export function ChatArea({ screenshot, }: ChatAreaProps) { const t = useTranslations("chat"); - const { getReaction, onReactAssistantMessage } = useChatMessageFeedback(messages); + const { getReaction, onReactAssistantMessage } = useChatMessageFeedback(messages, { + persist: persistMessageFeedback, + }); const stableOnRetryUserMessage = useStableEvent(onRetryUserMessage); const stableOnRetryAssistantMessage = useStableEvent(onRetryAssistantMessage); const stableOnContinueAssistantMessage = useStableEvent(onContinueAssistantMessage ?? ((): undefined => undefined)); diff --git a/frontend/features/chat/components/sections/chat-input.tsx b/frontend/features/chat/components/sections/chat-input.tsx index 3ef72e9e6..936b6d814 100644 --- a/frontend/features/chat/components/sections/chat-input.tsx +++ b/frontend/features/chat/components/sections/chat-input.tsx @@ -828,6 +828,19 @@ function ChatInputComponent({ file={stablePreviewAttachment} open={previewAttachment !== null} onOpenChange={closePreviewDialog} + loadContent={stablePreviewAttachment.localFile + ? async (_file, signal) => { + if (signal.aborted) { + throw new DOMException("The operation was aborted", "AbortError"); + } + return { + blob: stablePreviewAttachment.localFile as File, + contentType: stablePreviewAttachment.localFile?.type || "application/octet-stream", + disposition: null, + contentLength: stablePreviewAttachment.localFile?.size ?? null, + }; + } + : undefined} /> ) : null} @@ -887,7 +900,7 @@ function ChatInputComponent({ }); } - if (!temporaryMode && files.length > 0) { + if (files.length > 0) { if (!event.clipboardData.getData("text/plain")) { event.preventDefault(); } @@ -921,8 +934,7 @@ function ChatInputComponent({
- {!temporaryMode ? ( - { @@ -982,8 +994,7 @@ function ChatInputComponent({ {tComposer("screenshot")} - - ) : null} + {!modelOptionPolicyDisabled ? ( >; appendAttachmentsForKey: (conversationKey: string, items: PendingAttachment[]) => void; + temporary?: boolean; }) { const t = useTranslations("chat.attachments"); const resolveErrorMessage = useLocalizedErrorMessage(); @@ -215,7 +222,9 @@ export function useChatAttachments({ sizeLimitExceeded: (limit: string) => t("policy.sizeLimitExceeded", { limit }), }; for (const file of files) { - const rejection = resolveUploadPolicyRejection(file, chatFilePolicy, policyLabels); + const rejection = temporary && normalizeUploadMime(file).startsWith("video/") + ? t("temporaryVideoUnsupported") + : resolveUploadPolicyRejection(file, chatFilePolicy, policyLabels); if (rejection) { toast.error(t("policyRejected"), { description: t("fileRejected", { name: file.name, reason: rejection }), @@ -237,6 +246,30 @@ export function useChatAttachments({ return; } + if (temporary) { + const localAttachments = policyAcceptedFiles.map((file): PendingAttachment => { + const category = inferUploadCategory(file); + return { + fileID: `temporary_${createSecureUUID()}`, + fileName: file.name, + mimeType: file.type || "application/octet-stream", + detectedMime: file.type || "application/octet-stream", + fileCategory: category, + sizeBytes: file.size, + previewURL: category === "image" ? URL.createObjectURL(file) : undefined, + processingStatus: "ready", + processingReady: true, + extractStatus: "none", + embedStatus: "none", + ragReady: false, + ragOptOut: false, + localFile: file, + }; + }); + appendAttachmentsForKey(targetConversationKey, localAttachments); + return; + } + const batchPrefix = `${Date.now()}-${Math.random().toString(16).slice(2)}`; const placeholders = policyAcceptedFiles.map((file, index) => ({ tempID: `${batchPrefix}-${index}`, @@ -358,6 +391,7 @@ export function useChatAttachments({ releaseAttachments, resolveErrorMessage, t, + temporary, uploading, uploadingByKey, ], @@ -412,7 +446,7 @@ export function useChatAttachments({ uploading, uploadingAttachments, maxFilesPerMessage, - fileMode: chatFilePolicy?.fileMode ?? "auto", + fileMode: temporary ? "full_context" : (chatFilePolicy?.fileMode ?? "auto"), ragAvailable: chatFilePolicy?.ragAvailable ?? null, ragAvailabilityReason: chatFilePolicy?.ragAvailabilityReason ?? "", releaseAttachments, diff --git a/frontend/features/chat/hooks/use-chat-message-feedback.ts b/frontend/features/chat/hooks/use-chat-message-feedback.ts index d63b59540..5fa0049d5 100644 --- a/frontend/features/chat/hooks/use-chat-message-feedback.ts +++ b/frontend/features/chat/hooks/use-chat-message-feedback.ts @@ -10,7 +10,10 @@ import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; import { setMessageFeedback } from "@/shared/api/conversation"; import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -export function useChatMessageFeedback(messages: ChatAreaMessage[]) { +export function useChatMessageFeedback( + messages: ChatAreaMessage[], + { persist = true }: { persist?: boolean } = {}, +) { const t = useTranslations("chat.feedback"); const resolveErrorMessage = useLocalizedErrorMessage(); const [overrides, setOverrides] = React.useState>({}); @@ -55,6 +58,17 @@ export function useChatMessageFeedback(messages: ChatAreaMessage[]) { [publicID]: reaction, })); + if (!persist) { + if (reaction === "up") { + toast.success(t("liked")); + } else if (reaction === "down") { + toast.success(t("disliked")); + } else { + toast.success(t("cleared")); + } + return; + } + try { const token = await resolveAccessToken(); if (!token) { @@ -85,7 +99,7 @@ export function useChatMessageFeedback(messages: ChatAreaMessage[]) { toast.error(t("failed"), { description }); } }, - [getReaction, messages, resolveErrorMessage, t], + [getReaction, messages, persist, resolveErrorMessage, t], ); return { diff --git a/frontend/features/chat/hooks/use-chat-temporary-runtime.ts b/frontend/features/chat/hooks/use-chat-temporary-runtime.ts index 2fcdfe125..c2cfe9cbe 100644 --- a/frontend/features/chat/hooks/use-chat-temporary-runtime.ts +++ b/frontend/features/chat/hooks/use-chat-temporary-runtime.ts @@ -10,14 +10,28 @@ import { clearLiveUpstreamThinkTrace, upsertLiveUpstreamThinkTrace, } from "@/features/chat/model/upstream-think-store"; -import type { ChatAreaMessage } from "@/features/chat/types/messages"; -import { streamTemporaryChatMessage } from "@/shared/api/conversation"; +import type { PendingAttachment } from "@/features/chat/types/chat-runtime"; +import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; +import { + resolveErrorDetails, + resolveErrorMessage, + resolveErrorSummary, +} from "@/features/chat/utils/chat-runtime"; +import { + streamTemporaryChatMessage, + TEMPORARY_CHAT_MAX_ATTACHMENTS, + TEMPORARY_CHAT_MAX_IMAGE_ATTACHMENTS, +} from "@/shared/api/conversation"; +import type { TemporaryChatRequestAttachment } from "@/shared/api/conversation"; import type { ConversationOptions, TemporaryChatHistoryMessage } from "@/shared/api/conversation.types"; +import type { FileContentLoader } from "@/shared/components/file-preview/preview-dialog"; import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import { createSecureUUID } from "@/shared/lib/secure-id"; type TemporaryMessage = TemporaryChatHistoryMessage & { id: string; + historyOffset: number; + parentID?: string; model?: string; runID?: string; streaming?: boolean; @@ -28,6 +42,13 @@ type TemporaryMessage = TemporaryChatHistoryMessage & { activityLabel?: string; processTrace?: ChatAreaMessage["processTrace"]; knowledgeSources?: ChatAreaMessage["knowledgeSources"]; + inlineAlert?: ChatAreaMessage["inlineAlert"]; + attachments?: MessageAttachment[]; + localAttachments?: PendingAttachment[]; +}; + +type TemporaryHistoryMessage = TemporaryChatHistoryMessage & { + attachments?: PendingAttachment[]; }; type TemporaryChatRuntimeInput = { @@ -39,9 +60,106 @@ type TemporaryChatRuntimeInput = { selectedSkillIDs: number[]; selectedKnowledgeBaseIDs: string[]; htmlVisualPromptEnabled: boolean; + attachments: PendingAttachment[]; onDraftChange: (value: string) => void; + onAttachmentsConsumed: (items: PendingAttachment[]) => void; + releaseAttachments: (items: PendingAttachment[]) => void; +}; + +type SubmitTemporaryTurnInput = { + content: string; + localAttachments: PendingAttachment[]; + baseHistory: TemporaryHistoryMessage[]; + replaceFromIndex?: number; + consumeComposer: boolean; }; +function toMessageAttachment(item: PendingAttachment): MessageAttachment { + return { + fileID: item.fileID, + fileName: item.fileName, + mimeType: item.mimeType, + detectedMime: item.detectedMime, + fileCategory: item.fileCategory, + sizeBytes: item.sizeBytes, + kind: item.fileCategory === "image" || item.mimeType.startsWith("image/") ? "image" : "file", + previewURL: item.previewURL, + processingStatus: item.processingStatus, + processingReady: item.processingReady, + extractStatus: item.extractStatus, + embedStatus: item.embedStatus, + ragReady: item.ragReady, + ragReason: item.ragReason, + ocrUsed: item.ocrUsed, + }; +} + +function isImageAttachment(item: PendingAttachment): boolean { + return item.fileCategory === "image" || item.mimeType.startsWith("image/"); +} + +function prepareTemporaryRequestHistory(fullHistory: TemporaryHistoryMessage[]): { + attachments: TemporaryChatRequestAttachment[]; + messages: TemporaryChatHistoryMessage[]; +} { + const historyTruncated = fullHistory.length > 99; + const history = historyTruncated ? fullHistory.slice(-99) : fullHistory; + const selected: Array = []; + let imageCount = 0; + + for (let messageIndex = history.length - 1; messageIndex >= 0; messageIndex -= 1) { + const message = history[messageIndex]; + if (message.role !== "user") { + continue; + } + const messageAttachments = message.attachments ?? []; + for (let index = messageAttachments.length - 1; index >= 0; index -= 1) { + const attachment = messageAttachments[index]; + if (!attachment.localFile || selected.length >= TEMPORARY_CHAT_MAX_ATTACHMENTS) { + continue; + } + const image = isImageAttachment(attachment); + if (image && imageCount >= TEMPORARY_CHAT_MAX_IMAGE_ATTACHMENTS) { + continue; + } + if (image) { + imageCount += 1; + } + selected.push({ + file: attachment.localFile, + fileID: attachment.fileID, + kind: image ? "image" : "file", + messageIndex, + }); + } + } + + selected.reverse(); + const selectedIDs = new Set(selected.map((item) => item.fileID)); + const messages = history.map((message, messageIndex): TemporaryChatHistoryMessage => { + const content = [ + historyTruncated && messageIndex === 0 ? "[Earlier temporary conversation omitted from this request]" : "", + message.content.trim(), + ].filter(Boolean).join("\n\n"); + if (message.role !== "user") { + return { role: message.role, content }; + } + const omittedNames = (message.attachments ?? []) + .filter((item) => !selectedIDs.has(item.fileID)) + .map((item) => item.fileName); + if (omittedNames.length === 0) { + return { role: message.role, content }; + } + const notice = `[Earlier temporary attachments omitted from this request: ${JSON.stringify(omittedNames)}]`; + return { role: message.role, content: content ? `${content}\n\n${notice}` : notice }; + }); + + return { + attachments: selected.map(({ fileID: _fileID, ...item }) => item), + messages, + }; +} + export function useChatTemporaryRuntime({ active, draft, @@ -51,12 +169,18 @@ export function useChatTemporaryRuntime({ selectedSkillIDs, selectedKnowledgeBaseIDs, htmlVisualPromptEnabled, + attachments, onDraftChange, + onAttachmentsConsumed, + releaseAttachments, }: TemporaryChatRuntimeInput) { const t = useTranslations("chat.temporary"); + const tSubmit = useTranslations("chat.submit"); const [messages, setMessages] = React.useState([]); const [sending, setSending] = React.useState(false); - const historyRef = React.useRef([]); + const messagesRef = React.useRef([]); + const historyRef = React.useRef([]); + const retainedAttachmentsRef = React.useRef(new Map()); const sessionIDRef = React.useRef(""); const abortControllerRef = React.useRef(null); const sendingRef = React.useRef(false); @@ -67,15 +191,57 @@ export function useChatTemporaryRuntime({ liveRunIDsRef.current.clear(); }, []); + const clearRetainedAttachments = React.useCallback(() => { + const retained = Array.from(retainedAttachmentsRef.current.values()); + retainedAttachmentsRef.current.clear(); + if (retained.length > 0) { + releaseAttachments(retained); + } + }, [releaseAttachments]); + + const replaceMessages = React.useCallback((next: TemporaryMessage[]) => { + messagesRef.current = next; + setMessages(next); + }, []); + const updateMessage = React.useCallback( (messageID: string, update: (message: TemporaryMessage) => TemporaryMessage) => { - setMessages((current) => current.map((message) => + replaceMessages(messagesRef.current.map((message) => message.id === messageID ? update(message) : message )); }, - [], + [replaceMessages], ); + const replaceMessageTail = React.useCallback(( + fromIndex: number, + replacements: TemporaryMessage[], + preservedAttachmentIDs: ReadonlySet = new Set(), + ) => { + const current = messagesRef.current; + const released: PendingAttachment[] = []; + for (const message of current.slice(fromIndex)) { + if (message.runID) { + clearLiveUpstreamThinkTrace(message.runID); + liveRunIDsRef.current.delete(message.runID); + } + for (const attachment of message.localAttachments ?? []) { + if (preservedAttachmentIDs.has(attachment.fileID)) { + continue; + } + const retained = retainedAttachmentsRef.current.get(attachment.fileID); + if (retained) { + retainedAttachmentsRef.current.delete(attachment.fileID); + released.push(retained); + } + } + } + if (released.length > 0) { + releaseAttachments(released); + } + replaceMessages([...current.slice(0, fromIndex), ...replacements]); + }, [releaseAttachments, replaceMessages]); + const finishSending = React.useCallback((controller: AbortController) => { if (abortControllerRef.current !== controller) { return; @@ -94,10 +260,11 @@ export function useChatTemporaryRuntime({ sendingRef.current = false; sessionIDRef.current = ""; historyRef.current = []; + clearRetainedAttachments(); clearLiveTraces(); setSending(false); - setMessages([]); - }, [active, clearLiveTraces]); + replaceMessages([]); + }, [active, clearLiveTraces, clearRetainedAttachments, replaceMessages]); React.useEffect(() => { const abort = () => abortControllerRef.current?.abort(); @@ -107,20 +274,53 @@ export function useChatTemporaryRuntime({ abort(); sessionIDRef.current = ""; historyRef.current = []; + clearRetainedAttachments(); clearLiveTraces(); }; - }, [clearLiveTraces]); + }, [clearLiveTraces, clearRetainedAttachments]); const stop = React.useCallback(() => { abortControllerRef.current?.abort(); }, []); - const send = React.useCallback(async () => { - const content = draft.trim(); + const loadAttachmentContent = React.useCallback(async (file, signal) => { + if (signal.aborted) { + throw new DOMException("The operation was aborted", "AbortError"); + } + const source = retainedAttachmentsRef.current.get(file.fileID)?.localFile; + if (!source) { + throw new Error("Temporary attachment is no longer available"); + } + return { + blob: source, + contentType: source.type || "application/octet-stream", + disposition: null, + contentLength: source.size, + }; + }, []); + + const submitTurn = React.useCallback(async ({ + content, + localAttachments, + baseHistory, + replaceFromIndex, + consumeComposer, + }: SubmitTemporaryTurnInput): Promise => { + const normalizedContent = content.trim(); const selectedModel = model.trim(); - if (!active || !content || !selectedModel || sendingRef.current) { - return; + if ( + !active || + (!normalizedContent && localAttachments.length === 0) || + !selectedModel || + sendingRef.current + ) { + return false; + } + if (localAttachments.some((item) => !(item.localFile instanceof File))) { + toast.error(t("failed")); + return false; } + sendingRef.current = true; setSending(true); const controller = new AbortController(); @@ -128,33 +328,72 @@ export function useChatTemporaryRuntime({ const token = await resolveAccessToken().catch(() => ""); if (controller.signal.aborted) { finishSending(controller); - return; + return false; } if (!token) { finishSending(controller); toast.error(t("sessionExpired")); - return; + return false; } if (!sessionIDRef.current) { sessionIDRef.current = createSecureUUID(); } - const userMessage: TemporaryMessage = { id: createSecureUUID(), role: "user", content }; + for (const attachment of localAttachments) { + retainedAttachmentsRef.current.set(attachment.fileID, attachment); + } + if (consumeComposer) { + onAttachmentsConsumed(localAttachments); + onDraftChange(""); + } + + historyRef.current = baseHistory; + const historyOffset = baseHistory.length; + const currentMessages = messagesRef.current; + const insertionIndex = replaceFromIndex ?? currentMessages.length; + const userMessage: TemporaryMessage = { + id: createSecureUUID(), + role: "user", + content: normalizedContent, + historyOffset, + parentID: currentMessages[insertionIndex - 1]?.id, + attachments: localAttachments.map(toMessageAttachment), + localAttachments, + }; const assistantID = createSecureUUID(); const clientRunID = createSecureUUID(); - const history: TemporaryChatHistoryMessage[] = [ - ...historyRef.current, - { role: "user", content }, - ]; - onDraftChange(""); - setMessages((current) => [ - ...current, + const replacements: TemporaryMessage[] = [ userMessage, - { id: assistantID, role: "assistant", content: "", model: selectedModel, runID: clientRunID, streaming: true }, - ]); + { + id: assistantID, + role: "assistant", + content: "", + historyOffset, + parentID: userMessage.id, + model: selectedModel, + runID: clientRunID, + streaming: true, + }, + ]; + if (replaceFromIndex === undefined) { + replaceMessages([...currentMessages, ...replacements]); + } else { + replaceMessageTail( + replaceFromIndex, + replacements, + new Set(localAttachments.map((item) => item.fileID)), + ); + } + const userHistory: TemporaryHistoryMessage = { + role: "user", + content: normalizedContent, + attachments: localAttachments, + }; + const preparedRequest = prepareTemporaryRequestHistory([...baseHistory, userHistory]); let streamedAssistantText = ""; let moderationBlocked = false; + try { const completed = await streamTemporaryChatMessage( token, @@ -167,7 +406,7 @@ export function useChatTemporaryRuntime({ skillIDs: selectedSkillIDs.length > 0 ? selectedSkillIDs : undefined, knowledgeBaseIDs: selectedKnowledgeBaseIDs.length > 0 ? selectedKnowledgeBaseIDs : undefined, htmlVisualPrompt: htmlVisualPromptEnabled || undefined, - messages: history, + messages: preparedRequest.messages, }, { signal: controller.signal, @@ -212,57 +451,197 @@ export function useChatTemporaryRuntime({ })); }, }, + preparedRequest.attachments, ); if (controller.signal.aborted || abortControllerRef.current !== controller) { - return; + throw new DOMException("The operation was aborted", "AbortError"); } const mappedAssistant = mapServerMessage(completed.assistantMessage); historyRef.current = [ - ...historyRef.current, - { role: "user", content }, + ...baseHistory, + userHistory, { role: "assistant", content: completed.assistantMessage.content }, ]; updateMessage(assistantID, (message) => ({ ...message, content: completed.assistantMessage.content, streaming: false, + failed: false, inputTokens: completed.userMessage.inputTokens, outputTokens: completed.assistantMessage.outputTokens, latencyMS: completed.assistantMessage.latencyMS, activityLabel: undefined, processTrace: mappedAssistant.processTrace, knowledgeSources: mappedAssistant.knowledgeSources, + inlineAlert: undefined, })); } catch (error) { + if (abortControllerRef.current !== controller) { + return false; + } const aborted = controller.signal.aborted; if (!moderationBlocked && streamedAssistantText.trim()) { historyRef.current = [ - ...historyRef.current, - { role: "user", content }, + ...baseHistory, + userHistory, { role: "assistant", content: streamedAssistantText }, ]; } + if (moderationBlocked) { + updateMessage(assistantID, (message) => ({ + ...message, + streaming: false, + failed: true, + activityLabel: undefined, + inlineAlert: undefined, + })); + return false; + } + const errorMessage = aborted ? "" : resolveErrorMessage(error, tSubmit("retryLater")); + const errorDetails = aborted ? undefined : resolveErrorDetails(error); updateMessage(assistantID, (message) => ({ ...message, - content: message.content || (aborted ? t("stopped") : t("failed")), + content: aborted ? message.content || t("stopped") : message.content, streaming: false, failed: true, + activityLabel: undefined, + inlineAlert: aborted + ? undefined + : { + title: tSubmit("generationInterrupted"), + message: errorMessage, + details: errorDetails, + }, })); if (!aborted) { - toast.error(t("failed"), { - description: error instanceof Error ? error.message : undefined, + toast.error(tSubmit("sendFailed"), { + description: resolveErrorSummary(error, tSubmit("retryLater")), }); } } finally { finishSending(controller); } - }, [active, draft, finishSending, htmlVisualPromptEnabled, model, onDraftChange, options, selectedKnowledgeBaseIDs, selectedSkillIDs, selectedToolIDs, t, updateMessage]); + return true; + }, [ + active, + finishSending, + htmlVisualPromptEnabled, + model, + onAttachmentsConsumed, + onDraftChange, + options, + replaceMessageTail, + replaceMessages, + selectedKnowledgeBaseIDs, + selectedSkillIDs, + selectedToolIDs, + t, + tSubmit, + updateMessage, + ]); + + const send = React.useCallback(async () => { + const currentAttachments = attachments.filter((item) => item.localFile instanceof File); + if (currentAttachments.length !== attachments.length) { + toast.error(t("failed")); + return; + } + await submitTurn({ + content: draft, + localAttachments: currentAttachments, + baseHistory: [...historyRef.current], + consumeComposer: true, + }); + }, [attachments, draft, submitTurn, t]); + + const resolveUserTurn = React.useCallback((message: ChatAreaMessage) => { + const messageIndex = messagesRef.current.findIndex((item) => item.id === message.publicID); + if (messageIndex < 0) { + return null; + } + const target = messagesRef.current[messageIndex]; + if (target.role === "user") { + return { message: target, index: messageIndex }; + } + for (let index = messageIndex - 1; index >= 0; index -= 1) { + const candidate = messagesRef.current[index]; + if (candidate.role === "user" && candidate.historyOffset === target.historyOffset) { + return { message: candidate, index }; + } + } + return null; + }, []); + + const retryMessage = React.useCallback(async (message: ChatAreaMessage) => { + const turn = resolveUserTurn(message); + if (!turn) { + toast.error(t("failed")); + return; + } + await submitTurn({ + content: turn.message.content, + localAttachments: turn.message.localAttachments ?? [], + baseHistory: historyRef.current.slice(0, turn.message.historyOffset), + replaceFromIndex: turn.index, + consumeComposer: false, + }); + }, [resolveUserTurn, submitTurn, t]); + + const editUserMessage = React.useCallback(async (message: ChatAreaMessage, content: string) => { + const turn = resolveUserTurn(message); + if (!turn) { + toast.error(t("failed")); + return false; + } + return submitTurn({ + content, + localAttachments: turn.message.localAttachments ?? [], + baseHistory: historyRef.current.slice(0, turn.message.historyOffset), + replaceFromIndex: turn.index, + consumeComposer: false, + }); + }, [resolveUserTurn, submitTurn, t]); + + const editAssistantMessage = React.useCallback(async (message: ChatAreaMessage, content: string) => { + const nextContent = content.trim(); + const assistantIndex = messagesRef.current.findIndex( + (item) => item.id === message.publicID && item.role === "assistant", + ); + if (!nextContent || assistantIndex < 0 || sendingRef.current) { + return false; + } + const assistant = messagesRef.current[assistantIndex]; + const turn = resolveUserTurn(message); + if (!turn) { + toast.error(t("failed")); + return false; + } + replaceMessageTail(assistantIndex + 1, []); + updateMessage(assistant.id, (current) => ({ + ...current, + content: nextContent, + streaming: false, + failed: false, + activityLabel: undefined, + inlineAlert: undefined, + })); + historyRef.current = [ + ...historyRef.current.slice(0, turn.message.historyOffset), + { + role: "user", + content: turn.message.content, + attachments: turn.message.localAttachments ?? [], + }, + { role: "assistant", content: nextContent }, + ]; + return true; + }, [replaceMessageTail, resolveUserTurn, t, updateMessage]); const areaMessages = React.useMemo( () => messages.map((message): ChatAreaMessage => ({ key: message.id, publicID: message.id, - parentPublicID: null, + parentPublicID: message.parentID ?? null, sourcePublicID: null, role: message.role, content: message.content, @@ -277,6 +656,8 @@ export function useChatTemporaryRuntime({ activityLabel: message.activityLabel, processTrace: message.processTrace, knowledgeSources: message.knowledgeSources, + inlineAlert: message.inlineAlert, + attachments: message.attachments, })), [messages], ); @@ -286,5 +667,10 @@ export function useChatTemporaryRuntime({ sending, send, stop, + loadAttachmentContent, + onRetryUserMessage: retryMessage, + onRetryAssistantMessage: retryMessage, + onEditUserMessage: editUserMessage, + onEditAssistantMessage: editAssistantMessage, }; } diff --git a/frontend/features/chat/types/chat-runtime.ts b/frontend/features/chat/types/chat-runtime.ts index aee9820c1..3532c580c 100644 --- a/frontend/features/chat/types/chat-runtime.ts +++ b/frontend/features/chat/types/chat-runtime.ts @@ -69,6 +69,7 @@ export type PendingAttachment = { ragReason?: string; ocrUsed?: boolean; ragOptOut?: boolean; + localFile?: File; }; export type UploadingAttachment = { diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index e60b0b1ff..22e01118e 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -277,6 +277,7 @@ "dropTitle": "Drop files to attach", "duplicateReused": "Duplicate file detected and reused", "partialUploadFailed": "Some files failed to upload", + "temporaryVideoUnsupported": "Temporary chats currently support image and document attachments, but not video attachments.", "retryFailedFiles": "Please retry the failed files.", "retryLater": "Please try again later.", "screenshotUnsupported": "This browser does not support screenshots", diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index 1e1462be6..82b419469 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -277,6 +277,7 @@ "dropTitle": "松开以添加附件", "duplicateReused": "检测到文件重复，已复用", "partialUploadFailed": "部分文件上传失败", + "temporaryVideoUnsupported": "临时对话当前支持图片和文档附件，不支持视频附件", "retryFailedFiles": "请重试失败的文件。", "retryLater": "请稍后重试。", "screenshotUnsupported": "当前浏览器不支持屏幕截图", diff --git a/frontend/shared/api/conversation.ts b/frontend/shared/api/conversation.ts index 55a2d746e..41025aa39 100644 --- a/frontend/shared/api/conversation.ts +++ b/frontend/shared/api/conversation.ts @@ -57,6 +57,15 @@ import { ApiError, apiRequest, pathParam } from "@/shared/api/http-client"; type RawTraceBlock = MessageTraceBlockResponse; +export const TEMPORARY_CHAT_MAX_ATTACHMENTS = 20; +export const TEMPORARY_CHAT_MAX_IMAGE_ATTACHMENTS = 10; + +export type TemporaryChatRequestAttachment = { + file: File; + messageIndex: number; + kind: "file" | "image"; +}; + type RawProcessTrace = Omit< MessageProcessTraceResponse, "events" | "process" | "promptTrace" | "tools" | "upstreamThink" @@ -1181,14 +1190,28 @@ async function postMessageStream( options: ConversationStreamOptions, cache?: RequestCache, ): Promise { - const response = await authedFetch(endpoint, { + return postMessageStreamRequest(accessToken, endpoint, { method: "POST", - accessToken, headers: { "Content-Type": "application/json", }, body: JSON.stringify(payload), signal: options.signal, + }, options, cache); +} + +async function postMessageStreamRequest( + accessToken: string, + endpoint: string, + request: RequestInit, + options: ConversationStreamOptions, + cache?: RequestCache, +): Promise { + const { signal, ...requestWithoutSignal } = request; + const response = await authedFetch(endpoint, { + ...requestWithoutSignal, + accessToken, + signal: signal ?? undefined, cache, }, true); @@ -1243,7 +1266,29 @@ export async function streamTemporaryChatMessage( accessToken: string, payload: TemporaryChatMessageRequest, options: ConversationStreamOptions = {}, + attachments: TemporaryChatRequestAttachment[] = [], ): Promise { + if (attachments.length > TEMPORARY_CHAT_MAX_ATTACHMENTS) { + throw new ApiError(`temporary chat supports at most ${TEMPORARY_CHAT_MAX_ATTACHMENTS} attachments`, 400); + } + if (attachments.filter((item) => item.kind === "image").length > TEMPORARY_CHAT_MAX_IMAGE_ATTACHMENTS) { + throw new ApiError(`temporary chat supports at most ${TEMPORARY_CHAT_MAX_IMAGE_ATTACHMENTS} image attachments`, 400); + } + if (attachments.length > 0) { + const body = new FormData(); + body.append("payload", JSON.stringify(payload)); + body.append("attachmentMessageIndexes", JSON.stringify(attachments.map((item) => item.messageIndex))); + for (const attachment of attachments) { + body.append("attachments", attachment.file, attachment.file.name); + } + return postMessageStreamRequest( + accessToken, + "/api/v1/temporary-chat/messages/stream", + { method: "POST", body, signal: options.signal }, + options, + "no-store", + ); + } return postMessageStream( accessToken, "/api/v1/temporary-chat/messages/stream", diff --git a/frontend/shared/api/conversation.types.ts b/frontend/shared/api/conversation.types.ts index 5c8593aaf..08042b946 100644 --- a/frontend/shared/api/conversation.types.ts +++ b/frontend/shared/api/conversation.types.ts @@ -245,8 +245,9 @@ export type SendMessageResult = Omit & { +export type TemporaryChatHistoryMessage = Omit & { role: "user" | "assistant"; + content: string; }; export type TemporaryChatMessageRequest = Omit & { diff --git a/packages/api-contract/src/types.generated.ts b/packages/api-contract/src/types.generated.ts index ef7ea600b..9b59a6753 100644 --- a/packages/api-contract/src/types.generated.ts +++ b/packages/api-contract/src/types.generated.ts @@ -9658,7 +9658,7 @@ export namespace Skills { export namespace TemporaryChat { /** - * @description 由浏览器提交完整纯文本上下文；服务端不创建会话、消息、运行或断线续传记录 + * @description 由浏览器提交完整上下文和可选请求级附件；服务端不创建会话、消息、运行、文件或断线续传记录 * @tags chat * @name MessagesStreamCreate * @summary 流式发送临时对话消息