From 9633fdae99a465446c5b2f88697e377f1c177060 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:59:02 +0800 Subject: [PATCH 01/54] feat: add xAI video extension and refine media playback --- backend/docs/docs.go | 105 ++++++++++++++++- backend/docs/swagger.json | 105 ++++++++++++++++- backend/docs/swagger.yaml | 71 ++++++++++++ .../application/channel/model_catalog.go | 63 +++++++--- .../application/channel/model_catalog_test.go | 65 ++++++++++- .../channel/service_default_route.go | 2 + .../application/channel/service_model.go | 5 +- .../channel/service_view_normalization.go | 2 +- .../conversation/model_option_policy.go | 4 + .../conversation/service_media_cancel.go | 3 +- .../conversation/service_media_video.go | 101 +++++++++++++--- .../settings/model_option_policy.go | 1 + .../internal/application/settings/service.go | 3 + .../application/settings/service_seed_test.go | 26 +++++ backend/internal/infra/config/config.go | 3 + backend/internal/infra/llm/adapter.go | 10 +- backend/internal/infra/llm/adapter_test.go | 9 ++ backend/internal/infra/llm/client.go | 10 +- .../internal/infra/llm/endpoint_url_test.go | 6 + backend/internal/infra/llm/openai.go | 2 + backend/internal/infra/llm/xai_videos.go | 99 +++++++++++++++- backend/internal/infra/llm/xai_videos_test.go | 39 +++++++ .../transport/http/channel/dto_request.go | 2 +- .../http/conversation/dto_request.go | 12 ++ .../http/conversation/handler_media.go | 72 +++++++++++- .../transport/http/conversation/router.go | 1 + frontend/features/admin/api/llm.types.ts | 13 ++- .../conversation/admin-conversation.tsx | 40 ++++--- .../models/models-capabilities-presets.tsx | 40 ++++++- .../sections/models/models-order-sheet.tsx | 2 +- .../sections/models/models-sheet.tsx | 76 +++++++++--- .../sections/models/models-table.tsx | 2 +- .../sections/upstreams/upstreams-sheet.tsx | 108 +++++++++--------- .../sections/upstreams/upstreams-table.tsx | 1 + .../features/admin/model/billing-settings.ts | 6 +- .../admin/model/conversation-settings.ts | 3 + frontend/features/admin/types/llm.ts | 1 + frontend/features/admin/utils/llm-display.ts | 14 ++- .../chat/components/app-chat-area.tsx | 90 ++++++++++----- .../chat/components/message/message-bot.tsx | 108 +++++++++++++----- .../chat/components/message/message-meta.tsx | 28 ++--- .../chat/components/sections/chat-area.tsx | 72 ++++++++---- .../chat/components/sections/chat-input.tsx | 102 ++++++++++------- .../components/sections/chat-model-config.tsx | 5 +- .../chat/hooks/use-chat-message-submit.ts | 32 +++++- .../chat/hooks/use-chat-model-options.ts | 75 ++++++++---- frontend/features/chat/model/chat-task.ts | 29 ++++- frontend/features/chat/types/chat-runtime.ts | 9 +- .../i18n/messages/en-US/admin-billing.json | 2 +- .../i18n/messages/en-US/admin-models.json | 3 +- .../i18n/messages/en-US/admin-upstreams.json | 3 +- frontend/i18n/messages/en-US/chat.json | 8 ++ .../i18n/messages/zh-CN/admin-billing.json | 2 +- .../i18n/messages/zh-CN/admin-models.json | 3 +- .../i18n/messages/zh-CN/admin-upstreams.json | 3 +- frontend/i18n/messages/zh-CN/chat.json | 8 ++ frontend/shared/api/conversation.ts | 16 +++ frontend/shared/api/conversation.types.ts | 5 + .../components/file-preview/preview-media.tsx | 72 +++++++++--- .../markdown/streamdown-components.tsx | 69 ++++------- .../shared/components/media-action-bar.tsx | 54 +++++++++ frontend/shared/lib/model-option-policy.ts | 4 + packages/api-contract/src/types.generated.ts | 41 ++++++- 63 files changed, 1576 insertions(+), 394 deletions(-) create mode 100644 frontend/shared/components/media-action-bar.tsx diff --git a/backend/docs/docs.go b/backend/docs/docs.go index b40e91b3e..01625408e 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -10371,6 +10371,64 @@ const docTemplate = `{ } } }, + "/conversations/{id}/media/videos/extensions/stream": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/x-ndjson" + ], + "tags": [ + "Conversations" + ], + "summary": "扩展会话视频", + "parameters": [ + { + "type": "string", + "description": "会话 Public ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "视频扩展请求", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/MediaVideoExtensionRequest" + } + } + ], + "responses": { + "200": { + "description": "NDJSON stream", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/Envelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/Envelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/Envelope" + } + } + } + } + }, "/conversations/{id}/messages": { "get": { "security": [ @@ -19225,6 +19283,50 @@ const docTemplate = `{ } } }, + "MediaVideoExtensionRequest": { + "type": "object", + "required": [ + "prompt", + "sourceVideoFileID" + ], + "properties": { + "branchReason": { + "type": "string", + "enum": [ + "default", + "retry", + "edit" + ] + }, + "clientRunID": { + "type": "string", + "maxLength": 64 + }, + "model": { + "type": "string", + "maxLength": 128 + }, + "options": { + "type": "object", + "additionalProperties": true + }, + "parentMessagePublicID": { + "type": "string", + "maxLength": 32 + }, + "prompt": { + "type": "string" + }, + "sourceMessagePublicID": { + "type": "string", + "maxLength": 32 + }, + "sourceVideoFileID": { + "type": "string", + "maxLength": 128 + } + } + }, "MemoryErrorDoc": { "type": "object", "required": [ @@ -20405,7 +20507,8 @@ const docTemplate = `{ "chat", "image_generation", "image_edit", - "video_generation" + "video_generation", + "video_extension" ] } } diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index a099428df..652b35a74 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -10364,6 +10364,64 @@ } } }, + "/conversations/{id}/media/videos/extensions/stream": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/x-ndjson" + ], + "tags": [ + "Conversations" + ], + "summary": "扩展会话视频", + "parameters": [ + { + "type": "string", + "description": "会话 Public ID", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "视频扩展请求", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/MediaVideoExtensionRequest" + } + } + ], + "responses": { + "200": { + "description": "NDJSON stream", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/Envelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/Envelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/Envelope" + } + } + } + } + }, "/conversations/{id}/messages": { "get": { "security": [ @@ -19218,6 +19276,50 @@ } } }, + "MediaVideoExtensionRequest": { + "type": "object", + "required": [ + "prompt", + "sourceVideoFileID" + ], + "properties": { + "branchReason": { + "type": "string", + "enum": [ + "default", + "retry", + "edit" + ] + }, + "clientRunID": { + "type": "string", + "maxLength": 64 + }, + "model": { + "type": "string", + "maxLength": 128 + }, + "options": { + "type": "object", + "additionalProperties": true + }, + "parentMessagePublicID": { + "type": "string", + "maxLength": 32 + }, + "prompt": { + "type": "string" + }, + "sourceMessagePublicID": { + "type": "string", + "maxLength": 32 + }, + "sourceVideoFileID": { + "type": "string", + "maxLength": 128 + } + } + }, "MemoryErrorDoc": { "type": "object", "required": [ @@ -20398,7 +20500,8 @@ "chat", "image_generation", "image_edit", - "video_generation" + "video_generation", + "video_extension" ] } } diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index becf9714a..dfd916b58 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -3915,6 +3915,38 @@ definitions: - data - errorMsg type: object + MediaVideoExtensionRequest: + properties: + branchReason: + enum: + - default + - retry + - edit + type: string + clientRunID: + maxLength: 64 + type: string + model: + maxLength: 128 + type: string + options: + additionalProperties: true + type: object + parentMessagePublicID: + maxLength: 32 + type: string + prompt: + type: string + sourceMessagePublicID: + maxLength: 32 + type: string + sourceVideoFileID: + maxLength: 128 + type: string + required: + - prompt + - sourceVideoFileID + type: object MemoryErrorDoc: properties: data: {} @@ -4742,6 +4774,7 @@ definitions: - image_generation - image_edit - video_generation + - video_extension type: string type: object ModelProbeResponse: @@ -15631,6 +15664,44 @@ paths: summary: 更新会话标签 tags: - chat + /conversations/{id}/media/videos/extensions/stream: + post: + consumes: + - application/json + parameters: + - description: 会话 Public ID + in: path + name: id + required: true + type: string + - description: 视频扩展请求 + in: body + name: payload + required: true + schema: + $ref: '#/definitions/MediaVideoExtensionRequest' + produces: + - application/x-ndjson + responses: + "200": + description: NDJSON stream + schema: + type: string + "400": + description: Bad Request + schema: + $ref: '#/definitions/Envelope' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/Envelope' + "404": + description: Not Found + schema: + $ref: '#/definitions/Envelope' + summary: 扩展会话视频 + tags: + - Conversations /conversations/{id}/messages: get: consumes: diff --git a/backend/internal/application/channel/model_catalog.go b/backend/internal/application/channel/model_catalog.go index e19bba7a9..93b7e6db7 100644 --- a/backend/internal/application/channel/model_catalog.go +++ b/backend/internal/application/channel/model_catalog.go @@ -16,12 +16,15 @@ const ( TaskTypeImageEdit = "image_edit" // TaskTypeVideoGeneration 表示视频生成任务。 TaskTypeVideoGeneration = "video_generation" + // TaskTypeVideoExtension 表示基于源视频的扩展任务。 + TaskTypeVideoExtension = "video_extension" - modelKindChat = "chat" - modelKindAudio = "audio" - modelKindImageGen = "image_gen" - modelKindImageEdit = "image_edit" - modelKindVideoGen = "video_gen" + modelKindChat = "chat" + modelKindAudio = "audio" + modelKindImageGen = "image_gen" + modelKindImageEdit = "image_edit" + modelKindVideoGen = "video_gen" + modelKindVideoExtension = "video_extension" compatibleOpenAI = "openai" compatibleAnthropic = "anthropic" @@ -38,6 +41,7 @@ const ( protocolXAIImage = llm.AdapterXAIImage protocolXAIImageEdits = llm.AdapterXAIImageEdits protocolXAIVideo = llm.AdapterXAIVideo + protocolXAIVideoExtensions = llm.AdapterXAIVideoExtensions ) var protocolDefaultKindOrder = []string{ @@ -46,6 +50,7 @@ var protocolDefaultKindOrder = []string{ modelKindImageGen, modelKindImageEdit, modelKindVideoGen, + modelKindVideoExtension, } func normalizeCompatible(raw string) string { @@ -134,11 +139,12 @@ func systemFallbackProtocols(compatible string) map[string]string { } case compatibleXAI: return map[string]string{ - modelKindChat: llm.AdapterXAIResponses, - modelKindAudio: llm.AdapterXAIResponses, - modelKindImageGen: protocolXAIImage, - modelKindImageEdit: protocolXAIImageEdits, - modelKindVideoGen: protocolXAIVideo, + modelKindChat: llm.AdapterXAIResponses, + modelKindAudio: llm.AdapterXAIResponses, + modelKindImageGen: protocolXAIImage, + modelKindImageEdit: protocolXAIImageEdits, + modelKindVideoGen: protocolXAIVideo, + modelKindVideoExtension: protocolXAIVideoExtensions, } case compatibleOpenRouter: return map[string]string{ @@ -176,7 +182,8 @@ func isKnownProtocol(raw string) bool { protocolGeminiInteractions, protocolXAIImage, protocolXAIImageEdits, - protocolXAIVideo: + protocolXAIVideo, + protocolXAIVideoExtensions: return true default: return false @@ -210,7 +217,7 @@ func resolveRouteProtocol(explicit string, upCompatible string, defaultsJSON str return "", ErrProtocolRequired } -// resolveRouteProtocols 解析批量导入时的协议列表，图片生成/编辑模型会按协议能力生成一到两条绑定。 +// resolveRouteProtocols 解析批量导入时的协议列表，并为同一媒体模型补齐配套协议绑定。 func resolveRouteProtocols(explicit []string, upCompatible string, defaultsJSON string, kindsJSON string) ([]string, error) { protocols := make([]string, 0, len(explicit)) seen := make(map[string]struct{}, len(explicit)) @@ -250,6 +257,12 @@ func resolveRouteProtocols(explicit []string, upCompatible string, defaultsJSON } } } + if hasModelKind(kinds, modelKindVideoGen) && hasModelKind(kinds, modelKindVideoExtension) && normalizeCompatible(upCompatible) == compatibleXAI { + generationProtocol := defaultRouteProtocolForKind(upCompatible, defaultsJSON, modelKindVideoGen) + if generationProtocol == protocolXAIVideo { + return []string{protocolXAIVideo, protocolXAIVideoExtensions}, nil + } + } protocol, err := resolveRouteProtocol("", upCompatible, defaultsJSON, kindsJSON) if err != nil { @@ -318,7 +331,7 @@ func isProtocolAllowedForKinds(kindsJSON string, protocol string) bool { return false } -// isSupportedRouteProtocolCombination 限制同一绑定 pair 只能单协议，或图片生成/编辑双协议。 +// isSupportedRouteProtocolCombination 限制同一绑定 pair 只能单协议，或同一媒体模型的配套协议。 func isSupportedRouteProtocolCombination(protocols []string) bool { seen := make(map[string]struct{}, len(protocols)) for _, raw := range protocols { @@ -341,7 +354,12 @@ func isSupportedRouteProtocolCombination(protocols []string) bool { } _, hasGeneration = seen[protocolXAIImage] _, hasEdit = seen[protocolXAIImageEdits] - return hasGeneration && hasEdit + if hasGeneration && hasEdit { + return true + } + _, hasVideoGeneration := seen[protocolXAIVideo] + _, hasVideoExtension := seen[protocolXAIVideoExtensions] + return hasVideoGeneration && hasVideoExtension } func protocolDefaultForKind(defaultsJSON string, kind string) string { @@ -421,6 +439,8 @@ func isProtocolAllowedForKind(kind string, protocol string) bool { default: return false } + case modelKindVideoExtension: + return protocol == protocolXAIVideoExtensions default: return false } @@ -436,6 +456,8 @@ func NormalizeTaskType(raw string) string { return TaskTypeImageEdit case TaskTypeVideoGeneration: return TaskTypeVideoGeneration + case TaskTypeVideoExtension: + return TaskTypeVideoExtension default: return TaskTypeChat } @@ -453,7 +475,9 @@ func IsRouteAllowedForTask(taskType string, kindsJSON string, protocol string) b case TaskTypeImageEdit: return isProtocolAllowedForKind(modelKindImageEdit, protocol) case TaskTypeVideoGeneration: - return isProtocolAllowedForKind(modelKindVideoGen, protocol) + return isProtocolAllowedForKind(modelKindVideoGen, protocol) && protocol != protocolXAIVideoExtensions + case TaskTypeVideoExtension: + return isProtocolAllowedForKind(modelKindVideoExtension, protocol) default: return isProtocolAllowedForKind(modelKindChat, protocol) || isProtocolAllowedForKind(modelKindAudio, protocol) } @@ -464,7 +488,9 @@ func IsRouteAllowedForTask(taskType string, kindsJSON string, protocol string) b case TaskTypeImageEdit: return hasModelKind(kinds, modelKindImageEdit) && isProtocolAllowedForKind(modelKindImageEdit, protocol) case TaskTypeVideoGeneration: - return hasModelKind(kinds, modelKindVideoGen) && isProtocolAllowedForKind(modelKindVideoGen, protocol) + return hasModelKind(kinds, modelKindVideoGen) && isProtocolAllowedForKind(modelKindVideoGen, protocol) && protocol != protocolXAIVideoExtensions + case TaskTypeVideoExtension: + return hasModelKind(kinds, modelKindVideoExtension) && isProtocolAllowedForKind(modelKindVideoExtension, protocol) default: for _, kind := range kinds { if (kind == modelKindChat || kind == modelKindAudio) && isProtocolAllowedForKind(kind, protocol) { @@ -507,8 +533,9 @@ func inferKindsJSON(platformModelName string) string { return `["image_gen","image_edit"]` case code == "dall-e-3", strings.HasPrefix(code, "imagen-"): return `["image_gen"]` - case code == "sora", code == "veo-2", strings.HasPrefix(code, "kling"), - strings.HasPrefix(code, "veo-"), isXAIVideoGenerationModel(code): + case isXAIVideoGenerationModel(code): + return `["video_gen","video_extension"]` + case code == "sora", code == "veo-2", strings.HasPrefix(code, "kling"), strings.HasPrefix(code, "veo-"): return `["video_gen"]` case strings.HasPrefix(code, "gpt-4o-audio"): return `["audio"]` diff --git a/backend/internal/application/channel/model_catalog_test.go b/backend/internal/application/channel/model_catalog_test.go index 183e45b26..b5de164e8 100644 --- a/backend/internal/application/channel/model_catalog_test.go +++ b/backend/internal/application/channel/model_catalog_test.go @@ -54,6 +54,9 @@ func TestProtocolDefaultsForXAIUsesXAIResponsesForConversationKinds(t *testing.T if defaults[modelKindVideoGen] != "xai_video" { t.Fatalf("expected xAI video default, got %q in %s", defaults[modelKindVideoGen], raw) } + if defaults[modelKindVideoExtension] != "xai_video_extensions" { + t.Fatalf("expected xAI video extension default, got %q in %s", defaults[modelKindVideoExtension], raw) + } } func TestProtocolDefaultsForOpenRouterUsesOpenRouterResponsesForConversationKinds(t *testing.T) { @@ -445,14 +448,22 @@ func TestInferKindsJSONRecognizesGeminiOmniInteractionsModel(t *testing.T) { } } -func TestInferKindsJSONRecognizesVideoOnlyModels(t *testing.T) { - for _, modelName := range []string{"veo-3.1-fast", "grok-imagine-video", "grok-imagine-video-1.5-preview"} { +func TestInferKindsJSONRecognizesVideoGenerationModels(t *testing.T) { + for _, modelName := range []string{"veo-3.1-fast"} { if got := inferKindsJSON(modelName); got != `["video_gen"]` { t.Fatalf("expected %s to infer video generation kind, got %s", modelName, got) } } } +func TestInferKindsJSONRecognizesXAIVideoExtensionModels(t *testing.T) { + for _, modelName := range []string{"grok-imagine-video", "grok-imagine-video-1.5-preview"} { + if got := inferKindsJSON(modelName); got != `["video_gen","video_extension"]` { + t.Fatalf("expected %s to infer video generation and extension kinds, got %s", modelName, got) + } + } +} + func TestInferKindsJSONRecognizesXAIImageModels(t *testing.T) { for _, modelName := range []string{ "grok-imagine-image", @@ -515,7 +526,7 @@ func TestResolveRouteProtocolAcceptsExplicitProtocolForAnyDeclaredKind(t *testin } } -func TestSupportedRouteProtocolCombinationOnlyAllowsSameProviderImagePair(t *testing.T) { +func TestSupportedRouteProtocolCombinationOnlyAllowsCompatibleMediaPairs(t *testing.T) { tests := []struct { name string protocols []string @@ -525,6 +536,7 @@ func TestSupportedRouteProtocolCombinationOnlyAllowsSameProviderImagePair(t *tes {name: "single image generation", protocols: []string{"openai_image_generations"}, want: true}, {name: "openai image generation and edit", protocols: []string{"openai_image_generations", "openai_image_edits"}, want: true}, {name: "xai image generation and edit", protocols: []string{"xai_image", "xai_image_edits"}, want: true}, + {name: "xai video generation and extension", protocols: []string{"xai_video", "xai_video_extensions"}, want: true}, {name: "duplicate protocol", protocols: []string{"openai_responses", "openai_responses"}, want: true}, {name: "two chat protocols", protocols: []string{"openai_responses", "openai_chat_completions"}, want: false}, {name: "image generation with chat", protocols: []string{"openai_image_generations", "openai_responses"}, want: false}, @@ -603,6 +615,32 @@ func TestResolveRouteProtocolsExpandsXAIDualImageKinds(t *testing.T) { } } +func TestResolveRouteProtocolsExpandsXAIVideoProtocols(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, compatibleXAI, "", `["video_gen","video_extension"]`) + if err != nil { + t.Fatalf("resolve xAI video protocols: %v", err) + } + expected := []string{"xai_video", "xai_video_extensions"} + if len(protocols) != len(expected) { + t.Fatalf("expected %d protocols, got %#v", len(expected), protocols) + } + for i, expectedProtocol := range expected { + if protocols[i] != expectedProtocol { + t.Fatalf("expected protocol %d to be %q, got %#v", i, expectedProtocol, protocols) + } + } +} + +func TestResolveRouteProtocolsDoesNotAddExtensionWithoutExtensionKind(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, compatibleXAI, "", `["video_gen"]`) + if err != nil { + t.Fatalf("resolve xAI generation-only protocol: %v", err) + } + if len(protocols) != 1 || protocols[0] != "xai_video" { + t.Fatalf("expected generation-only xAI protocol, got %#v", protocols) + } +} + func TestResolveRouteProtocolsKeepsSingleProtocolForGenerationOnlyModels(t *testing.T) { tests := []struct { name string @@ -699,9 +737,24 @@ func TestIsRouteAllowedForTaskSeparatesChatAndImageProtocols(t *testing.T) { if !IsRouteAllowedForTask(TaskTypeVideoGeneration, `["video_gen"]`, "xai_video") { t.Fatalf("expected video generation task to allow xAI video protocol") } + if IsRouteAllowedForTask(TaskTypeVideoGeneration, `["video_gen"]`, "xai_video_extensions") { + t.Fatal("video generation task must reject the xAI video extensions protocol") + } if IsRouteAllowedForTask(TaskTypeVideoGeneration, `["chat"]`, "openai_responses") { t.Fatalf("expected video generation task to reject chat protocol") } + if !IsRouteAllowedForTask(TaskTypeVideoExtension, `["video_gen","video_extension"]`, "xai_video_extensions") { + t.Fatal("xAI video extensions route should support video extension") + } + if IsRouteAllowedForTask(TaskTypeVideoExtension, `["video_gen","video_extension"]`, "xai_video") { + t.Fatal("xAI video generations route must not serve video extension") + } + if IsRouteAllowedForTask(TaskTypeVideoExtension, `["video_gen","video_extension"]`, "gemini_interactions") { + t.Fatal("non-xAI video route must not support video extension") + } + if IsRouteAllowedForTask(TaskTypeVideoExtension, `["video_gen"]`, "xai_video_extensions") { + t.Fatal("video extension task must require the video_extension kind") + } if IsRouteAllowedForTask(TaskTypeChat, `["video_gen"]`, "gemini_interactions") { t.Fatalf("expected chat task to reject video generation protocol") } @@ -732,6 +785,12 @@ func TestDefaultRouteModelMatchesTaskFiltersByKind(t *testing.T) { if defaultRouteModelMatchesTask(`["chat"]`, TaskTypeVideoGeneration) { t.Fatal("expected video generation default route to reject chat model") } + if !defaultRouteModelMatchesTask(`["video_gen","video_extension"]`, TaskTypeVideoExtension) { + t.Fatal("expected video extension default route to accept video extension model") + } + if defaultRouteModelMatchesTask(`["video_gen"]`, TaskTypeVideoExtension) { + t.Fatal("expected video extension default route to reject generation-only model") + } } func TestDisplayProtocolDefaultsJSONHidesLegacyInvalidDefaults(t *testing.T) { diff --git a/backend/internal/application/channel/service_default_route.go b/backend/internal/application/channel/service_default_route.go index 99b056f45..0eb3ab3a4 100644 --- a/backend/internal/application/channel/service_default_route.go +++ b/backend/internal/application/channel/service_default_route.go @@ -50,6 +50,8 @@ func defaultRouteModelMatchesTask(kindsJSON string, taskType string) bool { return hasModelKind(kinds, modelKindImageEdit) case TaskTypeVideoGeneration: return hasModelKind(kinds, modelKindVideoGen) + case TaskTypeVideoExtension: + return hasModelKind(kinds, modelKindVideoExtension) default: return hasModelKind(kinds, modelKindChat) } diff --git a/backend/internal/application/channel/service_model.go b/backend/internal/application/channel/service_model.go index 1006a34ff..39da9a42a 100644 --- a/backend/internal/application/channel/service_model.go +++ b/backend/internal/application/channel/service_model.go @@ -330,7 +330,7 @@ func (s *Service) ListActivePlatformModelNames(ctx context.Context) (map[string] return keys, nil } -// SupportsVideoGeneration 返回平台模型是否具有真实可路由的视频生成能力。 +// SupportsVideoGeneration 返回平台模型是否具有可按时长计费的视频能力。 func (s *Service) SupportsVideoGeneration(ctx context.Context, platformModelName string) (bool, error) { name, err := normalizePlatformModelName(platformModelName) if err != nil { @@ -344,7 +344,8 @@ func (s *Service) SupportsVideoGeneration(ctx context.Context, platformModelName if item.ActiveSourceCount <= 0 || strings.TrimSpace(item.PlatformModelName) != name { continue } - return hasModelKind(parseKinds(item.KindsJSON), modelKindVideoGen), nil + kinds := parseKinds(item.KindsJSON) + return hasModelKind(kinds, modelKindVideoGen) || hasModelKind(kinds, modelKindVideoExtension), nil } return false, nil } diff --git a/backend/internal/application/channel/service_view_normalization.go b/backend/internal/application/channel/service_view_normalization.go index 14557d955..301653235 100644 --- a/backend/internal/application/channel/service_view_normalization.go +++ b/backend/internal/application/channel/service_view_normalization.go @@ -320,7 +320,7 @@ func validateKinds(kinds []string) bool { switch kind { case modelKindChat: hasPrimary = true - case modelKindAudio, modelKindImageGen, modelKindImageEdit, modelKindVideoGen: + case modelKindAudio, modelKindImageGen, modelKindImageEdit, modelKindVideoGen, modelKindVideoExtension: hasPrimary = true default: return false diff --git a/backend/internal/application/conversation/model_option_policy.go b/backend/internal/application/conversation/model_option_policy.go index b158c6901..ad2ee3c2d 100644 --- a/backend/internal/application/conversation/model_option_policy.go +++ b/backend/internal/application/conversation/model_option_policy.go @@ -499,6 +499,8 @@ func sanitizeModelOptionValues(options map[string]interface{}, protocolKey strin sanitizeOpenAIServiceTier(options) case "xai_video": llm.SanitizeXAIVideoOptions(options) + case "xai_video_extensions": + llm.SanitizeXAIVideoExtensionOptions(options) case "openai_image_generations", "openai_image_edits": value, ok := modelParamIntFromOption(options["partial_images"]) if !ok { @@ -578,6 +580,8 @@ func modelOptionPolicyProtocolKey(protocol string) string { return "xai_image_edits" case llm.AdapterXAIVideo: return "xai_video" + case llm.AdapterXAIVideoExtensions: + return "xai_video_extensions" case llm.AdapterXAIResponses: return "xai_responses" default: diff --git a/backend/internal/application/conversation/service_media_cancel.go b/backend/internal/application/conversation/service_media_cancel.go index f925d7cbe..455806918 100644 --- a/backend/internal/application/conversation/service_media_cancel.go +++ b/backend/internal/application/conversation/service_media_cancel.go @@ -220,7 +220,8 @@ func mediaDurationSecondsFromOptions(options map[string]interface{}) int64 { // withDefaultMediaVideoDuration 仅向明确支持 duration 参数的视频协议补齐产品缺省值。 // 其他协议仍以其返回的真实媒体时长为准，避免发送未声明的厂商参数。 func withDefaultMediaVideoDuration(options map[string]interface{}, protocol string) map[string]interface{} { - if mediaDurationSecondsFromOptions(options) > 0 || llm.NormalizeAdapter(protocol) != llm.AdapterXAIVideo { + adapter := llm.NormalizeAdapter(protocol) + if mediaDurationSecondsFromOptions(options) > 0 || (adapter != llm.AdapterXAIVideo && adapter != llm.AdapterXAIVideoExtensions) { return options } next := make(map[string]interface{}, len(options)+1) diff --git a/backend/internal/application/conversation/service_media_video.go b/backend/internal/application/conversation/service_media_video.go index ea170beef..33aae3734 100644 --- a/backend/internal/application/conversation/service_media_video.go +++ b/backend/internal/application/conversation/service_media_video.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "strings" "time" @@ -23,11 +24,20 @@ import ( const maxMediaVideoInputImages = 1 +// MediaVideoTaskType 区分普通视频生成与基于源视频的扩展。 +type MediaVideoTaskType string + +const ( + MediaVideoTaskGeneration MediaVideoTaskType = "video_generation" + MediaVideoTaskExtension MediaVideoTaskType = "video_extension" +) + // MediaVideoInput 定义视频生成任务的应用层入参。 type MediaVideoInput struct { UserID uint ConversationID uint RequestID string + TaskType MediaVideoTaskType Prompt string PlatformModelName string Options map[string]interface{} @@ -76,6 +86,11 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( if strings.TrimSpace(input.Prompt) == "" { return nil, ErrMediaVideoPromptRequired } + taskType := normalizeMediaVideoTaskType(input.TaskType) + routeTaskType := channel.TaskTypeVideoGeneration + if taskType == MediaVideoTaskExtension { + routeTaskType = channel.TaskTypeVideoExtension + } platformModelName := strings.TrimSpace(input.PlatformModelName) if platformModelName == "" { @@ -86,7 +101,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( } route, err := s.routeResolver.ResolveRoute(ctx, channel.ResolveRouteInput{ PlatformModelName: platformModelName, - TaskType: channel.TaskTypeVideoGeneration, + TaskType: routeTaskType, Scope: channel.RouteScopeUser, UserID: input.UserID, ConversationID: input.ConversationID, @@ -98,6 +113,9 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( if !llm.IsVideoGenerationAdapter(route.Protocol) { return nil, ErrMediaRouteProtocolMismatch } + if taskType == MediaVideoTaskExtension && llm.NormalizeAdapter(route.Protocol) != llm.AdapterXAIVideoExtensions { + return nil, ErrMediaRouteProtocolMismatch + } videoEndpoint := llm.DefaultEndpointForAdapter(route.Protocol) if strings.TrimSpace(conversation.Model) != strings.TrimSpace(route.PlatformModelName) { conversation.Model = strings.TrimSpace(route.PlatformModelName) @@ -106,7 +124,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( return nil, err } } - resolvedAttachments, videoInputParts, err := s.resolveMediaVideoInputs(ctx, input) + resolvedAttachments, videoInputParts, videoExtensionSource, err := s.resolveMediaVideoInputs(ctx, input, taskType) if err != nil { return nil, err } @@ -117,7 +135,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( RequestID: strings.TrimSpace(input.RequestID), UserID: input.UserID, ConversationID: input.ConversationID, - TaskType: channel.TaskTypeVideoGeneration, + TaskType: routeTaskType, Endpoint: videoEndpoint, Provider: strings.TrimSpace(conversation.Provider), ProviderProtocol: route.Protocol, @@ -245,9 +263,11 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( } }() moderationFileIDs := make([]string, 0, len(resolvedAttachments)) - for _, item := range resolvedAttachments { - if fileID := strings.TrimSpace(item.FileID); fileID != "" { - moderationFileIDs = append(moderationFileIDs, fileID) + if taskType != MediaVideoTaskExtension { + for _, item := range resolvedAttachments { + if fileID := strings.TrimSpace(item.FileID); fileID != "" { + moderationFileIDs = append(moderationFileIDs, fileID) + } } } moderationCoord = s.startModerationRun(ctx, SendMessageInput{ @@ -285,6 +305,11 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( if llm.NormalizeAdapter(route.Protocol) == llm.AdapterGeminiInteractions { filteredOptions = withGeminiInteractionResponseType(filteredOptions, "video") } + if llm.NormalizeAdapter(route.Protocol) == llm.AdapterXAIVideoExtensions { + llm.SanitizeXAIVideoExtensionOptions(filteredOptions) + } else if llm.NormalizeAdapter(route.Protocol) == llm.AdapterXAIVideo { + llm.SanitizeXAIVideoOptions(filteredOptions) + } filteredOptions = withDefaultMediaVideoDuration(filteredOptions, route.Protocol) durationSeconds := mediaDurationSecondsFromOptions(filteredOptions) buildFailureResult := func(failure error, usage llm.Usage) *SendMessageResult { @@ -311,7 +336,8 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( Role: "user", Content: strings.TrimSpace(input.Prompt), }}, - Options: filteredOptions, + Options: filteredOptions, + VideoExtensionSource: videoExtensionSource, } if len(videoInputParts) > 0 { parts := make([]llm.ContentPart, 0, 1+len(videoInputParts)) @@ -357,7 +383,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( return buildFailureResult(retErr, mediaOutputUsage(output)), retErr } videoDurations, generatedDurationSeconds := resolveGeneratedVideoDurations(output.GeneratedVideos, durationSeconds) - if generatedDurationSeconds > 0 { + if taskType != MediaVideoTaskExtension && generatedDurationSeconds > 0 { durationSeconds = generatedDurationSeconds } @@ -498,7 +524,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( } if moderationCoord != nil { // Omni Moderation has no video modality. The prompt and optional input - // image still participate in the same barrier as other media tasks. + // image participate in the barrier; an extension source is intentionally excluded. s.completeModerationAfterSuccess( ctx, moderationCoord, @@ -512,6 +538,13 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( return result, nil } +func normalizeMediaVideoTaskType(taskType MediaVideoTaskType) MediaVideoTaskType { + if taskType == MediaVideoTaskExtension { + return MediaVideoTaskExtension + } + return MediaVideoTaskGeneration +} + func mediaVideoUserContentType(hasInputs bool) string { if hasInputs { return "mixed" @@ -519,30 +552,66 @@ func mediaVideoUserContentType(hasInputs bool) string { return "text" } -func (s *Service) resolveMediaVideoInputs(ctx context.Context, input MediaVideoInput) ([]AttachmentInput, []llm.ContentPart, error) { +func (s *Service) resolveMediaVideoInputs(ctx context.Context, input MediaVideoInput, taskType MediaVideoTaskType) ([]AttachmentInput, []llm.ContentPart, *llm.ContentPart, error) { if len(input.FileIDs) == 0 { - return nil, nil, nil + if taskType == MediaVideoTaskExtension { + return nil, nil, nil, ErrMediaVideoInputInvalid + } + return nil, nil, nil, nil } attachments, err := s.resolveAttachments(ctx, input.UserID, input.FileIDs) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if len(attachments) > maxMediaVideoInputImages { - return nil, nil, ErrMediaVideoTooManyInputs + return nil, nil, nil, ErrMediaVideoTooManyInputs + } + if taskType == MediaVideoTaskExtension { + if len(attachments) != 1 { + return nil, nil, nil, ErrMediaVideoInputInvalid + } + part, readErr := s.readMediaVideoExtensionSource(ctx, input.UserID, attachments[0].FileID) + if readErr != nil { + return nil, nil, nil, readErr + } + return attachments, nil, &part, nil } parts := make([]llm.ContentPart, 0, len(attachments)) for _, attachment := range attachments { if normalizeAttachmentKind(attachment.Kind, attachment.MimeType) != "image" { - return nil, nil, ErrMediaVideoInputInvalid + return nil, nil, nil, ErrMediaVideoInputInvalid } part, readErr := s.readMediaImageEditFile(ctx, input.UserID, attachment.FileID) if readErr != nil { - return nil, nil, readErr + return nil, nil, nil, readErr } part.FileName = mediaImageEditInputFileName(attachment.FileName, part.MimeType) parts = append(parts, part) } - return attachments, parts, nil + return attachments, parts, nil, nil +} + +func (s *Service) readMediaVideoExtensionSource(ctx context.Context, userID uint, fileID string) (llm.ContentPart, error) { + content, err := s.OpenFileContent(ctx, userID, strings.TrimSpace(fileID)) + if err != nil { + return llm.ContentPart{}, err + } + defer content.Reader.Close() //nolint:errcheck + limit := s.cfg.Snapshot().MaxUploadFileBytes + if limit <= 0 { + limit = 20 * 1024 * 1024 + } + data, err := io.ReadAll(io.LimitReader(content.Reader, limit+1)) + if err != nil { + return llm.ContentPart{}, err + } + if int64(len(data)) > limit { + return llm.ContentPart{}, ErrFileTooLarge + } + if detectGeneratedVideoMIME(data) != "video/mp4" { + return llm.ContentPart{}, ErrMediaVideoInputInvalid + } + return llm.ContentPart{Kind: llm.ContentPartVideo, MimeType: "video/mp4", Data: data, FileName: content.File.FileName}, nil } func mediaInputAttachmentRows(conversationID uint, userID uint, attachments []AttachmentInput) []model.Attachment { diff --git a/backend/internal/application/settings/model_option_policy.go b/backend/internal/application/settings/model_option_policy.go index 5c9bdb16e..df52127b6 100644 --- a/backend/internal/application/settings/model_option_policy.go +++ b/backend/internal/application/settings/model_option_policy.go @@ -21,6 +21,7 @@ var validModelOptionProtocolKeys = map[string]struct{}{ "xai_image": {}, "xai_image_edits": {}, "xai_video": {}, + "xai_video_extensions": {}, "gemini_generate_content": {}, "google_image_generation": {}, "gemini_interactions": {}, diff --git a/backend/internal/application/settings/service.go b/backend/internal/application/settings/service.go index 3844fd2ef..e0acbf653 100644 --- a/backend/internal/application/settings/service.go +++ b/backend/internal/application/settings/service.go @@ -237,7 +237,10 @@ func isLegacyDefaultModelOptionAllowedPaths(value string) bool { ) legacyCombinedDefault := cloneStringSliceMap(legacyInteractionsWithoutSummaries) legacyCombinedDefault["gemini_generate_content"] = previousGenerateContentDefault["gemini_generate_content"] + previousWithoutXAIVideoExtensions := cloneStringSliceMap(latestDefault) + delete(previousWithoutXAIVideoExtensions, "xai_video_extensions") previousDefaults := []map[string][]string{ + previousWithoutXAIVideoExtensions, previousGenerateContentDefault, previousInteractionsDefault, previousCombinedDefault, diff --git a/backend/internal/application/settings/service_seed_test.go b/backend/internal/application/settings/service_seed_test.go index 27c82cd31..81816d6ea 100644 --- a/backend/internal/application/settings/service_seed_test.go +++ b/backend/internal/application/settings/service_seed_test.go @@ -206,6 +206,32 @@ func TestSeedAddsXAIVideoToPreviousDefaultModelOptionAllowedPaths(t *testing.T) } } +func TestSeedAddsXAIVideoExtensionsToPreviousDefaultModelOptionAllowedPaths(t *testing.T) { + previousDefault := map[string][]string{} + if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { + t.Fatalf("decode current model option defaults: %v", err) + } + delete(previousDefault, "xai_video_extensions") + previousJSON, err := json.Marshal(previousDefault) + if err != nil { + t.Fatalf("encode previous model option defaults: %v", err) + } + repo := newSettingsSeedRepo(domainsettings.SystemSetting{ + Namespace: "chat", + Key: "model_option_allowed_paths", + Value: string(previousJSON), + ValueType: "json", + }) + service := NewService(repo, "") + + if err := service.Seed(context.Background(), config.Config{}); err != nil { + t.Fatalf("seed settings: %v", err) + } + if got := repo.items["chat:model_option_allowed_paths"].Value; got != config.DefaultModelOptionAllowedPathsJSON() { + t.Fatalf("expected xAI video extensions defaults to be added, got %q", got) + } +} + func TestSeedAddsGeminiThinkingSummariesToPreviousDefaultModelOptionAllowedPaths(t *testing.T) { previousDefault := map[string][]string{} if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { diff --git a/backend/internal/infra/config/config.go b/backend/internal/infra/config/config.go index 4361d7cb8..914b5d8da 100644 --- a/backend/internal/infra/config/config.go +++ b/backend/internal/infra/config/config.go @@ -168,6 +168,9 @@ func DefaultModelOptionAllowedPathsJSON() string { "aspect_ratio", "duration", "resolution" + ], + "xai_video_extensions": [ + "duration" ] }` } diff --git a/backend/internal/infra/llm/adapter.go b/backend/internal/infra/llm/adapter.go index 62e23b01d..6de55ba03 100644 --- a/backend/internal/infra/llm/adapter.go +++ b/backend/internal/infra/llm/adapter.go @@ -23,6 +23,7 @@ const ( AdapterXAIImage = "xai_image" // POST /v1/images/generations AdapterXAIImageEdits = "xai_image_edits" // POST /v1/images/edits AdapterXAIVideo = "xai_video" // POST /v1/videos/generations + GET /v1/videos/{request_id} + AdapterXAIVideoExtensions = "xai_video_extensions" // POST /v1/videos/extensions + GET /v1/videos/{request_id} ) var ( @@ -64,7 +65,8 @@ func IsKnownAdapter(raw string) bool { AdapterXAIResponses, AdapterXAIImage, AdapterXAIImageEdits, - AdapterXAIVideo: + AdapterXAIVideo, + AdapterXAIVideoExtensions: return true default: return false @@ -75,7 +77,7 @@ func IsKnownAdapter(raw string) bool { func IsImplementedAdapter(raw string) bool { switch NormalizeAdapter(raw) { case AdapterOpenAIResponses, AdapterOpenRouterChat, AdapterOpenRouterResponses, AdapterOpenAIChatCompletions, AdapterOpenAIImageGenerations, AdapterOpenAIImageEdits, AdapterXAIResponses, - AdapterAnthropicMessages, AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, AdapterGeminiInteractions, AdapterXAIImage, AdapterXAIImageEdits, AdapterXAIVideo: + AdapterAnthropicMessages, AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, AdapterGeminiInteractions, AdapterXAIImage, AdapterXAIImageEdits, AdapterXAIVideo, AdapterXAIVideoExtensions: return true default: return false @@ -141,7 +143,7 @@ func IsImageEditAdapter(raw string) bool { // IsVideoGenerationAdapter 返回协议是否属于独立视频生成链路。 func IsVideoGenerationAdapter(raw string) bool { switch NormalizeAdapter(raw) { - case AdapterGeminiInteractions, AdapterXAIVideo: + case AdapterGeminiInteractions, AdapterXAIVideo, AdapterXAIVideoExtensions: return true default: return false @@ -159,6 +161,8 @@ func DefaultEndpointForAdapter(adapter string) string { return EndpointImageEdits case AdapterXAIVideo: return EndpointVideoGenerations + case AdapterXAIVideoExtensions: + return EndpointVideoExtensions case AdapterGeminiInteractions: return EndpointInteractions default: diff --git a/backend/internal/infra/llm/adapter_test.go b/backend/internal/infra/llm/adapter_test.go index a5f41da96..118f4b30a 100644 --- a/backend/internal/infra/llm/adapter_test.go +++ b/backend/internal/infra/llm/adapter_test.go @@ -83,4 +83,13 @@ func TestXAIVideoAdapterCapabilities(t *testing.T) { if got := DefaultEndpointForAdapter(AdapterXAIVideo); got != EndpointVideoGenerations { t.Fatalf("expected xAI video endpoint, got %q", got) } + if !IsKnownAdapter(AdapterXAIVideoExtensions) || !IsImplementedAdapter(AdapterXAIVideoExtensions) { + t.Fatalf("expected xAI video extensions adapter to be known and implemented") + } + if !IsVideoGenerationAdapter(AdapterXAIVideoExtensions) { + t.Fatalf("expected xAI video extensions adapter to use the video media pipeline") + } + if got := DefaultEndpointForAdapter(AdapterXAIVideoExtensions); got != EndpointVideoExtensions { + t.Fatalf("expected xAI video extensions endpoint, got %q", got) + } } diff --git a/backend/internal/infra/llm/client.go b/backend/internal/infra/llm/client.go index f86e3b1eb..8405fb082 100644 --- a/backend/internal/infra/llm/client.go +++ b/backend/internal/infra/llm/client.go @@ -32,6 +32,8 @@ const ( EndpointImageEdits = "image_edits" // EndpointVideoGenerations 表示异步视频生成端点。 EndpointVideoGenerations = "video_generations" + // EndpointVideoExtensions 表示 xAI 异步视频扩展端点。 + EndpointVideoExtensions = "video_extensions" // EndpointInteractions 表示 Gemini Interactions API 端点。 EndpointInteractions = "interactions" ) @@ -96,12 +98,13 @@ func resolveStreamIdleTimeout(ms int) time.Duration { const ( ContentPartText = "text" // 纯文本 ContentPartImage = "image" // 图片（原始字节，序列化时 base64 编码） + ContentPartVideo = "video" // 视频（原始字节，仅供支持视频输入的 adapter 使用） ContentPartFile = "file" // 文件提取文本（前端解析后注入） ) // ContentPart 表示多模态消息中的一个内容片段。 type ContentPart struct { - Kind string // text | image | file + Kind string // text | image | video | file Text string // Kind=text 或 Kind=file 时的文本内容 MimeType string // Kind=image 时的 MIME 类型（如 "image/jpeg"） Data []byte // Kind=image 时的原始字节（发送时 base64 编码） @@ -154,6 +157,8 @@ type GenerateInput struct { ResponsesBackground bool // ImageEditMask 仅供图片编辑 adapter 使用，表示透明区域掩码。 ImageEditMask *ContentPart + // VideoExtensionSource 仅供视频扩展 adapter 使用，表示待扩展的源视频。 + VideoExtensionSource *ContentPart } // ToolDefinition 是模型可调用工具的统一声明。 @@ -855,6 +860,7 @@ func NewClient(outboundPolicy security.OutboundPolicy) *Client { AdapterXAIImage: &xAIImageAdapter{client: client}, AdapterXAIImageEdits: &xAIImageEditsAdapter{client: client}, AdapterXAIVideo: &xAIVideoAdapter{client: client}, + AdapterXAIVideoExtensions: &xAIVideoExtensionsAdapter{client: client}, AdapterAnthropicMessages: &anthropicMessagesAdapter{client: client}, AdapterGoogleGenerateContent: &geminiGenerateContentAdapter{client: client}, AdapterGoogleImageGeneration: &geminiImageGenerationAdapter{client: client}, @@ -1681,6 +1687,8 @@ func normalizeEndpoint(raw string) string { return EndpointImageEdits case EndpointVideoGenerations: return EndpointVideoGenerations + case EndpointVideoExtensions: + return EndpointVideoExtensions case EndpointInteractions: return EndpointInteractions default: diff --git a/backend/internal/infra/llm/endpoint_url_test.go b/backend/internal/infra/llm/endpoint_url_test.go index dd26e3cd1..66661971e 100644 --- a/backend/internal/infra/llm/endpoint_url_test.go +++ b/backend/internal/infra/llm/endpoint_url_test.go @@ -51,6 +51,12 @@ func TestBuildOpenAICompatibleURLsRespectVersionedBasePath(t *testing.T) { endpoint: EndpointVideoGenerations, want: "https://api.x.ai/v1/videos/generations", }, + { + name: "xai video extensions endpoint", + baseURL: "https://api.x.ai/v1", + endpoint: EndpointVideoExtensions, + want: "https://api.x.ai/v1/videos/extensions", + }, { name: "xai proxy plain base gets v1 image endpoint", baseURL: "https://proxy.example.com", diff --git a/backend/internal/infra/llm/openai.go b/backend/internal/infra/llm/openai.go index e9c036e87..7757afb3b 100644 --- a/backend/internal/infra/llm/openai.go +++ b/backend/internal/infra/llm/openai.go @@ -355,6 +355,8 @@ func buildOpenAIRequestURL(baseURL string, endpoint string) string { return buildVersionedEndpointURL(baseURL, "v1", "/images/edits") case EndpointVideoGenerations: return buildVersionedEndpointURL(baseURL, "v1", "/videos/generations") + case EndpointVideoExtensions: + return buildVersionedEndpointURL(baseURL, "v1", "/videos/extensions") default: return buildVersionedEndpointURL(baseURL, "v1", "/responses") } diff --git a/backend/internal/infra/llm/xai_videos.go b/backend/internal/infra/llm/xai_videos.go index 7a1865a8a..b6d7d06e1 100644 --- a/backend/internal/infra/llm/xai_videos.go +++ b/backend/internal/infra/llm/xai_videos.go @@ -25,6 +25,9 @@ func (a *xAIVideoAdapter) Name() string { return AdapterXAIVideo } func (a *xAIVideoAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { route.Protocol = AdapterXAIVideo + if input.VideoExtensionSource != nil { + return nil, fmt.Errorf("xai video generation protocol does not accept an extension source") + } route.Endpoint = EndpointVideoGenerations return a.client.generateXAIVideo(ctx, route, input) } @@ -43,9 +46,39 @@ func (a *xAIVideoAdapter) ListModels(ctx context.Context, route RouteConfig) ([] return a.client.listModelsOpenAICompatible(ctx, route) } +// xAIVideoExtensionsAdapter 实现 xAI 异步视频扩展协议。 +type xAIVideoExtensionsAdapter struct { + client *Client +} + +func (a *xAIVideoExtensionsAdapter) Name() string { return AdapterXAIVideoExtensions } + +func (a *xAIVideoExtensionsAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { + if input.VideoExtensionSource == nil { + return nil, fmt.Errorf("xai video extension protocol requires an MP4 source") + } + route.Protocol = AdapterXAIVideoExtensions + route.Endpoint = EndpointVideoExtensions + return a.client.generateXAIVideo(ctx, route, input) +} + +func (a *xAIVideoExtensionsAdapter) GenerateStream( + context.Context, + RouteConfig, + GenerateInput, + func(GenerateStreamEvent) error, +) (*GenerateOutput, error) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedStream, AdapterXAIVideoExtensions) +} + +func (a *xAIVideoExtensionsAdapter) ListModels(ctx context.Context, route RouteConfig) ([]ModelItem, error) { + route.Protocol = AdapterXAIVideoExtensions + return a.client.listModelsOpenAICompatible(ctx, route) +} + // generateXAIVideo 提交视频任务，并在同一请求超时范围内轮询官方结果端点。 func (c *Client) generateXAIVideo(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { - requestBody, debugBody, err := buildXAIVideoRequestBody(route.UpstreamModel, input) + requestBody, debugBody, err := buildXAIVideoSubmissionBody(route.UpstreamModel, input) if err != nil { return nil, err } @@ -53,7 +86,7 @@ func (c *Client) generateXAIVideo(ctx context.Context, route RouteConfig, input if err != nil { return nil, err } - requestURL := buildOpenAIRequestURL(route.BaseURL, EndpointVideoGenerations) + requestURL := buildOpenAIRequestURL(route.BaseURL, route.Endpoint) if requestURL == "" { return nil, fmt.Errorf("invalid base url") } @@ -89,6 +122,42 @@ func (c *Client) generateXAIVideo(ctx context.Context, route RouteConfig, input return c.pollXAIVideoResult(requestCtx, route, requestID, generatedMediaDurationSeconds(requestBody["duration"])) } +func buildXAIVideoSubmissionBody(model string, input GenerateInput) (map[string]interface{}, []byte, error) { + if input.VideoExtensionSource != nil { + return buildXAIVideoExtensionRequestBody(model, input) + } + return buildXAIVideoRequestBody(model, input) +} + +func buildXAIVideoExtensionRequestBody(model string, input GenerateInput) (map[string]interface{}, []byte, error) { + prompt := strings.TrimSpace(buildOpenAIImageGenerationPrompt(input.Messages)) + source := input.VideoExtensionSource + if prompt == "" { + return nil, nil, fmt.Errorf("video extension prompt required") + } + if source == nil || source.Kind != ContentPartVideo || strings.ToLower(strings.TrimSpace(source.MimeType)) != "video/mp4" || len(source.Data) == 0 { + return nil, nil, fmt.Errorf("video extension source must be a non-empty MP4 video") + } + payload := map[string]interface{}{ + "model": strings.TrimSpace(model), + "prompt": prompt, + "video": map[string]interface{}{ + "url": "data:video/mp4;base64," + base64.StdEncoding.EncodeToString(source.Data), + }, + } + applyXAIVideoExtensionParams(payload, input.Options) + debugPayload := map[string]interface{}{ + "model": payload["model"], + "prompt": payload["prompt"], + "video_source": "data:video/mp4;base64,[REDACTED]", + } + if duration, ok := payload["duration"]; ok { + debugPayload["duration"] = duration + } + debugBody, _ := json.Marshal(debugPayload) + return payload, debugBody, nil +} + func newXAIMediaRequest(ctx context.Context, method string, requestURL string, payload []byte, route RouteConfig) (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, method, requestURL, bytes.NewReader(payload)) if err != nil { @@ -158,6 +227,32 @@ func applyXAIVideoParams(payload map[string]interface{}, options map[string]inte } } +func applyXAIVideoExtensionParams(payload map[string]interface{}, options map[string]interface{}) { + normalized := maps.Clone(options) + SanitizeXAIVideoExtensionOptions(normalized) + if duration, ok := normalized["duration"]; ok { + payload["duration"] = duration + } +} + +// SanitizeXAIVideoExtensionOptions 仅保留 xAI 视频扩展支持的时长参数。 +func SanitizeXAIVideoExtensionOptions(options map[string]interface{}) { + if len(options) == 0 { + return + } + for key := range options { + if key != "duration" { + delete(options, key) + } + } + duration, ok := xAIMediaIntegerOption(options, "duration") + if !ok || duration < 2 || duration > 10 { + delete(options, "duration") + return + } + options["duration"] = duration +} + // SanitizeXAIVideoOptions 将 xAI 视频协议参数收敛为实际会上送的规范值。 // Application 层复用该函数，保证有效参数、计费和 adapter 请求一致。 func SanitizeXAIVideoOptions(options map[string]interface{}) { diff --git a/backend/internal/infra/llm/xai_videos_test.go b/backend/internal/infra/llm/xai_videos_test.go index 0489c952a..4dd25ce30 100644 --- a/backend/internal/infra/llm/xai_videos_test.go +++ b/backend/internal/infra/llm/xai_videos_test.go @@ -85,6 +85,45 @@ func TestBuildXAIVideoRequestBodyDropsUnsupportedParams(t *testing.T) { } } +func TestBuildXAIVideoExtensionRequestBody(t *testing.T) { + payload, debugBody, err := buildXAIVideoExtensionRequestBody("grok-imagine-video", GenerateInput{ + Messages: []Message{{Role: "user", Content: "Continue the camera movement"}}, + VideoExtensionSource: &ContentPart{Kind: ContentPartVideo, MimeType: "video/mp4", Data: []byte("source-video")}, + Options: map[string]interface{}{"duration": 8, "aspect_ratio": "16:9", "resolution": "1080p"}, + }) + if err != nil { + t.Fatalf("build xAI video extension request: %v", err) + } + if payload["duration"] != 8 || payload["prompt"] != "Continue the camera movement" { + t.Fatalf("unexpected video extension payload: %#v", payload) + } + if _, ok := payload["aspect_ratio"]; ok { + t.Fatalf("video extension must not forward generation-only options: %#v", payload) + } + video := asMap(payload["video"]) + if !strings.HasPrefix(getString(video["url"]), "data:video/mp4;base64,") { + t.Fatalf("expected MP4 data URL, got %#v", video) + } + if strings.Contains(string(debugBody), "c291cmNlLXZpZGVv") { + t.Fatalf("debug body must redact source video: %s", debugBody) + } +} + +func TestBuildXAIVideoExtensionRequestBodyRejectsInvalidSourceAndDuration(t *testing.T) { + payload, _, err := buildXAIVideoExtensionRequestBody("grok-imagine-video", GenerateInput{ + Messages: []Message{{Role: "user", Content: "Continue"}}, + VideoExtensionSource: &ContentPart{Kind: ContentPartVideo, MimeType: "video/webm", Data: []byte("source")}, + }) + if err == nil || payload != nil { + t.Fatalf("expected invalid source error, got payload=%#v err=%v", payload, err) + } + options := map[string]interface{}{"duration": 11, "resolution": "720p"} + SanitizeXAIVideoExtensionOptions(options) + if len(options) != 0 { + t.Fatalf("unsupported extension options must be removed: %#v", options) + } +} + func TestXAIVideoPollDelayClampsUnsafeValues(t *testing.T) { tests := map[string]time.Duration{ "": time.Second, diff --git a/backend/internal/transport/http/channel/dto_request.go b/backend/internal/transport/http/channel/dto_request.go index e73793108..85b5238ba 100644 --- a/backend/internal/transport/http/channel/dto_request.go +++ b/backend/internal/transport/http/channel/dto_request.go @@ -189,5 +189,5 @@ type ImportUpstreamModelItemRequest struct { // ModelProbeRequest 后台模型连通性测试请求。 type ModelProbeRequest struct { - TaskType string `json:"taskType,omitempty" binding:"omitempty,oneof=chat image_generation image_edit video_generation"` + TaskType string `json:"taskType,omitempty" binding:"omitempty,oneof=chat image_generation image_edit video_generation video_extension"` } diff --git a/backend/internal/transport/http/conversation/dto_request.go b/backend/internal/transport/http/conversation/dto_request.go index b8562efdb..c4e316532 100644 --- a/backend/internal/transport/http/conversation/dto_request.go +++ b/backend/internal/transport/http/conversation/dto_request.go @@ -133,6 +133,18 @@ type MediaVideoRequest struct { BranchReason string `json:"branchReason,omitempty" binding:"omitempty,oneof=default retry edit"` } +// MediaVideoExtensionRequest 视频扩展请求。 +type MediaVideoExtensionRequest struct { + Prompt string `json:"prompt" binding:"required"` + Model string `json:"model,omitempty" binding:"omitempty,max=128"` + Options map[string]interface{} `json:"options,omitempty"` + ClientRunID string `json:"clientRunID,omitempty" binding:"omitempty,max=64"` + SourceVideoFileID string `json:"sourceVideoFileID" binding:"required,max=128"` + ParentMessagePublicID string `json:"parentMessagePublicID,omitempty" binding:"omitempty,max=32"` + SourceMessagePublicID string `json:"sourceMessagePublicID,omitempty" binding:"omitempty,max=32"` + BranchReason string `json:"branchReason,omitempty" binding:"omitempty,oneof=default retry edit"` +} + // SetMessageFeedbackRequest 设置消息反馈请求。 type SetMessageFeedbackRequest struct { Feedback string `json:"feedback,omitempty" binding:"omitempty,oneof=up down"` diff --git a/backend/internal/transport/http/conversation/handler_media.go b/backend/internal/transport/http/conversation/handler_media.go index d8f006eea..fd1f273f5 100644 --- a/backend/internal/transport/http/conversation/handler_media.go +++ b/backend/internal/transport/http/conversation/handler_media.go @@ -29,6 +29,38 @@ func (h *Handler) StreamImageEdit(c *gin.Context) { // StreamVideoGeneration 处理会话内视频生成流式状态接口。 func (h *Handler) StreamVideoGeneration(c *gin.Context) { + h.streamMediaVideo(c, appconversation.MediaVideoTaskGeneration) +} + +// StreamVideoExtension 处理会话内视频扩展流式状态接口。 +// @Summary 扩展会话视频 +// @Tags Conversations +// @Accept json +// @Produce application/x-ndjson +// @Param id path string true "会话 Public ID" +// @Param payload body MediaVideoExtensionRequest true "视频扩展请求" +// @Success 200 {string} string "NDJSON stream" +// @Failure 400 {object} response.Envelope +// @Failure 401 {object} response.Envelope +// @Failure 404 {object} response.Envelope +// @Router /conversations/{id}/media/videos/extensions/stream [post] +func (h *Handler) StreamVideoExtension(c *gin.Context) { + h.streamMediaVideo(c, appconversation.MediaVideoTaskExtension) +} + +type mediaVideoTransportRequest struct { + Prompt string + Model string + Options map[string]interface{} + ClientRunID string + FileIDs []string + ParentMessagePublicID string + SourceMessagePublicID string + BranchReason string +} + +// streamMediaVideo 统一视频生成与扩展的 HTTP、授权和事件转发流程。 +func (h *Handler) streamMediaVideo(c *gin.Context, taskType appconversation.MediaVideoTaskType) { userID := middleware.MustUserID(c) publicID, err := stringParam(c, "id") if err != nil { @@ -44,10 +76,39 @@ func (h *Handler) StreamVideoGeneration(c *gin.Context) { response.Error(c, http.StatusInternalServerError, "load conversation failed") return } - var req MediaVideoRequest - if err := c.ShouldBindJSON(&req); err != nil { - response.Error(c, http.StatusBadRequest, err.Error()) - return + var req mediaVideoTransportRequest + if taskType == appconversation.MediaVideoTaskExtension { + var payload MediaVideoExtensionRequest + if err := c.ShouldBindJSON(&payload); err != nil { + response.Error(c, http.StatusBadRequest, err.Error()) + return + } + req = mediaVideoTransportRequest{ + Prompt: payload.Prompt, + Model: payload.Model, + Options: payload.Options, + ClientRunID: payload.ClientRunID, + FileIDs: []string{payload.SourceVideoFileID}, + ParentMessagePublicID: payload.ParentMessagePublicID, + SourceMessagePublicID: payload.SourceMessagePublicID, + BranchReason: payload.BranchReason, + } + } else { + var payload MediaVideoRequest + if err := c.ShouldBindJSON(&payload); err != nil { + response.Error(c, http.StatusBadRequest, err.Error()) + return + } + req = mediaVideoTransportRequest{ + Prompt: payload.Prompt, + Model: payload.Model, + Options: payload.Options, + ClientRunID: payload.ClientRunID, + FileIDs: payload.FileIDs, + ParentMessagePublicID: payload.ParentMessagePublicID, + SourceMessagePublicID: payload.SourceMessagePublicID, + BranchReason: payload.BranchReason, + } } req.ClientRunID = appconversation.EnsureMessageGenerationRunID(req.ClientRunID) req.Options = sanitizeMessageOptions(req.Options) @@ -67,6 +128,7 @@ func (h *Handler) StreamVideoGeneration(c *gin.Context) { UserID: userID, ConversationID: conversation.ID, RequestID: middleware.MustRequestID(c), + TaskType: taskType, Prompt: req.Prompt, PlatformModelName: req.Model, Options: req.Options, @@ -306,7 +368,7 @@ func mediaImageBillingInput( func mediaVideoBillingInput( userID uint, conversation *model.Conversation, - req *MediaVideoRequest, + req *mediaVideoTransportRequest, result *appconversation.SendMessageResult, ) appconversation.SendMessageBillingInput { input := appconversation.SendMessageBillingInput{ diff --git a/backend/internal/transport/http/conversation/router.go b/backend/internal/transport/http/conversation/router.go index d7d2e62fe..87b1e2f39 100644 --- a/backend/internal/transport/http/conversation/router.go +++ b/backend/internal/transport/http/conversation/router.go @@ -39,6 +39,7 @@ func (m *Module) RegisterRoutes(authRequired *gin.RouterGroup) { authRequired.POST("/conversations/:id/media/images/generations/stream", m.Handler.StreamImageGeneration) authRequired.POST("/conversations/:id/media/images/edits/stream", m.Handler.StreamImageEdit) authRequired.POST("/conversations/:id/media/videos/generations/stream", m.Handler.StreamVideoGeneration) + authRequired.POST("/conversations/:id/media/videos/extensions/stream", m.Handler.StreamVideoExtension) authRequired.GET("/context-artifacts/:id", m.Handler.GetContextArtifact) authRequired.GET("/conversation-runs/:run_id/stream", m.Handler.ResumeMessageGenerationStream) authRequired.POST("/conversation-runs/:run_id/cancel", m.Handler.CancelMessageGeneration) diff --git a/frontend/features/admin/api/llm.types.ts b/frontend/features/admin/api/llm.types.ts index 1ec5fde40..f53798b12 100644 --- a/frontend/features/admin/api/llm.types.ts +++ b/frontend/features/admin/api/llm.types.ts @@ -13,8 +13,8 @@ import type { ModelDataResponse, ModelDisplayGroupDataResponse, ModelDisplayGroupResponse, - ModelIconAssetResponse, ModelIconAssetListItemResponse, + ModelIconAssetResponse, ModelProbeBatchResponse, ModelProbeDebugRequestResponse, ModelProbeDebugResponse, @@ -23,13 +23,16 @@ import type { ModelResponse, ModelUpstreamSourceDataResponse, ModelUpstreamSourceResponse, + ModelVendorDataResponse, + ModelVendorDeleteConflictDetails, + ModelVendorResponse, ReorderModelsRequest, SetModelProtocolsRequest, SetModelsDisplayGroupRequest, UpdateModelDisplayGroupRequest, UpdateModelRequest, - UpdateModelVendorRequest, UpdateModelUpstreamSourceRequest, + UpdateModelVendorRequest, UpdateUpstreamRequest, UpsertUpstreamModelRequest, UpstreamAPIKeyResponse, @@ -39,9 +42,6 @@ import type { UpstreamRemoteModelResponse, UpstreamRemoteModelsResponse, UpstreamResponse, - ModelVendorDataResponse, - ModelVendorDeleteConflictDetails, - ModelVendorResponse, } from "@deeix/api-contract"; export type AdminLLMStatus = "active" | "inactive"; @@ -61,7 +61,8 @@ export type AdminLLMAdapter = | "xai_responses" | "xai_image" | "xai_image_edits" - | "xai_video"; + | "xai_video" + | "xai_video_extensions"; export type AdminLLMModelVendor = string; export type AdminLLMCompatible = | "openai" diff --git a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx index d899e6210..272b63238 100644 --- a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx +++ b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx @@ -1,13 +1,9 @@ "use client"; -import * as React from "react"; import { CircleHelp, Download, Save } from "lucide-react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; - -import { TaskModelField, type ModelOption } from "../shared/task-model-field"; -import { SettingsFieldEditor } from "../shared/settings-runtime-panel"; -import { ConversationPromptPresetsSection } from "@/features/admin/components/sections/conversation/conversation-prompt-presets"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -20,39 +16,42 @@ import { DialogTrigger, } from "@/components/ui/dialog"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -import { downloadBlob, readExportManifest } from "@/shared/lib/export-download"; -import { - SettingsFieldInset, - SettingsFieldItem, - SettingsFieldList, - SettingsFieldRow, - SettingsPage, - SettingsSection, - SettingsSectionSeparator, -} from "@/shared/components/settings-layout"; import { exportAllConversations, getAdminReferenceData, listAdminSettings, patchAdminSettings } from "@/features/admin/api"; +import { ConversationPromptPresetsSection } from "@/features/admin/components/sections/conversation/conversation-prompt-presets"; import { buildConversationSettingsFields, CONVERSATION_DEFAULT_MODEL_SYSTEM, CONVERSATION_TASK_MODEL_FOLLOW, + type ConversationSettingsField, fieldID, flattenConversationSettings, resolveVisibleConversationFields, toEditorField, - type ConversationSettingsField, } from "@/features/admin/model/conversation-settings"; -import { resolveAdminErrorMessage } from "@/features/admin/utils/admin-error"; import { buildTaskModelOptions } from "@/features/admin/model/task-model-options"; +import { resolveAdminErrorMessage } from "@/features/admin/utils/admin-error"; import type { PatchSettingItem } from "@/shared/api/settings.types"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { + SettingsFieldInset, + SettingsFieldItem, + SettingsFieldList, + SettingsFieldRow, + SettingsPage, + SettingsSection, + SettingsSectionSeparator, +} from "@/shared/components/settings-layout"; +import { downloadBlob, readExportManifest } from "@/shared/lib/export-download"; import { HARD_DENIED_MODEL_OPTION_PATHS, MODEL_OPTION_POLICY_PROTOCOL_LABELS, MODEL_OPTION_POLICY_PROTOCOLS, + type ModelOptionRuleMap, parseModelOptionRuleMap, uniqueModelOptionPaths, - type ModelOptionRuleMap, } from "@/shared/lib/model-option-policy"; +import { SettingsFieldEditor } from "../shared/settings-runtime-panel"; +import { type ModelOption, TaskModelField } from "../shared/task-model-field"; function isModelOptionPolicyField(field: ConversationSettingsField): boolean { return field.section === "optionPassthrough"; @@ -288,6 +287,9 @@ generationConfig.safetySettings.threshold`} "duration", "resolution" ], + "xai_video_extensions": [ + "duration" + ], "openai_chat_completions": [ "service_tier", "thinking.type" diff --git a/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx b/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx index f6293d93b..ac0ddf43a 100644 --- a/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx +++ b/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx @@ -1,12 +1,11 @@ "use client"; -import { useState } from "react"; import { Check, Copy, Search } from "lucide-react"; +import { useState } from "react"; import { toast } from "sonner"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { Dialog, DialogContent, @@ -15,11 +14,12 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { cn } from "@/lib/utils"; import type { AdminLLMAdapter, AdminLLMModelDTO } from "@/features/admin/api/llm.types"; -import { parseProtocolsJSON } from "@/shared/lib/model-protocols"; +import { cn } from "@/lib/utils"; import { MODEL_OPTION_POLICY_PROTOCOL_LABELS, resolveModelOptionPolicyProtocol } from "@/shared/lib/model-option-policy"; +import { parseProtocolsJSON } from "@/shared/lib/model-protocols"; type CapabilityPreset = { id: string; @@ -62,6 +62,7 @@ const XAI_IMAGE_ASPECT_RATIOS = [ const XAI_IMAGE_RESOLUTIONS = ["1k", "2k"]; const XAI_VIDEO_ASPECT_RATIOS = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]; const XAI_VIDEO_DURATIONS = Array.from({ length: 15 }, (_, index) => String(index + 1)); +const XAI_VIDEO_EXTENSION_DURATIONS = Array.from({ length: 9 }, (_, index) => String(index + 2)); const XAI_VIDEO_RESOLUTIONS = ["480p", "720p", "1080p"]; const XAI_IMAGE_OPTION_CONTROLS = [ @@ -581,6 +582,37 @@ const MODEL_CAPABILITY_PRESETS: CapabilityPreset[] = [ ], }, }, + { + id: "xai_video_extensions", + protocol: "xai_video_extensions", + payload: { + defaultOptions: { duration: 6 }, + optionControls: [ + { + path: "duration", + type: "select", + label: "Extension duration (seconds)", + description: "Additional video duration from 2 to 10 seconds.", + options: XAI_VIDEO_EXTENSION_DURATIONS, + }, + ], + mediaTasks: { + video_extension: { + enabled: true, + defaultOptions: { duration: 6 }, + optionControls: [ + { + path: "duration", + type: "select", + label: "Extension duration (seconds)", + description: "Additional video duration from 2 to 10 seconds.", + options: XAI_VIDEO_EXTENSION_DURATIONS, + }, + ], + }, + }, + }, + }, ]; const MODEL_CAPABILITY_PRESET_ORDER = new Map(MODEL_CAPABILITY_PRESETS.map((preset, index) => [preset.id, index])); diff --git a/frontend/features/admin/components/sections/models/models-order-sheet.tsx b/frontend/features/admin/components/sections/models/models-order-sheet.tsx index 60295a30d..91055948d 100644 --- a/frontend/features/admin/components/sections/models/models-order-sheet.tsx +++ b/frontend/features/admin/components/sections/models/models-order-sheet.tsx @@ -83,7 +83,7 @@ function KindBadges({ kindsJSON }: { kindsJSON: string }) {
{kinds.map((kind) => ( - {["chat", "audio", "image_gen", "image_edit", "video_gen"].includes(kind) + {["chat", "audio", "image_gen", "image_edit", "video_gen", "video_extension"].includes(kind) ? t(`kinds.${kind}`) : kind} diff --git a/frontend/features/admin/components/sections/models/models-sheet.tsx b/frontend/features/admin/components/sections/models/models-sheet.tsx index e18bd0697..231e9cf1d 100644 --- a/frontend/features/admin/components/sections/models/models-sheet.tsx +++ b/frontend/features/admin/components/sections/models/models-sheet.tsx @@ -391,23 +391,65 @@ export function ModelSheet({ open, mode, target, models, vendors, displayGroups, } function handleBindRowModelChange(rowID: string, upstreamModelID: string) { - setBindRows((current) => - current.map((row) => { - if (row.id !== rowID) { - return row; - } - const upstreamModels = upstreamModelsByID[row.draft.upstreamID] ?? []; - const selected = upstreamModels.find((item) => String(item.id) === upstreamModelID); - return { - ...row, - draft: { - ...row.draft, - upstreamModelID, - protocol: selected?.suggestedProtocol ?? "", - }, - }; - }), - ); + const targetRow = bindRows.find((row) => row.id === rowID); + const selected = targetRow + ? (upstreamModelsByID[targetRow.draft.upstreamID] ?? []).find( + (item) => String(item.id) === upstreamModelID, + ) + : undefined; + if (selected?.suggestedProtocol === "xai_video") { + setForm((current) => ({ + ...current, + kinds: Array.from(new Set([...current.kinds, "video_gen", "video_extension"])), + })); + } + setBindRows((current) => { + const currentTargetRow = current.find((row) => row.id === rowID); + if (!currentTargetRow) { + return current; + } + const protocols: AdminLLMAdapter[] = selected?.suggestedProtocol === "xai_video" + ? ["xai_video", "xai_video_extensions"] + : selected?.suggestedProtocol + ? [selected.suggestedProtocol] + : []; + const existingProtocols = new Set( + current + .filter( + (row) => + row.id !== rowID && + row.draft.upstreamID === currentTargetRow.draft.upstreamID && + row.draft.upstreamModelID === upstreamModelID, + ) + .map((row) => row.draft.protocol) + .filter(Boolean), + ); + const missingProtocols = protocols.filter((protocol) => !existingProtocols.has(protocol)); + const primaryProtocol = missingProtocols[0] ?? ""; + const companionRows = missingProtocols.slice(1).map((protocol) => + createModelSourceBindDraftRow({ + ...currentTargetRow.draft, + upstreamModelID, + protocol, + }), + ); + + return current.flatMap((row) => + row.id === rowID + ? [ + { + ...row, + draft: { + ...row.draft, + upstreamModelID, + protocol: primaryProtocol, + }, + }, + ...companionRows, + ] + : [row], + ); + }); } function setBindRowField( diff --git a/frontend/features/admin/components/sections/models/models-table.tsx b/frontend/features/admin/components/sections/models/models-table.tsx index cbfdce887..1528421a2 100644 --- a/frontend/features/admin/components/sections/models/models-table.tsx +++ b/frontend/features/admin/components/sections/models/models-table.tsx @@ -176,7 +176,7 @@ function KindsBadges({ kindsJson }: { kindsJson: string | null | undefined }) {
{kinds.map((kind) => ( - {["chat", "audio", "image_gen", "image_edit", "video_gen"].includes(kind) + {["chat", "audio", "image_gen", "image_edit", "video_gen", "video_extension"].includes(kind) ? t(`kinds.${kind}`) : kind} diff --git a/frontend/features/admin/components/sections/upstreams/upstreams-sheet.tsx b/frontend/features/admin/components/sections/upstreams/upstreams-sheet.tsx index 17fa6a4e7..25ba7f8ac 100644 --- a/frontend/features/admin/components/sections/upstreams/upstreams-sheet.tsx +++ b/frontend/features/admin/components/sections/upstreams/upstreams-sheet.tsx @@ -1,21 +1,46 @@ "use client"; -import { useState, useEffect } from "react"; -import type { - AdminLLMCompatible, - AdminLLMStatus, - AdminLLMUpstreamAPIKey, - AdminLLMUpstreamView, - CreateAdminLLMUpstreamRequest, - UpdateAdminLLMUpstreamRequest, -} from "@/features/admin/api/llm.types"; import { - createAdminLLMUpstream, - updateAdminLLMUpstream, -} from "@/features/admin/api"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; + Braces, + ChevronDown, + Fingerprint, + KeyRound, + ListPlus, + Network, + RotateCcw, + ScanSearch, + Trash2, +} from "lucide-react"; +import { useTranslations } from "next-intl"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; import { Select, SelectContent, @@ -30,55 +55,30 @@ import { SheetHeader, SheetTitle, } from "@/components/ui/sheet"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; +import { SpinnerLabel } from "@/components/ui/spinner"; import { Textarea } from "@/components/ui/textarea"; -import { - Accordion, - AccordionContent, - AccordionItem, - AccordionTrigger, -} from "@/components/ui/accordion"; import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { SpinnerLabel } from "@/components/ui/spinner"; -import { Label } from "@/components/ui/label"; -import { toast } from "sonner"; -import { useTranslations } from "next-intl"; -import { - Braces, - ChevronDown, - Fingerprint, - KeyRound, - ListPlus, - Network, - RotateCcw, - ScanSearch, - Trash2, -} from "lucide-react"; + createAdminLLMUpstream, + updateAdminLLMUpstream, +} from "@/features/admin/api"; +import type { + AdminLLMCompatible, + AdminLLMStatus, + AdminLLMUpstreamAPIKey, + AdminLLMUpstreamView, + CreateAdminLLMUpstreamRequest, + UpdateAdminLLMUpstreamRequest, +} from "@/features/admin/api/llm.types"; import { COMPATIBLE_OPTIONS, resolveProtocolLabel } from "@/features/admin/utils/llm-display"; -import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; -import { JsonCodeEditor } from "@/shared/components/json-code-editor"; import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { JsonCodeEditor } from "@/shared/components/json-code-editor"; +import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; const PROTOCOL_DEFAULT_KINDS = [ "chat", @@ -86,6 +86,7 @@ const PROTOCOL_DEFAULT_KINDS = [ "image_gen", "image_edit", "video_gen", + "video_extension", ] as const; const NO_PROTOCOL_DEFAULT = "__system_default__"; @@ -142,6 +143,7 @@ const PROTOCOL_OPTIONS_BY_KIND: Record<(typeof PROTOCOL_DEFAULT_KINDS)[number], "gemini_interactions", "xai_video", ], + video_extension: ["xai_video_extensions"], }; const CODE_TEXTAREA_CLASS = "font-mono text-xs placeholder:font-sans placeholder:text-xs"; diff --git a/frontend/features/admin/components/sections/upstreams/upstreams-table.tsx b/frontend/features/admin/components/sections/upstreams/upstreams-table.tsx index 8dc05c176..24cd82c1d 100644 --- a/frontend/features/admin/components/sections/upstreams/upstreams-table.tsx +++ b/frontend/features/admin/components/sections/upstreams/upstreams-table.tsx @@ -40,6 +40,7 @@ const PROTOCOL_DEFAULT_KIND_ORDER = [ "image_gen", "image_edit", "video_gen", + "video_extension", ]; const PROTOCOL_DEFAULT_KINDS = new Set(PROTOCOL_DEFAULT_KIND_ORDER); diff --git a/frontend/features/admin/model/billing-settings.ts b/frontend/features/admin/model/billing-settings.ts index aa7c4f097..b3f3d2271 100644 --- a/frontend/features/admin/model/billing-settings.ts +++ b/frontend/features/admin/model/billing-settings.ts @@ -308,7 +308,7 @@ const DEFAULT_IMPORT_MESSAGES: ModelPricingImportMessages = { duplicateModel: (model) => `${model} appears more than once`, pricingObject: (model) => `${model} pricing must be an object`, invalidPricingMode: (model) => `${model}.pricingMode must be token, call, duration, or tiered`, - durationVideoOnly: (model) => `${model}.pricingMode=duration requires the video_gen model capability`, + durationVideoOnly: (model) => `${model}.pricingMode=duration requires a video model capability`, invalidNumber: (model, field) => `${model}.${field} must be a number greater than or equal to 0`, invalidTieredPricing: (model, field) => `${model}.${field} must contain a non-empty tiers array`, invalidTieredPricingJSON: (model) => `${model}.tieredPricingJSON is not valid JSON`, @@ -570,7 +570,9 @@ export function buildPricingRows(models: AdminLLMModelDTO[], pricingItems: Admin icon: pricing?.modelIcon || model.icon || "", pricing, isFree: pricing?.isFree ?? false, - supportsVideoGeneration: parseKindsJSON(model.kindsJSON).includes("video_gen"), + supportsVideoGeneration: parseKindsJSON(model.kindsJSON).some( + (kind) => kind === "video_gen" || kind === "video_extension", + ), }; }); } diff --git a/frontend/features/admin/model/conversation-settings.ts b/frontend/features/admin/model/conversation-settings.ts index 0b318c35c..544ae1f2d 100644 --- a/frontend/features/admin/model/conversation-settings.ts +++ b/frontend/features/admin/model/conversation-settings.ts @@ -176,6 +176,9 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "aspect_ratio", "duration", "resolution" + ], + "xai_video_extensions": [ + "duration" ] }`; diff --git a/frontend/features/admin/types/llm.ts b/frontend/features/admin/types/llm.ts index c9ea6583d..0c25974aa 100644 --- a/frontend/features/admin/types/llm.ts +++ b/frontend/features/admin/types/llm.ts @@ -36,6 +36,7 @@ export const ADAPTER_LABELS: Record = { xai_image: resolveProtocolLabel("xai_image"), xai_image_edits: resolveProtocolLabel("xai_image_edits"), xai_video: resolveProtocolLabel("xai_video"), + xai_video_extensions: resolveProtocolLabel("xai_video_extensions"), }; // --------------------------------------------------------------------------- diff --git a/frontend/features/admin/utils/llm-display.ts b/frontend/features/admin/utils/llm-display.ts index 4ef328051..bac73e116 100644 --- a/frontend/features/admin/utils/llm-display.ts +++ b/frontend/features/admin/utils/llm-display.ts @@ -1,11 +1,12 @@ +import type { LucideIcon } from "lucide-react"; import { AudioLines, Bot, + Clapperboard, ImageIcon, Paintbrush, Video, } from "lucide-react"; -import type { LucideIcon } from "lucide-react"; import type { AdminLLMAdapter } from "@/features/admin/api/llm.types"; export const MODEL_KIND_META: Record< @@ -17,6 +18,7 @@ export const MODEL_KIND_META: Record< image_gen: { label: "Image generation", shortLabel: "Image generation", icon: ImageIcon }, image_edit: { label: "Image editing", shortLabel: "Image editing", icon: Paintbrush }, video_gen: { label: "Video generation", shortLabel: "Video generation", icon: Video }, + video_extension: { label: "Video extension", shortLabel: "Video extension", icon: Clapperboard }, }; export const COMPATIBLE_OPTIONS = [ @@ -48,6 +50,7 @@ export const PROTOCOL_OPTIONS: ReadonlyArray = [ { value: "xai_image", label: "Images Generations (xAI)", kinds: ["image_gen"] }, { value: "xai_image_edits", label: "Images Edits (xAI)", kinds: ["image_edit"] }, { value: "xai_video", label: "Video Generations (xAI)", kinds: ["video_gen"] }, + { value: "xai_video_extensions", label: "Video Extensions (xAI)", kinds: ["video_extension"] }, { value: "openrouter_chat_completions", label: "Chat Completions (OpenRouter)", kinds: ["chat"] }, { value: "openrouter_responses", label: "Responses (OpenRouter)", kinds: ["chat"] }, ] as const; @@ -69,6 +72,10 @@ const IMAGE_ROUTE_PROTOCOL_PAIRS: ReadonlyArray = [ + ["xai_video", "xai_video_extensions"], +]; + const LLM_STATUS_LABELS: Record = { active: "Enabled", inactive: "Disabled", @@ -124,8 +131,9 @@ export function isSupportedRouteProtocolSelection(protocols: readonly AdminLLMAd if (uniqueProtocols.length <= 1) { return true; } - return uniqueProtocols.length === 2 && IMAGE_ROUTE_PROTOCOL_PAIRS.some(([generationProtocol, editProtocol]) => - uniqueProtocols.includes(generationProtocol) && uniqueProtocols.includes(editProtocol), + return uniqueProtocols.length === 2 && [...IMAGE_ROUTE_PROTOCOL_PAIRS, ...VIDEO_ROUTE_PROTOCOL_PAIRS].some( + ([primaryProtocol, secondaryProtocol]) => + uniqueProtocols.includes(primaryProtocol) && uniqueProtocols.includes(secondaryProtocol), ); } diff --git a/frontend/features/chat/components/app-chat-area.tsx b/frontend/features/chat/components/app-chat-area.tsx index f63c831f9..244c16261 100644 --- a/frontend/features/chat/components/app-chat-area.tsx +++ b/frontend/features/chat/components/app-chat-area.tsx @@ -1,10 +1,19 @@ "use client"; -import * as React from "react"; import { useRouter, useSearchParams } from "next/navigation"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; - +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { ConversationShareDialog, sharePatchFromDTO, @@ -14,53 +23,43 @@ import { import { ChatArea, ChatAreaLoadError, ChatAreaSkeleton } from "@/features/chat/components/sections/chat-area"; import { ChatArtifactWorkspace } from "@/features/chat/components/sections/chat-artifact"; import { ChatEmptyState } from "@/features/chat/components/sections/chat-empty"; +import { ChatInput } from "@/features/chat/components/sections/chat-input"; +import { ChatScreenshotPreviewDialog } from "@/features/chat/components/sections/chat-screenshot-preview-dialog"; import { useChatSession } from "@/features/chat/context/chat-session-context"; import { useChatArtifacts } from "@/features/chat/hooks/use-chat-artifacts"; import { useChatAttachments } from "@/features/chat/hooks/use-chat-attachments"; -import { useChatComposerState } from "@/features/chat/hooks/use-chat-composer-state"; import { useChatComposerSelection } from "@/features/chat/hooks/use-chat-composer-selection"; -import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; +import { useChatComposerState } from "@/features/chat/hooks/use-chat-composer-state"; +import { useChatData } from "@/features/chat/hooks/use-chat-data"; import { useChatModelOptions } from "@/features/chat/hooks/use-chat-model-options"; import { useChatRuntime } from "@/features/chat/hooks/use-chat-runtime"; -import { useChatViewerProfile } from "@/features/chat/hooks/use-chat-viewer-profile"; import { useChatScreenshot } from "@/features/chat/hooks/use-chat-screenshot"; -import { parseConversationLabelsJSON } from "@/shared/lib/conversation-labels"; +import { useChatViewerProfile } from "@/features/chat/hooks/use-chat-viewer-profile"; import { useChatVisualPrompt } from "@/features/chat/hooks/use-chat-visual-prompt"; -import { ChatInput } from "@/features/chat/components/sections/chat-input"; -import { ChatScreenshotPreviewDialog } from "@/features/chat/components/sections/chat-screenshot-preview-dialog"; -import { resolveChatContentWidthClassName } from "@/shared/model/chat-content-width"; -import { DeleteFilesOption } from "@/shared/components/delete-files-option"; -import { useSettingsChatPreferences } from "@/features/settings/hooks/use-settings-chat-preferences"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; +import { useNewConversationDefaults } from "@/features/chat/hooks/use-new-conversation-defaults"; import { cloneConversationOptions, isConversationOptionsObject, sanitizeConversationOptions, } from "@/features/chat/model/conversation-options"; -import { useChatData } from "@/features/chat/hooks/use-chat-data"; -import { useNewConversationDefaults } from "@/features/chat/hooks/use-new-conversation-defaults"; import { toPendingAttachment } from "@/features/chat/model/message-submit"; +import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; +import { useSettingsChatPreferences } from "@/features/settings/hooks/use-settings-chat-preferences"; +import { cn } from "@/lib/utils"; import { getConversation } from "@/shared/api/conversation"; -import { listAvailableMCPTools } from "@/shared/api/mcp"; -import { getUserSettings, patchUserSettings } from "@/shared/api/user-settings"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import type { ConversationDTO, ConversationOptions } from "@/shared/api/conversation.types"; import type { FileObjectDTO } from "@/shared/api/file.types"; +import { listAvailableMCPTools } from "@/shared/api/mcp"; import type { MCPToolDTO } from "@/shared/api/mcp.types"; +import { getUserSettings, patchUserSettings } from "@/shared/api/user-settings"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { DeleteFilesOption } from "@/shared/components/delete-files-option"; +import { parseConversationLabelsJSON } from "@/shared/lib/conversation-labels"; import { hasMultipleImageAttachmentProcessors, normalizeImageAttachmentProcessorSelection, } from "@/shared/lib/mcp-tool-selection"; -import { cn } from "@/lib/utils"; +import { resolveChatContentWidthClassName } from "@/shared/model/chat-content-width"; const MODEL_OPTIONS_STORAGE_PREFIX = "deeix-chat:chat-model-options:"; const DEFAULT_MCP_TOOLS_SETTING_KEY = "chat.default_mcp_tool_ids"; @@ -774,6 +773,42 @@ export function AppChatArea() { ], ); + const onExtendGeneratedVideoAttachment = React.useCallback( + (attachment: MessageAttachment, sourceModelName?: string) => { + const normalizedSourceModelName = sourceModelName?.trim() || ""; + const sourceModel = modelOptions.find( + (item) => + item.platformModelName === normalizedSourceModelName && + item.videoExtension?.enabled, + ); + const extensionModel = + sourceModel ?? + (selectedModel?.videoExtension?.enabled ? selectedModel : undefined) ?? + modelOptions.find((item) => item.videoExtension?.enabled); + + if (!extensionModel) { + toast.error(t("submit.mediaMode.blockedDescriptions.video_extension_unsupported")); + return; + } + + releaseAttachments(attachments); + setAttachments([toPendingAttachment(attachment)]); + if (extensionModel.platformModelName !== selectedPlatformModelName) { + setSelectedPlatformModelName(extensionModel.platformModelName); + } + }, + [ + attachments, + modelOptions, + releaseAttachments, + selectedModel, + selectedPlatformModelName, + setAttachments, + setSelectedPlatformModelName, + t, + ], + ); + const onAttachExistingFile = React.useCallback( (file: FileObjectDTO) => { const alreadyAttached = attachments.some((item) => item.fileID === file.fileID); @@ -1293,6 +1328,7 @@ export function AppChatArea() { onModelChange={setSelectedPlatformModelName} onModelCatalogRefresh={refreshModelCatalogForComposer} onEditImageAttachment={onEditGeneratedImageAttachment} + onExtendVideoAttachment={onExtendGeneratedVideoAttachment} onOpenCodeArtifact={artifactWorkspace.openArtifact} onCycleMessageBranch={onCycleMessageBranch} onToggleStar={onToggleActiveConversationStar} diff --git a/frontend/features/chat/components/message/message-bot.tsx b/frontend/features/chat/components/message/message-bot.tsx index 0addff7bf..d4226e683 100644 --- a/frontend/features/chat/components/message/message-bot.tsx +++ b/frontend/features/chat/components/message/message-bot.tsx @@ -1,23 +1,9 @@ "use client"; -import * as React from "react"; -import { ChevronDown, CircleAlert, Film } from "lucide-react"; +import { ChevronDown, CircleAlert, Film, GalleryHorizontalEnd } from "lucide-react"; import { useTranslations } from "next-intl"; - -import { AssistantMessageMeta } from "@/features/chat/components/message/message-meta"; -import { MessageAttachmentRow } from "@/features/chat/components/message/message-attachment"; -import { MessageKnowledgeSources } from "@/features/chat/components/message/message-knowledge-sources"; -import { MessageProcessTrace, MessageTraceEventBlocks } from "@/features/chat/components/message/message-process-trace"; +import * as React from "react"; import { GrainientBackground } from "@/components/reactbits/backgrounds/grainient"; -import type { AssistantReaction } from "@/features/chat/components/message/message-meta"; -import type { - ChatAreaMessage, - ChatInlineAlert, - MessageAttachment, -} from "@/features/chat/types/messages"; -import { MarkdownImage, type MarkdownArtifactActions } from "@/shared/components/markdown/streamdown-components"; -import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; -import { PreviewMedia } from "@/shared/components/file-preview/preview-media"; import { Accordion, AccordionContent, @@ -31,20 +17,34 @@ import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Textarea } from "@/components/ui/textarea"; -import { cn } from "@/lib/utils"; -import { isUpstreamStreamingDebugBody, summarizeUpstreamError } from "@/features/chat/utils/chat-runtime"; -import { fetchFileContent, type FileContentResult } from "@/shared/api/file"; -import type { PreviewDialogFile } from "@/shared/components/file-preview/preview-dialog"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; +import { MessageAttachmentRow } from "@/features/chat/components/message/message-attachment"; +import { MessageKnowledgeSources } from "@/features/chat/components/message/message-knowledge-sources"; +import type { AssistantReaction } from "@/features/chat/components/message/message-meta"; +import { AssistantMessageMeta } from "@/features/chat/components/message/message-meta"; +import { MessageProcessTrace, MessageTraceEventBlocks } from "@/features/chat/components/message/message-process-trace"; import { resolveLeadingImagePreview } from "@/features/chat/model/media-image-preview"; import { clearLiveUpstreamThinkTrace, mergeLiveUpstreamThinkTrace, useLiveUpstreamThinkTrace, } from "@/features/chat/model/upstream-think-store"; -import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; +import type { + ChatAreaMessage, + ChatInlineAlert, + MessageAttachment, +} from "@/features/chat/types/messages"; +import { isUpstreamStreamingDebugBody, summarizeUpstreamError } from "@/features/chat/utils/chat-runtime"; +import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; +import { cn } from "@/lib/utils"; +import { type FileContentResult, fetchFileContent } from "@/shared/api/file"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import type { PreviewDialogFile } from "@/shared/components/file-preview/preview-dialog"; +import { PreviewMedia } from "@/shared/components/file-preview/preview-media"; +import { type MarkdownArtifactActions, MarkdownImage } from "@/shared/components/markdown/streamdown-components"; +import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; +import { MediaActionBar, MediaActionButton } from "@/shared/components/media-action-bar"; import { useBranding } from "@/shared/config/branding-provider"; +import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; const EMPTY_TRACE_EVENTS: NonNullable["events"] = []; @@ -69,6 +69,16 @@ function isVideoAttachment(attachment: MessageAttachment): boolean { ); } +function isMP4VideoAttachment(attachment: MessageAttachment): boolean { + const mimeType = attachment.mimeType.toLowerCase(); + const detectedMime = attachment.detectedMime?.toLowerCase() || ""; + return ( + mimeType === "video/mp4" || + detectedMime === "video/mp4" || + attachment.fileName.toLowerCase().endsWith(".mp4") + ); +} + function resolveFileIDFromImageSrc(src: string): string | null { if (typeof window === "undefined") { return null; @@ -146,6 +156,7 @@ type ChatMessageBotProps = { readOnly?: boolean; attachmentContentLoader?: (file: PreviewDialogFile) => Promise; onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; + onExtendVideoAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; artifactActions?: MarkdownArtifactActions; showBranchNavigator?: boolean; contentWidthClassName?: string; @@ -174,6 +185,7 @@ export function ChatMessageBot({ readOnly = false, attachmentContentLoader, onEditImageAttachment, + onExtendVideoAttachment, artifactActions, showBranchNavigator = true, contentWidthClassName = "max-w-[1080px]", @@ -239,6 +251,15 @@ export function ChatMessageBot({ : item.attachments ?? [], [inlineVideoAttachment, item.attachments], ); + const extendableVideoAttachment = + inlineVideoAttachment && isMP4VideoAttachment(inlineVideoAttachment) + ? inlineVideoAttachment + : null; + const onExtendVideo = React.useCallback(() => { + if (extendableVideoAttachment) { + onExtendVideoAttachment?.(extendableVideoAttachment, item.platformModelName); + } + }, [extendableVideoAttachment, item.platformModelName, onExtendVideoAttachment]); const hideGeneratedVideoMarkdown = inlineVideoAttachment ? isGeneratedVideoMarkdownContent(item.content, item.attachments ?? []) : false; @@ -389,7 +410,15 @@ export function ChatMessageBot({
{inlineVideoAttachment ? ( - + ) : null} {item.inlineAlert ? ( @@ -688,24 +717,42 @@ type InlineVideoPreviewState = | { status: "error"; message: string } | { status: "ready"; source: string; contentType: string }; +function InlineVideoLoadingPlaceholder() { + return ( +
+ +
+ ); +} + function MessageInlineVideoPreview({ attachment, loadContent, + onExtend, }: { attachment: MessageAttachment; loadContent?: (file: PreviewDialogFile) => Promise; + onExtend?: () => void; }) { const tPreview = useTranslations("files.previewDialog"); const tMessages = useTranslations("chat.messages"); const resolveErrorMessage = useLocalizedErrorMessage(); const objectURLRef = React.useRef(null); - const [state, setState] = React.useState({ status: "loading" }); const fileID = attachment.fileID; const fileName = attachment.fileName; const mimeType = attachment.mimeType; const detectedMime = attachment.detectedMime; const previewURL = attachment.previewURL; const sizeBytes = attachment.sizeBytes; + const [state, setState] = React.useState(() => + previewURL + ? { + status: "ready", + source: previewURL, + contentType: detectedMime || mimeType, + } + : { status: "loading" }, + ); const revokeObjectURL = React.useCallback(() => { if (!objectURLRef.current) { return; @@ -787,7 +834,7 @@ function MessageInlineVideoPreview({ ]); if (state.status === "loading") { - return ; + return ; } if (state.status === "error") { @@ -800,7 +847,7 @@ function MessageInlineVideoPreview({ } return ( -
+
+ {onExtend ? ( + + + + + + ) : null}
); } diff --git a/frontend/features/chat/components/message/message-meta.tsx b/frontend/features/chat/components/message/message-meta.tsx index e185f19b0..2ebab062c 100644 --- a/frontend/features/chat/components/message/message-meta.tsx +++ b/frontend/features/chat/components/message/message-meta.tsx @@ -1,41 +1,45 @@ "use client"; -import * as React from "react"; import { ArrowDownToLine, ArrowUpFromLine, Brain, + CircleDollarSign, ClockArrowUp, ClockCheck, - CircleDollarSign, - TicketSlash, + Cpu, DatabaseSearch, DatabaseZap, - Cpu, FilePenLine, Forward, + TicketSlash, } from "lucide-react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; import { Brush } from "@/components/animate-ui/icons/brush"; +import { Check } from "@/components/animate-ui/icons/check"; import { ChevronLeft } from "@/components/animate-ui/icons/chevron-left"; import { ChevronRight } from "@/components/animate-ui/icons/chevron-right"; -import { Check } from "@/components/animate-ui/icons/check"; import { Copy } from "@/components/animate-ui/icons/copy"; import { GitFork } from "@/components/animate-ui/icons/git-fork"; import { Heart } from "@/components/animate-ui/icons/heart"; import { RotateCcw } from "@/components/animate-ui/icons/rotate-ccw"; import { ThumbsDown } from "@/components/animate-ui/icons/thumbs-down"; import { ThumbsUp } from "@/components/animate-ui/icons/thumbs-up"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { Input } from "@/components/ui/input"; -import { Button } from "@/components/ui/button"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -import { upsertUserMemory } from "@/shared/api/memory"; -import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; import { resolvePersistedPublicID } from "@/features/chat/model/message-submit"; +import type { ChatBillingCost, ChatMessageBranchNavigator } from "@/features/chat/types/messages"; +import { useLocalizedErrorMessage } from "@/i18n/use-localized-error"; +import { cn } from "@/lib/utils"; +import { upsertUserMemory } from "@/shared/api/memory"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { usePointerInteraction } from "@/shared/hooks/use-pointer-interaction"; +import type { BillingDisplayCurrency, BillingDisplayLabels, BillingDisplayOptions } from "@/shared/lib/billing-display"; import { billingRateMultiplierNote, cacheWriteBillingLabel, @@ -44,10 +48,6 @@ import { formatBillingDisplayPreciseAmountFromUSD, formatBillingDisplayUnitPriceFromUSD, } from "@/shared/lib/billing-display"; -import type { BillingDisplayCurrency, BillingDisplayLabels, BillingDisplayOptions } from "@/shared/lib/billing-display"; -import type { ChatBillingCost, ChatMessageBranchNavigator } from "@/features/chat/types/messages"; -import { usePointerInteraction } from "@/shared/hooks/use-pointer-interaction"; -import { cn } from "@/lib/utils"; export type ChatMetaMessage = { publicID: string; diff --git a/frontend/features/chat/components/sections/chat-area.tsx b/frontend/features/chat/components/sections/chat-area.tsx index ca0e1a061..5f0778ef2 100644 --- a/frontend/features/chat/components/sections/chat-area.tsx +++ b/frontend/features/chat/components/sections/chat-area.tsx @@ -1,31 +1,9 @@ "use client"; -import * as React from "react"; import { ArrowDownToLine, Check } from "lucide-react"; import { useTranslations } from "next-intl"; - -import { ChatLabel } from "@/features/chat/components/sections/chat-label"; -import { useChatMessageFeedback } from "@/features/chat/hooks/use-chat-message-feedback"; -import { - AssistantMessageSkeleton, - ChatInlineAlertCard, - ChatMessageBot, -} from "@/features/chat/components/message/message-bot"; -import { areChatAreaMessagesRenderEqual } from "@/features/chat/model/chat-message-render"; -import { type AssistantReaction } from "@/features/chat/components/message/message-meta"; -import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; -import { ChatMessageUser } from "@/features/chat/components/message/message-user"; -import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; -import type { OpenCodeArtifactInput } from "@/features/chat/model/chat-artifacts"; +import * as React from "react"; import { CenteredEmptyState } from "@/components/ui/empty-state"; -import { Skeleton } from "@/components/ui/skeleton"; -import { ConversationShareExportIconDropdown } from "@/shared/components/conversation-share-export-menu"; -import { ChatScreenshotSelectionBar } from "@/features/chat/components/sections/chat-screenshot-selection-bar"; -import { useCopyAction } from "@/shared/components/copy-action"; -import type { ChatModelOption } from "@/features/chat/types/chat-runtime"; -import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; -import type { FileContentResult } from "@/shared/api/file"; -import type { PreviewDialogFile } from "@/shared/components/file-preview/preview-dialog"; import { MessageScroller, MessageScrollerButton, @@ -35,15 +13,36 @@ import { MessageScrollerViewport, useMessageScroller, } from "@/components/ui/message-scroller"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + AssistantMessageSkeleton, + ChatInlineAlertCard, + ChatMessageBot, +} from "@/features/chat/components/message/message-bot"; +import { type AssistantReaction } from "@/features/chat/components/message/message-meta"; +import { ChatMessageUser } from "@/features/chat/components/message/message-user"; +import { ChatLabel } from "@/features/chat/components/sections/chat-label"; import { ChatMessagePositionRail, chatMessageScrollerID, } from "@/features/chat/components/sections/chat-message-position-rail"; import { ChatResponseOutlineRail } from "@/features/chat/components/sections/chat-response-outline-rail"; +import { ChatScreenshotSelectionBar } from "@/features/chat/components/sections/chat-screenshot-selection-bar"; +import { useChatMessageFeedback } from "@/features/chat/hooks/use-chat-message-feedback"; +import type { OpenCodeArtifactInput } from "@/features/chat/model/chat-artifacts"; +import { areChatAreaMessagesRenderEqual } from "@/features/chat/model/chat-message-render"; +import type { ChatModelOption } from "@/features/chat/types/chat-runtime"; +import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; import { cn } from "@/lib/utils"; +import type { FileContentResult } from "@/shared/api/file"; import { AppLogo, DeeixLogo } from "@/shared/components/app-logo"; -import { useBranding } from "@/shared/config/branding-provider"; +import { ConversationShareExportIconDropdown } from "@/shared/components/conversation-share-export-menu"; +import { useCopyAction } from "@/shared/components/copy-action"; +import type { PreviewDialogFile } from "@/shared/components/file-preview/preview-dialog"; +import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; import { PoweredByDeeix } from "@/shared/components/powered-by-deeix"; +import { useBranding } from "@/shared/config/branding-provider"; +import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; function ScrollToPendingUser({ scrollKey }: { scrollKey: string }) { const handledScrollKeyRef = React.useRef(""); @@ -122,6 +121,7 @@ type ChatAreaProps = { onModelCatalogRefresh?: () => void | Promise; attachmentContentLoader?: (file: PreviewDialogFile) => Promise; onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; + onExtendVideoAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; onOpenCodeArtifact?: (message: ChatAreaMessage, artifact: OpenCodeArtifactInput) => void; onCycleMessageBranch: (parentPublicID: string | null, direction: "previous" | "next") => void; onToggleStar?: () => void | Promise; @@ -286,6 +286,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ onModelCatalogRefresh, attachmentContentLoader, onEditImageAttachment, + onExtendVideoAttachment, onCycleMessageBranch, onReactAssistantMessage, onOpenCodeArtifact, @@ -316,6 +317,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ onModelCatalogRefresh?: () => void | Promise; attachmentContentLoader?: (file: PreviewDialogFile) => Promise; onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; + onExtendVideoAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; onCycleMessageBranch: (parentPublicID: string | null, direction: "previous" | "next") => void; onReactAssistantMessage: (publicID: string, reaction: AssistantReaction) => void; onOpenCodeArtifact?: (message: ChatAreaMessage, artifact: OpenCodeArtifactInput) => void; @@ -358,6 +360,18 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ : undefined, [isAssistant, item, onOpenCodeArtifact], ); + const sourceSupportsVideoExtension = React.useMemo( + () => + Boolean( + item.platformModelName && + modelOptions.some( + (model) => + model.platformModelName === item.platformModelName && + model.videoExtension?.enabled, + ), + ), + [item.platformModelName, modelOptions], + ); const copyKey = item.publicID || item.key; const onCopy = React.useCallback(async () => { @@ -400,6 +414,9 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ copySucceeded={isCopied(copyKey)} attachmentContentLoader={attachmentContentLoader} onEditImageAttachment={onEditImageAttachment} + onExtendVideoAttachment={ + sourceSupportsVideoExtension ? onExtendVideoAttachment : undefined + } artifactActions={artifactActions} markdownRender={markdownRender} showModelInfo={showModelInfo} @@ -447,6 +464,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ previous.onModelCatalogRefresh === next.onModelCatalogRefresh && previous.attachmentContentLoader === next.attachmentContentLoader && previous.onEditImageAttachment === next.onEditImageAttachment && + previous.onExtendVideoAttachment === next.onExtendVideoAttachment && previous.onOpenCodeArtifact === next.onOpenCodeArtifact && areChatAreaMessagesRenderEqual(previous.item, next.item) )); @@ -471,6 +489,7 @@ export function ChatArea({ onModelCatalogRefresh, attachmentContentLoader, onEditImageAttachment, + onExtendVideoAttachment, onOpenCodeArtifact, onCycleMessageBranch, onToggleStar, @@ -509,9 +528,13 @@ export function ChatArea({ const stableOnEditImageAttachment = useStableEvent((attachment: MessageAttachment, sourceModelName?: string) => { onEditImageAttachment?.(attachment, sourceModelName); }); + const stableOnExtendVideoAttachment = useStableEvent((attachment: MessageAttachment, sourceModelName?: string) => { + onExtendVideoAttachment?.(attachment, sourceModelName); + }); const stableOnCycleMessageBranch = useStableEvent(onCycleMessageBranch); const stableOnReactAssistantMessage = useStableEvent(onReactAssistantMessage); const editImageAttachmentHandler = onEditImageAttachment ? stableOnEditImageAttachment : undefined; + const extendVideoAttachmentHandler = onExtendVideoAttachment ? stableOnExtendVideoAttachment : undefined; const shareLabel = shareActive ? t("manageShare") : t("shareConversation"); const shareExportLabel = t("labelMenu.shareAndExport"); const tScreenshot = useTranslations("chat.screenshot"); @@ -643,6 +666,7 @@ export function ChatArea({ onModelCatalogRefresh={onModelCatalogRefresh ? stableOnModelCatalogRefresh : undefined} attachmentContentLoader={attachmentContentLoader} onEditImageAttachment={editImageAttachmentHandler} + onExtendVideoAttachment={extendVideoAttachmentHandler} onCycleMessageBranch={stableOnCycleMessageBranch} onReactAssistantMessage={stableOnReactAssistantMessage} onOpenCodeArtifact={onOpenCodeArtifact} diff --git a/frontend/features/chat/components/sections/chat-input.tsx b/frontend/features/chat/components/sections/chat-input.tsx index 3370496d3..b9c15cdee 100644 --- a/frontend/features/chat/components/sections/chat-input.tsx +++ b/frontend/features/chat/components/sections/chat-input.tsx @@ -1,46 +1,19 @@ "use client"; -import * as React from "react"; -import dynamic from "next/dynamic"; import { Box, CornerDownRight, Eye, EyeOff, Film, Image, ImageOff, ImagePlus, LoaderCircle, PencilLine, Trash2 } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; +import dynamic from "next/dynamic"; import { useLocale, useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; import { AudioLines } from "@/components/animate-ui/icons/audio-lines"; import { Blocks } from "@/components/animate-ui/icons/blocks"; +import { Crop } from "@/components/animate-ui/icons/crop"; +import { Link as LinkIcon } from "@/components/animate-ui/icons/link"; import { Pause } from "@/components/animate-ui/icons/pause"; import { Send } from "@/components/animate-ui/icons/send"; -import { Link as LinkIcon } from "@/components/animate-ui/icons/link"; -import { Crop } from "@/components/animate-ui/icons/crop"; import { X as XIcon } from "@/components/animate-ui/icons/x"; -import { PlusIcon } from "@/components/ui/plus"; -import type { - ChatModelOption, - PendingAttachment, - UploadingAttachment, -} from "@/features/chat/types/chat-runtime"; -import { - formatClipboardMarkdownPaste, - resolveClipboardMarkdownPaste, -} from "@/features/chat/utils/markdown-paste"; -import { - useChatSpeechInput, - type SpeechInputErrorCode, -} from "@/features/chat/hooks/use-chat-speech-input"; -import { useMarkdownPreviewSync } from "@/features/chat/hooks/use-markdown-preview-sync"; -import { - useChatMentionMenu, - type ChatMentionMenuKind, -} from "@/features/chat/hooks/use-chat-mention-menu"; -import { ChatMentionMenuPortal } from "@/features/chat/components/shared/chat-mention-menu"; -import { ChatMCP } from "@/features/chat/components/sections/chat-mcp"; -import { ChatKnowledgeBases } from "@/features/chat/components/sections/chat-knowledge-bases"; -import { ChatModelPicker } from "@/features/chat/components/sections/chat-model-picker"; -import { ChatModelConfig } from "@/features/chat/components/sections/chat-model-config"; -import { formatBytes, resolveFileExtension, resolveFileIcon } from "@/shared/lib/file-display"; -import type { ChatSubmitDecision } from "@/features/chat/model/chat-task"; -import { isMediaSubmitTask, resolveChatSubmitDecision } from "@/features/chat/model/chat-task"; import { Attachment, AttachmentAction, @@ -64,19 +37,46 @@ import { InputGroupButton, InputGroupTextarea, } from "@/components/ui/input-group"; +import { PlusIcon } from "@/components/ui/plus"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { resolveFileProcessingBadge } from "@/shared/lib/file-processing"; -import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; -import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; +import { ChatKnowledgeBases } from "@/features/chat/components/sections/chat-knowledge-bases"; +import { ChatMCP } from "@/features/chat/components/sections/chat-mcp"; +import { ChatModelConfig } from "@/features/chat/components/sections/chat-model-config"; +import { ChatModelPicker } from "@/features/chat/components/sections/chat-model-picker"; +import { ChatMentionMenuPortal } from "@/features/chat/components/shared/chat-mention-menu"; +import { + type ChatMentionMenuKind, + useChatMentionMenu, +} from "@/features/chat/hooks/use-chat-mention-menu"; +import { + type SpeechInputErrorCode, + useChatSpeechInput, +} from "@/features/chat/hooks/use-chat-speech-input"; +import { useMarkdownPreviewSync } from "@/features/chat/hooks/use-markdown-preview-sync"; +import type { ChatSubmitDecision } from "@/features/chat/model/chat-task"; +import { isMediaSubmitTask, resolveChatSubmitDecision } from "@/features/chat/model/chat-task"; +import type { + ChatModelOption, + PendingAttachment, + UploadingAttachment, +} from "@/features/chat/types/chat-runtime"; +import { + formatClipboardMarkdownPaste, + resolveClipboardMarkdownPaste, +} from "@/features/chat/utils/markdown-paste"; +import type { SendShortcut } from "@/features/settings/types/settings"; import { cn } from "@/lib/utils"; import type { ConversationOptions } from "@/shared/api/conversation.types"; import type { FileObjectDTO } from "@/shared/api/file.types"; import type { MCPToolDTO } from "@/shared/api/mcp.types"; import type { SkillSummaryDTO } from "@/shared/api/skills.types"; +import { StreamdownRender } from "@/shared/components/markdown/streamdown-render"; +import { useDialogSnapshot } from "@/shared/hooks/use-dialog-snapshot"; +import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; +import { formatBytes, resolveFileExtension, resolveFileIcon } from "@/shared/lib/file-display"; +import { resolveFileProcessingBadge } from "@/shared/lib/file-processing"; import type { ModelOptionPolicy } from "@/shared/lib/model-option-policy"; -import type { SendShortcut } from "@/features/settings/types/settings"; import { isSendShortcutEvent } from "@/shared/lib/platform-shortcuts"; -import type { BillingDisplayCurrency } from "@/shared/lib/billing-display"; const FilePreviewDialog = dynamic( () => import("@/shared/components/file-preview/preview-dialog").then((module) => module.FilePreviewDialog), @@ -202,6 +202,17 @@ function resolveComposerModeIndicator( tone: "default", }; } + if (decision.task === "video_extension") { + return { + label: t("mediaMode.videoExtension"), + intro: t("mediaMode.videoExtensionIntro"), + description: decision.blockedReason + ? t(`mediaMode.blockedDescriptions.${decision.blockedReason}`) + : t("mediaMode.videoExtensionDescription"), + icon: Film, + tone: decision.blockedReason ? "warning" : "default", + }; + } return null; } @@ -418,6 +429,17 @@ function ChatInputComponent({ const isMediaMode = isMediaSubmitTask(submitTask); const composerModeIndicator = resolveComposerModeIndicator(submitDecision, tComposer); const ComposerModeIcon = composerModeIndicator?.icon; + const taskOptionConfig = submitTask === "video_extension" ? selectedModel?.videoExtension : null; + const modelConfigOptions = React.useMemo(() => { + if (!taskOptionConfig) { + return options; + } + const duration = Number(options.duration); + return { + ...options, + duration: Number.isInteger(duration) && duration >= 2 && duration <= 10 ? duration : 6, + }; + }, [options, taskOptionConfig]); const modelOptionPolicyDisabled = modelOptionPolicy?.mode?.trim() === "disabled"; const showMCPToolsButton = availableTools.length > 0 && !isMediaMode; const showHTMLVisualPromptButton = !isMediaMode; @@ -950,10 +972,10 @@ function ChatInputComponent({ {!modelOptionPolicyDisabled ? ( = { xai_image: "Images Generations", xai_image_edits: "Images Edits", xai_video: "Video Generations", + xai_video_extensions: "Video Extensions", xai_responses: "xAI Responses", }; diff --git a/frontend/features/chat/hooks/use-chat-message-submit.ts b/frontend/features/chat/hooks/use-chat-message-submit.ts index 45f11c417..4f9bcf1be 100644 --- a/frontend/features/chat/hooks/use-chat-message-submit.ts +++ b/frontend/features/chat/hooks/use-chat-message-submit.ts @@ -44,6 +44,7 @@ import { streamMessage as streamConversationMessage, streamImageEdit, streamImageGeneration, + streamVideoExtension, streamVideoGeneration, updateMessage, } from "@/shared/api/conversation"; @@ -51,6 +52,7 @@ import type { ConversationDTO, ConversationOptions, MediaImageRequest, + MediaVideoExtensionRequest, MediaVideoRequest, MessageDTO, SendMessageRequest, @@ -96,6 +98,13 @@ function resolveImageLoadingAspectRatio(options: ConversationOptions): ImageLoad return "square"; } +function resolveVideoExtensionOptions(options: ConversationOptions): ConversationOptions { + const duration = Number(options.duration); + return { + duration: Number.isInteger(duration) && duration >= 2 && duration <= 10 ? duration : 6, + }; +} + function streamEventErrorToApiError( event: Extract, fallback: string, @@ -840,7 +849,7 @@ export function useChatMessageSubmit({ ? resolveImageLoadingAspectRatio(sanitizedOptions) : undefined; const assistantContentType = - submitTask === "chat" ? "markdown" : submitTask === "video_generation" ? "video" : "image"; + submitTask === "chat" ? "markdown" : submitTask === "video_generation" || submitTask === "video_extension" ? "video" : "image"; let targetConversationID = queuedSubmission?.conversationPublicID ?? conversationIDRef.current; let targetConversation = queuedSubmission?.conversation ?? activeConversationRef.current; let metadataRefreshInFlight = false; @@ -1033,9 +1042,12 @@ export function useChatMessageSubmit({ } touchByPublicID(targetConversationID, { title: optimisticTitle }); } + const effectiveOptions = submitTask === "video_extension" + ? resolveVideoExtensionOptions(sanitizedOptions) + : sanitizedOptions; const commonStreamPayload = { model: requestPlatformModelName, - options: Object.keys(sanitizedOptions).length > 0 ? sanitizedOptions : undefined, + options: Object.keys(effectiveOptions).length > 0 ? effectiveOptions : undefined, clientRunID: clientRunID, fileIDs: effectiveAttachments.length > 0 ? effectiveAttachments.map((item) => item.fileID) : undefined, parentMessagePublicID: resolvedParentPublicID || undefined, @@ -1180,6 +1192,22 @@ export function useChatMessageSubmit({ prompt: payloadContent, }; completed = await streamVideoGeneration(token, targetConversationID, mediaPayload, streamOptions); + } else if (submitTask === "video_extension") { + const sourceVideoFileID = effectiveAttachments[0]?.fileID; + if (!sourceVideoFileID) { + throw new Error("video extension source is missing"); + } + const mediaPayload: MediaVideoExtensionRequest = { + model: commonStreamPayload.model, + options: commonStreamPayload.options, + clientRunID: commonStreamPayload.clientRunID, + parentMessagePublicID: commonStreamPayload.parentMessagePublicID, + sourceMessagePublicID: commonStreamPayload.sourceMessagePublicID, + branchReason: commonStreamPayload.branchReason, + prompt: payloadContent, + sourceVideoFileID, + }; + completed = await streamVideoExtension(token, targetConversationID, mediaPayload, streamOptions); } else { const mediaPayload: MediaImageRequest = { ...commonStreamPayload, diff --git a/frontend/features/chat/hooks/use-chat-model-options.ts b/frontend/features/chat/hooks/use-chat-model-options.ts index 71b3247df..ab2aeaea2 100644 --- a/frontend/features/chat/hooks/use-chat-model-options.ts +++ b/frontend/features/chat/hooks/use-chat-model-options.ts @@ -1,39 +1,38 @@ "use client"; -import * as React from "react"; import { useTranslations } from "next-intl"; - +import * as React from "react"; +import { sanitizeConversationOptions } from "@/features/chat/model/conversation-options"; import type { ChatModelOption, ModelOptionControl, ModelOptionControlType, } from "@/features/chat/types/chat-runtime"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -import { parseProtocolsJSON } from "@/shared/lib/model-protocols"; -import { sanitizeConversationOptions } from "@/features/chat/model/conversation-options"; -import { - DEFAULT_CHAT_CONTENT_WIDTH, - parseChatContentWidth, - type ChatContentWidth, -} from "@/shared/model/chat-content-width"; +import { USER_SETTINGS_UPDATED_EVENT } from "@/features/settings/events/user-settings-events"; +import type { SendShortcut } from "@/features/settings/types/settings"; +import { parseSendShortcut } from "@/features/settings/utils/chat-settings"; +import { getBillingConfig } from "@/shared/api/billing"; import { listConversationRuns } from "@/shared/api/conversation"; +import type { ConversationOptions } from "@/shared/api/conversation.types"; import { listPublicModels } from "@/shared/api/model"; -import { getBillingConfig } from "@/shared/api/billing"; +import type { PublicModelDTO } from "@/shared/api/model.types"; import { getMCPPolicy, getModelOptionPolicy } from "@/shared/api/settings"; import { getUserSettings } from "@/shared/api/user-settings"; -import type { PublicModelDTO } from "@/shared/api/model.types"; -import type { ModelNativeToolConfig, ModelOptionPolicy } from "@/shared/lib/model-option-policy"; -import { nativeToolDefinitionVariantsFromConfig, nativeToolPayloadSignature } from "@/shared/lib/native-tool-payload"; -import { parseKindsJSON } from "@/shared/model/llm-schema"; -import { resolveConversationDefaultModel } from "@/shared/model/conversation-default-model"; -import type { ConversationOptions } from "@/shared/api/conversation.types"; -import type { SendShortcut } from "@/features/settings/types/settings"; -import { parseSendShortcut } from "@/features/settings/utils/chat-settings"; -import { USER_SETTINGS_UPDATED_EVENT } from "@/features/settings/events/user-settings-events"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import { - normalizeBillingDisplayCurrency, type BillingDisplayCurrency, + normalizeBillingDisplayCurrency, } from "@/shared/lib/billing-display"; +import type { ModelNativeToolConfig, ModelOptionPolicy } from "@/shared/lib/model-option-policy"; +import { parseProtocolsJSON } from "@/shared/lib/model-protocols"; +import { nativeToolDefinitionVariantsFromConfig, nativeToolPayloadSignature } from "@/shared/lib/native-tool-payload"; +import { + type ChatContentWidth, + DEFAULT_CHAT_CONTENT_WIDTH, + parseChatContentWidth, +} from "@/shared/model/chat-content-width"; +import { resolveConversationDefaultModel } from "@/shared/model/conversation-default-model"; +import { parseKindsJSON } from "@/shared/model/llm-schema"; type ModelCatalogRefreshResult = { models: PublicModelDTO[]; @@ -308,6 +307,39 @@ function resolveOptionControls(raw: string): ModelOptionControl[] { return controls.filter((item, index) => controls.findIndex((candidate) => candidate.path === item.path) === index); } +function resolveVideoExtensionConfig(raw: string, protocols: string[]): ChatModelOption["videoExtension"] { + const parsed = parseJSONObject(raw); + const mediaTasks = parsed?.mediaTasks; + const taskSource = mediaTasks && typeof mediaTasks === "object" && !Array.isArray(mediaTasks) + ? (mediaTasks as Record).video_extension + : undefined; + const task = taskSource && typeof taskSource === "object" && !Array.isArray(taskSource) + ? (taskSource as Record) + : null; + const protocolSupported = protocols.includes("xai_video_extensions"); + if (!protocolSupported || task?.enabled === false) { + return null; + } + const defaultOptions = task?.defaultOptions && typeof task.defaultOptions === "object" && !Array.isArray(task.defaultOptions) + ? sanitizeConversationOptions(task.defaultOptions as ConversationOptions) + : { duration: 6 }; + const rawControls = Array.isArray(task?.optionControls) ? task.optionControls : [{ path: "duration", type: "select", label: "Duration", description: "2–10 seconds", options: ["2", "3", "4", "5", "6", "7", "8", "9", "10"] }]; + const controls = rawControls.flatMap((item): ModelOptionControl[] => { + if (!item || typeof item !== "object" || Array.isArray(item)) return []; + const source = item as Record; + const path = normalizeOptionControlPath(source.path); + if (path !== "duration") return []; + return [{ + path, + type: normalizeOptionControlType(source.type) ?? "select", + label: normalizeOptionControlString(source.label), + description: normalizeOptionControlString(source.description), + options: normalizeOptionControlOptions(source.options) ?? ["2", "3", "4", "5", "6", "7", "8", "9", "10"], + }]; + }); + return { enabled: true, defaultOptions, optionControls: controls }; +} + function resolveNativeToolKeys(raw: string): string[] { const parsed = parseJSONObject(raw); const rawKeys = parsed?.nativeToolKeys; @@ -354,6 +386,7 @@ function toChatModelOption( nativeToolKeys: resolveNativeToolKeys(item.capabilitiesJSON), nativeTools, pricing: item.pricing, + videoExtension: resolveVideoExtensionConfig(item.capabilitiesJSON, protocols), }; } diff --git a/frontend/features/chat/model/chat-task.ts b/frontend/features/chat/model/chat-task.ts index cf48c56bf..773e85d9f 100644 --- a/frontend/features/chat/model/chat-task.ts +++ b/frontend/features/chat/model/chat-task.ts @@ -1,7 +1,7 @@ import type { ChatModelOption, PendingAttachment } from "@/features/chat/types/chat-runtime"; import type { ConversationOptions } from "@/shared/api/conversation.types"; -export type ChatSubmitTask = "chat" | "image_generation" | "image_edit" | "video_generation"; +export type ChatSubmitTask = "chat" | "image_generation" | "image_edit" | "video_generation" | "video_extension"; export type ChatSubmitBlockReason = | "image_edit_input_required" | "image_edit_unsupported" @@ -9,6 +9,8 @@ export type ChatSubmitBlockReason = | "image_task_rejects_non_image_attachments" | "video_generation_too_many_images" | "video_task_rejects_non_image_attachments" + | "video_extension_requires_single_mp4" + | "video_extension_unsupported" | "model_task_unsupported"; export type ChatSubmitDecision = { @@ -21,12 +23,21 @@ export type ChatSubmitDecision = { supportsImageGeneration: boolean; supportsImageEdit: boolean; supportsVideoGeneration: boolean; + supportsVideoExtension: boolean; }; function isImageAttachment(item: PendingAttachment): boolean { return item.fileCategory === "image" || item.mimeType.toLowerCase().startsWith("image/"); } +function isMP4Attachment(item: PendingAttachment): boolean { + return item.mimeType.toLowerCase() === "video/mp4" || item.detectedMime?.toLowerCase() === "video/mp4"; +} + +function isVideoAttachment(item: PendingAttachment): boolean { + return item.mimeType.toLowerCase().startsWith("video/") || item.detectedMime?.toLowerCase().startsWith("video/") === true; +} + function buildDecision( task: ChatSubmitTask, blockedReason: ChatSubmitBlockReason | null, @@ -75,11 +86,14 @@ export function resolveChatSubmitDecision( const kinds = new Set(model?.kinds ?? []); const attachmentCount = attachments.length; const imageAttachmentCount = attachments.filter(isImageAttachment).length; + const mp4AttachmentCount = attachments.filter(isMP4Attachment).length; + const videoAttachmentCount = attachments.filter(isVideoAttachment).length; const nonImageAttachmentCount = attachmentCount - imageAttachmentCount; const supportsChat = kinds.size === 0 || kinds.has("chat") || kinds.has("audio"); const supportsImageGeneration = kinds.has("image_gen"); const supportsImageEdit = kinds.has("image_edit"); const supportsVideoGeneration = kinds.has("video_gen"); + const supportsVideoExtension = model?.videoExtension?.enabled === true; const baseDecision = { attachmentCount, imageAttachmentCount, @@ -88,9 +102,20 @@ export function resolveChatSubmitDecision( supportsImageGeneration, supportsImageEdit, supportsVideoGeneration, + supportsVideoExtension, }; const requestedType = requestedResponseType(options); + if (videoAttachmentCount > 0) { + if (attachmentCount !== 1 || mp4AttachmentCount !== 1) { + return buildDecision("video_extension", "video_extension_requires_single_mp4", baseDecision); + } + if (!supportsVideoExtension) { + return buildDecision("video_extension", "video_extension_unsupported", baseDecision); + } + return buildDecision("video_extension", null, baseDecision); + } + if ( nonImageAttachmentCount > 0 && (supportsImageGeneration || supportsImageEdit || supportsVideoGeneration) && @@ -169,5 +194,5 @@ export function resolveChatSubmitDecision( } export function isMediaSubmitTask(task: ChatSubmitTask): boolean { - return task === "image_generation" || task === "image_edit" || task === "video_generation"; + return task === "image_generation" || task === "image_edit" || task === "video_generation" || task === "video_extension"; } diff --git a/frontend/features/chat/types/chat-runtime.ts b/frontend/features/chat/types/chat-runtime.ts index 5a9bde6fb..aee9820c1 100644 --- a/frontend/features/chat/types/chat-runtime.ts +++ b/frontend/features/chat/types/chat-runtime.ts @@ -1,8 +1,8 @@ import type { ChatInlineAlert, + ChatMessageProcessTrace, ImageLoadingAspectRatio, MessageAttachment, - ChatMessageProcessTrace, } from "@/features/chat/types/messages"; import type { ConversationOptions } from "@/shared/api/conversation.types"; import type { PublicModelPricingDTO } from "@/shared/api/model.types"; @@ -30,6 +30,13 @@ export type ChatModelOption = { nativeToolKeys: string[]; nativeTools: ModelNativeToolConfig[]; pricing: PublicModelPricingDTO | null; + videoExtension: ModelMediaTaskConfig | null; +}; + +export type ModelMediaTaskConfig = { + enabled: boolean; + defaultOptions: ConversationOptions; + optionControls: ModelOptionControl[]; }; export type ModelOptionControlType = "boolean" | "number" | "select" | "text"; diff --git a/frontend/i18n/messages/en-US/admin-billing.json b/frontend/i18n/messages/en-US/admin-billing.json index 1d83bade9..289549373 100644 --- a/frontend/i18n/messages/en-US/admin-billing.json +++ b/frontend/i18n/messages/en-US/admin-billing.json @@ -335,7 +335,7 @@ "duplicateModel": "{model} appears more than once", "pricingObject": "{model} pricing must be an object", "invalidPricingMode": "{model}.pricingMode must be token, call, duration, or tiered", - "durationVideoOnly": "{model}.pricingMode=duration requires the video_gen capability", + "durationVideoOnly": "{model}.pricingMode=duration requires a video model capability", "invalidNumber": "{model}.{field} must be a number greater than or equal to 0", "invalidTieredPricing": "{model}.{field} must contain a non-empty tiers array", "invalidTieredPricingJSON": "{model}.tieredPricingJSON is not valid JSON" diff --git a/frontend/i18n/messages/en-US/admin-models.json b/frontend/i18n/messages/en-US/admin-models.json index 8e42ab95a..30c008c3d 100644 --- a/frontend/i18n/messages/en-US/admin-models.json +++ b/frontend/i18n/messages/en-US/admin-models.json @@ -39,7 +39,8 @@ "audio": "Audio", "image_gen": "Image generation", "image_edit": "Image editing", - "video_gen": "Video generation" + "video_gen": "Video generation", + "video_extension": "Video extension" }, "table": { "searchPlaceholder": "Search model, vendor, group, or description", diff --git a/frontend/i18n/messages/en-US/admin-upstreams.json b/frontend/i18n/messages/en-US/admin-upstreams.json index 0cc0c76dc..ab8864697 100644 --- a/frontend/i18n/messages/en-US/admin-upstreams.json +++ b/frontend/i18n/messages/en-US/admin-upstreams.json @@ -13,7 +13,8 @@ "audio": "Audio", "image_gen": "Image generation", "image_edit": "Image editing", - "video_gen": "Video generation" + "video_gen": "Video generation", + "video_extension": "Video extension" }, "status": { "active": "Active", diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index c7224c7f3..3b130d329 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -214,6 +214,7 @@ "dislikeReply": "Dislike", "retryReply": "Retry", "continueReply": "Continue reply", + "extendVideo": "Extend video", "forkMessage": "Fork conversation here", "forkTitle": "{title} (Fork)", "rememberPreference": "Remember preference", @@ -285,6 +286,8 @@ "image_task_rejects_non_image_attachments": "Image tasks only support image attachments. Remove non-image files.", "video_generation_too_many_images": "Video generation supports one input image at most. Keep one image and try again.", "video_task_rejects_non_image_attachments": "Video generation only supports text prompts and image attachments. Remove non-image files.", + "video_extension_requires_single_mp4": "Video extension accepts exactly one MP4 file. Remove the other attachments and try again.", + "video_extension_unsupported": "The selected model does not support video extension. Switch to a model using the xAI video protocol.", "model_task_unsupported": "This model cannot process the current input. Select another model and try again." }, "mediaStatus": { @@ -502,6 +505,9 @@ "videoGeneration": "Video generation", "videoGenerationIntro": "Video generation mode is active.", "videoGenerationDescription": "Creates a video from the text prompt. Upload one image to use it as a reference input.", + "videoExtension": "Video extension", + "videoExtensionIntro": "Video extension mode is active.", + "videoExtensionDescription": "Extends the uploaded MP4 video from your prompt. Only the extension duration is configurable.", "invalidFile": "Invalid file detected", "invalidFileIntro": "This file cannot be used for a media task.", "blockedDescriptions": { @@ -511,6 +517,8 @@ "image_task_rejects_non_image_attachments": "Image tasks only support image attachments. Remove non-image files.", "video_generation_too_many_images": "Video generation supports one input image at most.", "video_task_rejects_non_image_attachments": "Video generation only supports text prompts and image attachments. Remove non-image files.", + "video_extension_requires_single_mp4": "Video extension accepts exactly one MP4 file. Remove the other attachments.", + "video_extension_unsupported": "The selected model does not support video extension. Switch to a model using the xAI video protocol.", "model_task_unsupported": "The selected model cannot process this input. Switch models and try again." } }, diff --git a/frontend/i18n/messages/zh-CN/admin-billing.json b/frontend/i18n/messages/zh-CN/admin-billing.json index a63d56001..af9003b17 100644 --- a/frontend/i18n/messages/zh-CN/admin-billing.json +++ b/frontend/i18n/messages/zh-CN/admin-billing.json @@ -335,7 +335,7 @@ "duplicateModel": "{model} 重复出现", "pricingObject": "{model} 的定价配置必须是对象", "invalidPricingMode": "{model}.pricingMode 必须是 token、call、duration 或 tiered", - "durationVideoOnly": "{model}.pricingMode=duration 需要 video_gen 能力", + "durationVideoOnly": "{model}.pricingMode=duration 需要视频模型能力", "invalidNumber": "{model}.{field} 必须是大于等于 0 的数字", "invalidTieredPricing": "{model}.{field} 需要包含非空 tiers 数组", "invalidTieredPricingJSON": "{model}.tieredPricingJSON 不是合法 JSON" diff --git a/frontend/i18n/messages/zh-CN/admin-models.json b/frontend/i18n/messages/zh-CN/admin-models.json index 18cc16096..d0e574547 100644 --- a/frontend/i18n/messages/zh-CN/admin-models.json +++ b/frontend/i18n/messages/zh-CN/admin-models.json @@ -39,7 +39,8 @@ "audio": "语音", "image_gen": "图像生成", "image_edit": "图像编辑", - "video_gen": "视频生成" + "video_gen": "视频生成", + "video_extension": "视频扩展" }, "table": { "searchPlaceholder": "搜索模型、厂商、分组或描述", diff --git a/frontend/i18n/messages/zh-CN/admin-upstreams.json b/frontend/i18n/messages/zh-CN/admin-upstreams.json index a860000f3..11fd33c9b 100644 --- a/frontend/i18n/messages/zh-CN/admin-upstreams.json +++ b/frontend/i18n/messages/zh-CN/admin-upstreams.json @@ -13,7 +13,8 @@ "audio": "语音", "image_gen": "图像生成", "image_edit": "图像编辑", - "video_gen": "视频生成" + "video_gen": "视频生成", + "video_extension": "视频扩展" }, "status": { "active": "正常", diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index 9437c614e..c304a6059 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -214,6 +214,7 @@ "dislikeReply": "点踩", "retryReply": "重试", "continueReply": "继续回复", + "extendVideo": "扩展视频", "forkMessage": "分支到新聊天", "forkTitle": "{title}（分支）", "rememberPreference": "记住偏好", @@ -285,6 +286,8 @@ "image_task_rejects_non_image_attachments": "图像任务仅支持图片附件，请移除非图片文件。", "video_generation_too_many_images": "视频生成最多支持一张输入图片，请保留一张图片后重试。", "video_task_rejects_non_image_attachments": "视频生成仅支持文字提示和图片附件，请移除非图片文件。", + "video_extension_requires_single_mp4": "视频扩展仅支持一个 MP4 文件，请移除其他附件后重试。", + "video_extension_unsupported": "当前模型不支持视频扩展，请切换至支持 xAI 视频协议的模型。", "model_task_unsupported": "该模型无法处理当前输入，请切换模型后重试。" }, "mediaStatus": { @@ -502,6 +505,9 @@ "videoGeneration": "视频生成", "videoGenerationIntro": "当前为视频生成模式。", "videoGenerationDescription": "将根据文字提示生成视频；上传一张图片后会作为视频参考输入。", + "videoExtension": "视频扩展", + "videoExtensionIntro": "当前为视频扩展模式。", + "videoExtensionDescription": "将根据文字提示扩展已上传的 MP4 视频，仅需设置扩展时长。", "invalidFile": "监测到无效文件", "invalidFileIntro": "当前文件无法用于媒体任务。", "blockedDescriptions": { @@ -511,6 +517,8 @@ "image_task_rejects_non_image_attachments": "图像任务仅支持图片附件，请移除非图片文件。", "video_generation_too_many_images": "视频生成最多支持一张输入图片。", "video_task_rejects_non_image_attachments": "视频生成仅支持文字提示和图片附件，请移除非图片文件。", + "video_extension_requires_single_mp4": "视频扩展仅支持一个 MP4 文件，请移除其他附件。", + "video_extension_unsupported": "当前模型不支持视频扩展，请切换至支持 xAI 视频协议的模型。", "model_task_unsupported": "当前模型不支持此输入，请切换模型后重试。" } }, diff --git a/frontend/shared/api/conversation.ts b/frontend/shared/api/conversation.ts index 117d133c6..dbf862cd9 100644 --- a/frontend/shared/api/conversation.ts +++ b/frontend/shared/api/conversation.ts @@ -28,6 +28,7 @@ import type { CreateConversationShareRequest, DeleteConversationData, MediaImageRequest, + MediaVideoExtensionRequest, MediaVideoRequest, MessageDTO, MessageFeedbackResult, @@ -1175,3 +1176,18 @@ export async function streamVideoGeneration( options, ); } + +export async function streamVideoExtension( + accessToken: string, + conversationPublicID: string, + payload: MediaVideoExtensionRequest, + options: ConversationStreamOptions = {}, +): Promise { + return postConversationStream( + accessToken, + conversationPublicID, + "/media/videos/extensions/stream", + payload, + options, + ); +} diff --git a/frontend/shared/api/conversation.types.ts b/frontend/shared/api/conversation.types.ts index 3ad414eff..ba1133859 100644 --- a/frontend/shared/api/conversation.types.ts +++ b/frontend/shared/api/conversation.types.ts @@ -5,6 +5,7 @@ import type { CreateConversationProjectRequest as ContractCreateConversationProjectRequest, CreateConversationRequest as ContractCreateConversationRequest, CreateConversationShareRequest as ContractCreateConversationShareRequest, + MediaVideoExtensionRequest as ContractMediaVideoExtensionRequest, RenameConversationRequest as ContractRenameConversationRequest, ReorderConversationProjectsRequest as ContractReorderConversationProjectsRequest, RevokeConversationSharesRequest as ContractRevokeConversationSharesRequest, @@ -220,6 +221,10 @@ export type MediaVideoRequest = { branchReason?: "default" | "retry" | "edit"; }; +export type MediaVideoExtensionRequest = Omit & { + options?: ConversationOptions; +}; + export type SendMessageResult = Omit & { userMessage: MessageDTO; assistantMessage: MessageDTO; diff --git a/frontend/shared/components/file-preview/preview-media.tsx b/frontend/shared/components/file-preview/preview-media.tsx index e8232a506..217fc1045 100644 --- a/frontend/shared/components/file-preview/preview-media.tsx +++ b/frontend/shared/components/file-preview/preview-media.tsx @@ -1,10 +1,10 @@ "use client"; -import * as React from "react"; -import { createPortal } from "react-dom"; import { FileAudio2, Maximize2, Minimize2, Minus, Pause, Play, Plus } from "lucide-react"; import Image from "next/image"; import { useTranslations } from "next-intl"; +import * as React from "react"; +import { createPortal } from "react-dom"; import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; @@ -542,13 +542,20 @@ export function PreviewMedia({ className={cn( "relative max-w-full", !inline && "mx-auto", - videoIsFullscreen ? "flex h-full w-full items-center justify-center bg-neutral-950 p-6" : "w-full", + videoIsFullscreen + ? "flex h-screen w-screen max-w-none items-center justify-center bg-neutral-950" + : "w-full", )} >
void togglePlay()} onDoubleClick={() => void toggleVideoFullscreen()} @@ -581,35 +592,59 @@ export function PreviewMedia({ variant="ghost" size="icon" aria-label={tPreview("playVideo")} - className="absolute left-1/2 top-1/2 z-20 size-12 -translate-x-1/2 -translate-y-1/2 rounded-full bg-neutral-950/75 text-neutral-50 backdrop-blur-md hover:bg-neutral-950/90 hover:text-neutral-50" + className={cn( + "absolute left-1/2 top-1/2 z-20 -translate-x-1/2 -translate-y-1/2 rounded-full text-neutral-50 backdrop-blur-md hover:text-neutral-50", + inline + ? "size-11 border border-white/15 bg-neutral-950/55 shadow-lg shadow-black/20 hover:bg-neutral-950/70" + : "size-12 bg-neutral-950/75 hover:bg-neutral-950/90", + )} onClick={() => void togglePlay()} > - + ) : null}
-
-
+
+
{formatTime(currentTime)} -
+
handleSeek(event.target.value)} />
@@ -628,7 +663,10 @@ export function PreviewMedia({ variant="ghost" size="icon" aria-label={videoIsFullscreen ? tPreview("exitFullscreen") : tPreview("enterFullscreen")} - className="size-7 shrink-0 rounded-full text-neutral-300 hover:bg-neutral-50/10 hover:text-neutral-50" + className={cn( + "shrink-0 rounded-full text-neutral-300 hover:bg-neutral-50/10 hover:text-neutral-50", + inline ? "size-6" : "size-7", + )} onClick={() => void toggleVideoFullscreen()} > {videoIsFullscreen ? : } diff --git a/frontend/shared/components/markdown/streamdown-components.tsx b/frontend/shared/components/markdown/streamdown-components.tsx index e354bc1b8..bc485b40e 100644 --- a/frontend/shared/components/markdown/streamdown-components.tsx +++ b/frontend/shared/components/markdown/streamdown-components.tsx @@ -1,8 +1,8 @@ "use client"; -import * as React from "react"; import { CornerUpLeft, Download, Eye, Maximize2, WandSparkles } from "lucide-react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { ChevronDown } from "@/components/animate-ui/icons/chevron-down"; import { ChevronUp } from "@/components/animate-ui/icons/chevron-up"; @@ -10,6 +10,13 @@ import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/utils"; +import { CopyActionButton } from "@/shared/components/copy-action"; +import { MediaActionBar, MediaActionButton } from "@/shared/components/media-action-bar"; +import { + type ArtifactPreviewKind, + resolveArtifactPreviewKind, +} from "@/shared/lib/artifact-preview"; import { downloadMarkdownImageSource, loadProtectedMarkdownImageBlobURL, @@ -17,12 +24,6 @@ import { resolveMarkdownImageSource, resolveProtectedMarkdownImageSource, } from "@/shared/lib/markdown-image-source"; -import { - resolveArtifactPreviewKind, - type ArtifactPreviewKind, -} from "@/shared/lib/artifact-preview"; -import { CopyActionButton } from "@/shared/components/copy-action"; -import { cn } from "@/lib/utils"; import { sanitizeHTMLStyle } from "./streamdown-style"; const CODE_BLOCK_COLLAPSE_LINE_THRESHOLD = 16; @@ -708,54 +709,24 @@ export function MarkdownImage({ alt, className, onError, onLoad, src, ...props } /> )} {canUseImageActions ? ( - - - - - - {t("previewImage")} - + setPreviewOpen(true)}> + + {canEditImage ? ( - - - - - {t("editImage")} - + imageActions?.onEditImage?.(src)}> + + ) : null} - - - - - {t("downloadImage")} - - + void handleDownload()}> + + + ) : null} diff --git a/frontend/shared/components/media-action-bar.tsx b/frontend/shared/components/media-action-bar.tsx new file mode 100644 index 000000000..abe35a8b2 --- /dev/null +++ b/frontend/shared/components/media-action-bar.tsx @@ -0,0 +1,54 @@ +"use client"; + +import type { ButtonHTMLAttributes, HTMLAttributes, ReactNode } from "react"; + +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/utils"; + +export function MediaActionBar({ + children, + className, + ...props +}: HTMLAttributes) { + return ( + + {children} + + ); +} + +export function MediaActionButton({ + children, + className, + label, + type = "button", + ...props +}: Omit, "aria-label"> & { + children: ReactNode; + label: string; +}) { + return ( + + + + + {label} + + ); +} diff --git a/frontend/shared/lib/model-option-policy.ts b/frontend/shared/lib/model-option-policy.ts index 9201c84a4..c0078bcee 100644 --- a/frontend/shared/lib/model-option-policy.ts +++ b/frontend/shared/lib/model-option-policy.ts @@ -14,6 +14,7 @@ export const MODEL_OPTION_POLICY_PROTOCOLS = [ "xai_image", "xai_image_edits", "xai_video", + "xai_video_extensions", ] as const; export type ModelOptionPolicyProtocol = (typeof MODEL_OPTION_POLICY_PROTOCOLS)[number]; @@ -75,6 +76,7 @@ export const MODEL_OPTION_POLICY_PROTOCOL_LABELS: Record; + /** @maxLength 32 */ + parentMessagePublicID?: string; + prompt: string; + /** @maxLength 32 */ + sourceMessagePublicID?: string; + /** @maxLength 128 */ + sourceVideoFileID: string; +} + export interface MemoryErrorDoc { data: any; details?: any; @@ -2067,7 +2083,12 @@ export interface ModelProbeDebugResponseResponse { } export interface ModelProbeRequest { - taskType?: "chat" | "image_generation" | "image_edit" | "video_generation"; + taskType?: + | "chat" + | "image_generation" + | "image_edit" + | "video_generation" + | "video_extension"; } export interface ModelProbeResponse { @@ -7999,6 +8020,24 @@ export namespace Conversations { export type ResponseBody = ConversationUpdateResponseDoc; } + /** + * No description + * @tags Conversations + * @name MediaVideosExtensionsStreamCreate + * @summary 扩展会话视频 + * @request POST:/conversations/{id}/media/videos/extensions/stream + */ + export namespace MediaVideosExtensionsStreamCreate { + export type RequestParams = { + /** 会话 Public ID */ + id: string; + }; + export type RequestQuery = {}; + export type RequestBody = MediaVideoExtensionRequest; + export type RequestHeaders = {}; + export type ResponseBody = string; + } + /** * @description 查询会话内消息列表 * @tags chat From ed9affc3429f7afb234ab409957e0c308f24071c Mon Sep 17 00:00:00 2001 From: liyp Date: Sat, 22 Aug 2026 13:51:40 +0800 Subject: [PATCH 02/54] fix: always snapshot streamed thinking events into trace Keep the in-memory trace snapshot updated on every streaming think flush so per-round startedAt/roundID reach the frontend, while DB persistence stays throttled by ProcessTracePersistInflight. Expose block-level startedAt through the trace API (with regenerated contracts) and cover snapshot, throttle, and round-identity semantics with unit tests. --- backend/docs/docs.go | 3 + backend/docs/swagger.json | 3 + backend/docs/swagger.yaml | 2 + .../conversation/service_message_hydration.go | 1 + .../application/conversation/service_trace.go | 12 +- .../service_trace_think_snapshot_test.go | 176 ++++++++++++++++++ backend/internal/domain/conversation/types.go | 1 + .../http/conversation/dto_response.go | 28 ++- .../http/conversation/dto_response_test.go | 32 ++++ packages/api-contract/src/types.generated.ts | 1 + 10 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 backend/internal/application/conversation/service_trace_think_snapshot_test.go diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 01625408e..ed2c5b353 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -19839,6 +19839,9 @@ const docTemplate = `{ "stage": { "type": "string" }, + "startedAt": { + "type": "string" + }, "status": { "type": "string" }, diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 652b35a74..4424ad439 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -19832,6 +19832,9 @@ "stage": { "type": "string" }, + "startedAt": { + "type": "string" + }, "status": { "type": "string" }, diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index dfd916b58..319301467 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -4304,6 +4304,8 @@ definitions: type: string stage: type: string + startedAt: + type: string status: type: string summary: diff --git a/backend/internal/application/conversation/service_message_hydration.go b/backend/internal/application/conversation/service_message_hydration.go index 1b66e1808..5759e3929 100644 --- a/backend/internal/application/conversation/service_message_hydration.go +++ b/backend/internal/application/conversation/service_message_hydration.go @@ -101,6 +101,7 @@ func buildMessageProcessTraceDTO(rows []model.MessageTrace, eventRows []model.Me Stage: row.Stage, RoundID: row.RoundID, ParentEventID: row.ParentEventID, + StartedAt: row.StartedAt, UpdatedAt: row.UpdatedAt, PayloadJSON: row.PayloadJSON, } diff --git a/backend/internal/application/conversation/service_trace.go b/backend/internal/application/conversation/service_trace.go index 1d5913a10..8fef0978c 100644 --- a/backend/internal/application/conversation/service_trace.go +++ b/backend/internal/application/conversation/service_trace.go @@ -766,9 +766,14 @@ func (r *messageTraceRecorder) flushUpstreamThinkLiveUpdate(draft *messageTraceD if !r.enabled() || draft == nil { return } - if persistSnapshot && r.shouldPersistUpstreamThinkSnapshot() { - r.persistDraft(draft, false) - r.upstreamThinkLastPersist = time.Now() + if persistSnapshot { + // 思考事件必须先进入内存快照，前端才能拿到每轮的 startedAt/roundID； + // DB 落盘仍由 ProcessTracePersistInflight 节流。 + r.upsertSnapshotEvent(draft, tracePayloadJSON(draft.payload)) + if r.shouldPersistUpstreamThinkSnapshot() { + r.persistDraft(draft, false) + r.upstreamThinkLastPersist = time.Now() + } } update := upstreamThinkLiveUpdate{ kind: r.upstreamThinkPendingKind, @@ -992,6 +997,7 @@ func traceDraftToBlock(draft *messageTraceDraft) *model.MessageTraceBlock { Stage: draft.stage, RoundID: draft.roundID, ParentEventID: draft.parentEventID, + StartedAt: draft.startedAt, UpdatedAt: updatedAt, PayloadJSON: payloadJSON, } diff --git a/backend/internal/application/conversation/service_trace_think_snapshot_test.go b/backend/internal/application/conversation/service_trace_think_snapshot_test.go new file mode 100644 index 000000000..1039ece4c --- /dev/null +++ b/backend/internal/application/conversation/service_trace_think_snapshot_test.go @@ -0,0 +1,176 @@ +package conversation + +import ( + "context" + "testing" + "time" + + model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +// traceRecordRepoStub 记录 trace 行写入，用于验证"内存快照始终更新、DB 落盘受节流"这条边界。 +type traceRecordRepoStub struct { + repository.ConversationRepository + traceRows []model.MessageTrace + traceEventRows []model.MessageTraceEventRow +} + +func (s *traceRecordRepoStub) UpsertConversationMessageTrace(_ context.Context, item *model.MessageTrace) error { + if item != nil { + s.traceRows = append(s.traceRows, *item) + } + return nil +} + +func (s *traceRecordRepoStub) UpsertConversationMessageTraceEvent(_ context.Context, item *model.MessageTraceEventRow) error { + if item != nil { + s.traceEventRows = append(s.traceEventRows, *item) + } + return nil +} + +func newSnapshotRecorder(persistInflight bool) (*messageTraceRecorder, *traceRecordRepoStub) { + stub := &traceRecordRepoStub{} + return &messageTraceRecorder{ + cfg: config.Config{ + ProcessTraceEnabled: true, + ProcessTraceVisibleToUser: true, + ProcessTraceStoreUpstreamThink: true, + ProcessTracePersistInflight: persistInflight, + }, + ctx: context.Background(), + assistant: &model.Message{ID: 1, ConversationID: 2, UserID: 3, RunID: "run_snapshot"}, + service: &Service{repo: stub}, + }, stub +} + +func TestStreamingThinkFlushUpdatesMemorySnapshotWithoutInFlightPersistence(t *testing.T) { + recorder, stub := newSnapshotRecorder(false) + + recorder.appendUpstreamReasoning(messageTraceThinkKindContent, "第一轮深度分析", nil) + + if len(stub.traceRows) != 0 || len(stub.traceEventRows) != 0 { + t.Fatalf("in-flight persistence must stay disabled when ProcessTracePersistInflight=false, got %d rows / %d events", + len(stub.traceRows), len(stub.traceEventRows)) + } + trace := recorder.snapshot() + if trace == nil || len(trace.Events) != 1 { + t.Fatalf("expected one in-memory think snapshot event, got %#v", trace) + } + event := trace.Events[0] + if event.EventType != "think" || event.Stage != messageTraceStageThink { + t.Fatalf("unexpected snapshot event: %#v", event) + } + if event.RoundID == "" || event.RoundID != recorder.upstreamThink.roundID { + t.Fatalf("expected snapshot roundID to follow live draft, got %q want %q", event.RoundID, recorder.upstreamThink.roundID) + } + if event.StartedAt.IsZero() || !event.StartedAt.Equal(recorder.upstreamThink.startedAt) { + t.Fatalf("expected snapshot event startedAt to carry live draft start, got %v want %v", event.StartedAt, recorder.upstreamThink.startedAt) + } + if event.Status != messageTraceStatusStreaming { + t.Fatalf("expected streaming status in live snapshot, got %q", event.Status) + } + if event.UpdatedAt.IsZero() { + t.Fatal("expected updatedAt on live snapshot event") + } + if trace.UpstreamThink == nil || !trace.UpstreamThink.StartedAt.Equal(recorder.upstreamThink.startedAt) { + t.Fatalf("expected block-level startedAt in snapshot, got %#v", trace.UpstreamThink) + } +} + +func TestStreamingThinkInflightPersistenceWritesRoundIdentityAndStaysThrottled(t *testing.T) { + recorder, stub := newSnapshotRecorder(true) + + recorder.appendUpstreamReasoning(messageTraceThinkKindContent, "思考内容", nil) + + if len(stub.traceRows) != 1 || len(stub.traceEventRows) != 1 { + t.Fatalf("expected first throttle window to persist trace + event rows, got %d / %d", + len(stub.traceRows), len(stub.traceEventRows)) + } + row := stub.traceRows[0] + if row.RoundID == "" || row.RoundID != recorder.upstreamThink.roundID { + t.Fatalf("expected persisted row roundID to match live draft, got %q want %q", row.RoundID, recorder.upstreamThink.roundID) + } + if row.StartedAt.IsZero() || !row.StartedAt.Equal(recorder.upstreamThink.startedAt) { + t.Fatalf("expected persisted row startedAt to match live draft, got %v want %v", row.StartedAt, recorder.upstreamThink.startedAt) + } + if row.Status != messageTraceStatusStreaming { + t.Fatalf("expected streaming status row, got %q", row.Status) + } + + // 同一节流窗口内的后续 flush 必须跳过 DB 落盘，但内存快照仍要更新。 + recorder.flushUpstreamThinkLiveUpdate(recorder.upstreamThink, true, true) + if len(stub.traceRows) != 1 || len(stub.traceEventRows) != 1 { + t.Fatalf("in-flight persistence must stay throttled inside window, got %d / %d rows", + len(stub.traceRows), len(stub.traceEventRows)) + } + if len(recorder.snapshot().Events) != 1 { + t.Fatalf("memory snapshot must still update on every flush, got %#v", recorder.snapshot().Events) + } +} + +func TestThinkSnapshotUsesFreshStartedAtPerRound(t *testing.T) { + recorder, stub := newSnapshotRecorder(false) + // 本用例只验证内存快照，complete 会触发后台持久化 goroutine（写 stub）， + // 去掉 service 避免跨 goroutine 写入与断言产生竞态。 + recorder.service = nil + + recorder.appendUpstreamReasoning(messageTraceThinkKindContent, "第一轮思考", nil) + firstStart := recorder.upstreamThink.startedAt + firstRound := recorder.upstreamThink.roundID + firstEventID := recorder.upstreamThink.eventID + + recorder.completeUpstreamThink() + recorder.appendUpstreamReasoning(messageTraceThinkKindContent, "第二轮思考", nil) + // 第二轮首段内容落在上一轮 completion 的 live flush 窗口内不会自动落快照，强制 flush 一次。 + recorder.flushUpstreamThinkLiveUpdate(recorder.upstreamThink, true, true) + + if recorder.upstreamThink.roundID == firstRound || recorder.upstreamThink.eventID == firstEventID { + t.Fatalf("expected a fresh round identity after completion, round=%q event=%q", + recorder.upstreamThink.roundID, recorder.upstreamThink.eventID) + } + if recorder.upstreamThink.startedAt.Equal(firstStart) { + t.Fatalf("expected fresh startedAt per round, got %v", recorder.upstreamThink.startedAt) + } + events := recorder.snapshot().Events + if len(events) != 2 { + t.Fatalf("expected snapshots for both rounds, got %#v", events) + } + if !events[0].StartedAt.Equal(firstStart) || !events[1].StartedAt.Equal(recorder.upstreamThink.startedAt) { + t.Fatalf("snapshot startedAt must follow each round's own draft, round1=%v round2=%v", + events[0].StartedAt, events[1].StartedAt) + } + if events[0].Status != messageTraceStatusCompleted || events[0].EndedAt == nil { + t.Fatalf("expected completed first round snapshot with endedAt, got %#v", events[0]) + } + if events[1].Status != messageTraceStatusStreaming { + t.Fatalf("expected streaming second round snapshot, got %#v", events[1]) + } + if len(stub.traceRows) != 0 { + t.Fatalf("in-flight persistence must stay disabled, got %d rows", len(stub.traceRows)) + } +} + +func TestBuildMessageProcessTraceDTOBlockCarriesStartedAt(t *testing.T) { + started := time.Now().Add(-2 * time.Minute) + trace := buildMessageProcessTraceDTO([]model.MessageTrace{{ + TraceType: messageTraceTypeUpstreamThink, + Status: messageTraceStatusCompleted, + Title: "模型思考", + Summary: "完成", + ContentMarkdown: "思考内容", + RoundID: "round_1", + StartedAt: started, + }}, nil) + if trace == nil || trace.UpstreamThink == nil { + t.Fatalf("expected upstream think block, got %#v", trace) + } + if !trace.UpstreamThink.StartedAt.Equal(started) { + t.Fatalf("expected hydrated block startedAt, got %v want %v", trace.UpstreamThink.StartedAt, started) + } + if trace.Status != messageTraceStatusCompleted { + t.Fatalf("expected completed status, got %q", trace.Status) + } +} diff --git a/backend/internal/domain/conversation/types.go b/backend/internal/domain/conversation/types.go index e75c6d4df..cc2a83bcb 100644 --- a/backend/internal/domain/conversation/types.go +++ b/backend/internal/domain/conversation/types.go @@ -99,6 +99,7 @@ type MessageTraceBlock struct { Stage string RoundID string ParentEventID string + StartedAt time.Time UpdatedAt time.Time PayloadJSON string } diff --git a/backend/internal/transport/http/conversation/dto_response.go b/backend/internal/transport/http/conversation/dto_response.go index 2ef438ce6..14ad94cef 100644 --- a/backend/internal/transport/http/conversation/dto_response.go +++ b/backend/internal/transport/http/conversation/dto_response.go @@ -538,15 +538,16 @@ func toDeleteFileResponse(r *appupload.DeleteFileResult) DeleteFileResponse { // MessageTraceBlockResponse 消息轨迹块响应 DTO。 type MessageTraceBlockResponse struct { - Title string `json:"title"` - Summary string `json:"summary"` - ContentMarkdown string `json:"contentMarkdown"` - Status string `json:"status"` - Stage string `json:"stage,omitempty"` - RoundID string `json:"roundID,omitempty"` - ParentEventID string `json:"parentEventID,omitempty"` - UpdatedAt time.Time `json:"updatedAt"` - PayloadJSON string `json:"payloadJSON,omitempty"` + Title string `json:"title"` + Summary string `json:"summary"` + ContentMarkdown string `json:"contentMarkdown"` + Status string `json:"status"` + Stage string `json:"stage,omitempty"` + RoundID string `json:"roundID,omitempty"` + ParentEventID string `json:"parentEventID,omitempty"` + StartedAt *time.Time `json:"startedAt,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` + PayloadJSON string `json:"payloadJSON,omitempty"` } // MessageTraceEventResponse 消息轨迹事件响应 DTO。 @@ -836,6 +837,13 @@ type MessageModerationResponse struct { Categories []string `json:"categories,omitempty"` } +func toOptionalTraceTime(t time.Time) *time.Time { + if t.IsZero() { + return nil + } + return &t +} + func toTraceBlockResponse(b *model.MessageTraceBlock) *MessageTraceBlockResponse { if b == nil { return nil @@ -848,6 +856,7 @@ func toTraceBlockResponse(b *model.MessageTraceBlock) *MessageTraceBlockResponse Stage: b.Stage, RoundID: b.RoundID, ParentEventID: b.ParentEventID, + StartedAt: toOptionalTraceTime(b.StartedAt), UpdatedAt: b.UpdatedAt, PayloadJSON: sanitizeTracePayloadJSON(b.PayloadJSON), } @@ -865,6 +874,7 @@ func toPublicTraceBlockResponse(b *model.MessageTraceBlock) *MessageTraceBlockRe Stage: b.Stage, RoundID: b.RoundID, ParentEventID: b.ParentEventID, + StartedAt: toOptionalTraceTime(b.StartedAt), UpdatedAt: b.UpdatedAt, PayloadJSON: sanitizePublicTracePayloadJSON(b.PayloadJSON), } diff --git a/backend/internal/transport/http/conversation/dto_response_test.go b/backend/internal/transport/http/conversation/dto_response_test.go index 9b750d3ed..35745dbff 100644 --- a/backend/internal/transport/http/conversation/dto_response_test.go +++ b/backend/internal/transport/http/conversation/dto_response_test.go @@ -3,6 +3,9 @@ package conversation import ( "strings" "testing" + "time" + + model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" ) func TestSanitizePublicTracePayloadRemovesKnowledgeEvidence(t *testing.T) { @@ -23,3 +26,32 @@ func TestSanitizePublicTracePayloadRemovesKnowledgeEvidence(t *testing.T) { t.Fatalf("sanitizePublicTracePayloadJSON() removed safe retrieval diagnostics: %s", got) } } + +func TestTraceBlockResponseCarriesStartedAtWhenSet(t *testing.T) { + started := time.Now().Add(-time.Minute) + resp := toTraceBlockResponse(&model.MessageTraceBlock{ + Title: "模型思考", + Status: "completed", + RoundID: "round_1", + StartedAt: started, + UpdatedAt: started.Add(30 * time.Second), + PayloadJSON: `{"tool_calls":[]}`, + }) + if resp == nil || resp.StartedAt == nil || !resp.StartedAt.Equal(started) { + t.Fatalf("expected startedAt in block response, got %#v", resp) + } +} + +func TestTraceBlockResponseOmitsZeroStartedAt(t *testing.T) { + resp := toTraceBlockResponse(&model.MessageTraceBlock{ + Title: "处理", + Status: "completed", + PayloadJSON: `{"tool_calls":[]}`, + }) + if resp == nil || resp.StartedAt != nil { + t.Fatalf("expected omitted startedAt for zero value, got %#v", resp) + } + if resp.PayloadJSON == "" { + t.Fatalf("expected payload to survive mapping, got %#v", resp) + } +} diff --git a/packages/api-contract/src/types.generated.ts b/packages/api-contract/src/types.generated.ts index 3d3b3b16b..e2b97563e 100644 --- a/packages/api-contract/src/types.generated.ts +++ b/packages/api-contract/src/types.generated.ts @@ -1891,6 +1891,7 @@ export interface MessageTraceBlockResponse { payloadJSON?: string; roundID?: string; stage?: string; + startedAt?: string; status: string; summary: string; title: string; From 8c3b1b4ef6797eb304ec6044c479c229338e3a32 Mon Sep 17 00:00:00 2001 From: liyp Date: Sat, 22 Aug 2026 13:51:44 +0800 Subject: [PATCH 03/54] feat: interleave thinking and tool steps with formatted tool cards in agent trace timeline Group think and tool trace events into agent-loop rounds rendered as a unified timeline: per-round think rows with durations, deduplicated tool call cards (arguments/result with copy), and a collapsible run summary header. Consumes block-level startedAt from the trace API. --- .../chat/components/message/message-bot.tsx | 1 + .../message/message-thinking-trace.tsx | 481 ++++++++++++++++-- .../components/message/message-tool-trace.tsx | 360 +++++++++++-- .../chat/hooks/use-process-trace-labels.ts | 32 ++ .../features/chat/model/message-submit.ts | 1 + .../chat/model/upstream-think-store.ts | 6 +- frontend/features/chat/types/messages.ts | 1 + frontend/i18n/messages/en-US/chat.json | 18 +- frontend/i18n/messages/zh-CN/chat.json | 18 +- 9 files changed, 818 insertions(+), 100 deletions(-) diff --git a/frontend/features/chat/components/message/message-bot.tsx b/frontend/features/chat/components/message/message-bot.tsx index d4226e683..ccebad738 100644 --- a/frontend/features/chat/components/message/message-bot.tsx +++ b/frontend/features/chat/components/message/message-bot.tsx @@ -368,6 +368,7 @@ export function ChatMessageBot({ activeThinkBlock={upstreamThink} messageStreaming={messageStreaming} autoCollapseReady={hasStreamdownContent || Boolean(item.inlineAlert)} + runDurationMS={item.latencyMS} />
item.kind === "tool"), - thinkBlock: mergeThinkTraceBlock(events, activeThinkBlock), +type TraceRoundGroup = { + key: string; + seq: number; + thinkEvents: TraceDisplayEvent[]; + toolEvents: TraceDisplayEvent[]; + thinkBlock?: ChatTraceBlock; + toolBlock?: ChatTraceBlock; +}; + +function traceRoundGroupKey(event: Pick, kind: "think" | "tool"): string { + const roundID = event.roundID?.trim() || ""; + if (roundID) { + return `${kind}:${roundID}`; + } + return `${kind}:${event.eventID || event.seq}`; +} + +/** + * Group think and tool events into agent-loop rounds so each thinking step can be + * rendered right above the tool calls it produced. Think events key their own + * round; tool events join the preceding think round via roundID / parentEventID + * and keep a standalone group when the round ran without thinking. + */ +function groupTraceDisplayEvents( + displayEvents: TraceDisplayEvent[], + activeThinkBlock?: ChatTraceBlock, + activeToolBlock?: ChatTraceBlock, +): TraceRoundGroup[] { + const groups = new Map(); + const thinkEventIDToKey = new Map(); + const thinkRoundIDToKey = new Map(); + + const ensureGroup = (key: string, seq: number): TraceRoundGroup => { + let group = groups.get(key); + if (!group) { + group = { key, seq, thinkEvents: [], toolEvents: [] }; + groups.set(key, group); + } else if (seq < group.seq) { + group.seq = seq; + } + return group; + }; + + for (const item of displayEvents) { + if (item.kind !== "think") { + continue; + } + const key = traceRoundGroupKey(item.event, "think"); + ensureGroup(key, item.event.seq).thinkEvents.push(item); + if (item.event.roundID?.trim()) { + thinkRoundIDToKey.set(item.event.roundID.trim(), key); + } + if (item.event.eventID) { + thinkEventIDToKey.set(item.event.eventID, key); + } + } + + for (const item of displayEvents) { + if (item.kind !== "tool") { + continue; + } + const roundID = item.event.roundID?.trim() || ""; + const parentID = item.event.parentEventID?.trim() || ""; + const key = + (roundID && thinkRoundIDToKey.get(roundID)) || + (parentID && thinkEventIDToKey.get(parentID)) || + traceRoundGroupKey(item.event, "tool"); + ensureGroup(key, item.event.seq).toolEvents.push(item); + } + + // Live streaming blocks join their own round; unmatched blocks are appended last. + const attachActiveBlock = (block: ChatTraceBlock | undefined, kind: "think" | "tool") => { + if (!block) { + return; + } + const roundID = block.roundID?.trim() || ""; + const parentID = block.parentEventID?.trim() || ""; + const matchedKey = (roundID && thinkRoundIDToKey.get(roundID)) || (parentID && thinkEventIDToKey.get(parentID)); + if (matchedKey) { + const matched = groups.get(matchedKey); + if (matched) { + if (kind === "think") { + matched.thinkBlock = block; + } else { + matched.toolBlock = block; + } + return; + } + } + const toolOnlyKey = `tool:${roundID}`; + if (roundID && groups.has(toolOnlyKey)) { + const toolOnly = groups.get(toolOnlyKey); + if (toolOnly) { + if (kind === "think") { + toolOnly.thinkBlock = block; + } else { + toolOnly.toolBlock = block; + } + return; + } + } + const key = `active:${kind}:${roundID || parentID || "latest"}`; + const group = ensureGroup(key, Number.MAX_SAFE_INTEGER); + if (kind === "think") { + group.thinkBlock = block; + } else { + group.toolBlock = block; + } }; + attachActiveBlock(activeThinkBlock, "think"); + attachActiveBlock(activeToolBlock, "tool"); + + return [...groups.values()].sort((left, right) => left.seq - right.seq); +} + +function thinkEventDurationMS(thinkEvents: TraceDisplayEvent[]): number | undefined { + let total = 0; + for (const item of thinkEvents) { + const { startedAt, endedAt, updatedAt } = item.event; + if (!startedAt) { + continue; + } + const startMS = new Date(startedAt).getTime(); + if (!Number.isFinite(startMS)) { + continue; + } + // 部分历史事件只有 startedAt（未走到 complete 落盘），用快照更新时间兜底。 + const rawEnd = endedAt?.trim() ? endedAt : updatedAt; + if (!rawEnd) { + continue; + } + const endMS = new Date(rawEnd).getTime(); + if (!Number.isFinite(endMS) || endMS <= startMS) { + continue; + } + total += endMS - startMS; + } + return total > 0 ? total : undefined; +} + +function thinkBlockDurationMS(block: ChatTraceBlock): number | undefined { + const { startedAt, updatedAt } = block; + if (!startedAt || !updatedAt) { + return undefined; + } + const startMS = new Date(startedAt).getTime(); + const endMS = new Date(updatedAt).getTime(); + if (!Number.isFinite(startMS) || !Number.isFinite(endMS) || endMS <= startMS) { + return undefined; + } + return endMS - startMS; +} + +function formatThinkDuration(durationMS: number | undefined): string | undefined { + if (!durationMS || durationMS <= 0) { + return undefined; + } + if (durationMS < 10000) { + return `${(durationMS / 1000).toFixed(1)}s`; + } + return `${Math.round(durationMS / 1000)}s`; } -function firstTraceSeq(events: TraceDisplayEvent[], kind: TraceDisplayEvent["kind"]): number | undefined { - const matched = events.filter((item) => item.kind === kind).map((item) => item.event.seq); - if (matched.length === 0) { +function formatRunDuration(durationMS: number | undefined): string | undefined { + if (!durationMS || durationMS <= 0) { return undefined; } - return Math.min(...matched); + const wholeSeconds = Math.max(1, Math.round(durationMS / 1000)); + if (wholeSeconds < 60) { + return `${wholeSeconds}s`; + } + const minutes = Math.floor(wholeSeconds / 60); + const seconds = wholeSeconds % 60; + return `${minutes}m ${seconds}s`; +} + +function isGroupToolStreaming(group: TraceRoundGroup, messageStreaming: boolean): boolean { + if (!messageStreaming) { + return false; + } + if (group.toolBlock?.status === "streaming" || hasActiveToolTraceCalls(group.toolBlock?.payloadJson)) { + return true; + } + return group.toolEvents.some( + (item) => item.event.status === "streaming" || hasActiveToolTraceCalls(item.event.payloadJson), + ); +} + +type TraceTimelineItem = + | { kind: "think"; key: string; block: ChatTraceBlock; streaming: boolean; durationMS?: number } + | { kind: "tool"; key: string; step: ToolChainStep }; + +function TraceThinkRow({ + block, + streaming, + durationMS, + autoCollapseReady, + labels, +}: { + block: ChatTraceBlock; + streaming: boolean; + durationMS?: number; + autoCollapseReady?: boolean; + labels: ProcessTraceLabels; +}) { + const [open, setOpen] = React.useState(streaming); + const wasStreamingRef = React.useRef(streaming); + + React.useEffect(() => { + if (streaming) { + setOpen(true); + wasStreamingRef.current = true; + return; + } + if (wasStreamingRef.current && autoCollapseReady) { + setOpen(false); + } + if (autoCollapseReady) { + wasStreamingRef.current = false; + } + }, [autoCollapseReady, streaming]); + + const durationText = formatThinkDuration(durationMS); + + return ( +
  • +
    +
    + + {streaming ? : } + +
    + +
    + {open ? ( +
    + +
    + ) : null} +
    • + ); +} + +function AgentTraceTimeline({ + items, + labels, + autoCollapseReady, +}: { + items: TraceTimelineItem[]; + labels: ProcessTraceLabels; + autoCollapseReady?: boolean; +}) { + return ( +
    + +
      + {items.map((item) => + item.kind === "think" ? ( + + ) : ( + + ), + )} +
    +
    + ); } export function MessageTraceEventBlocks({ @@ -133,65 +425,150 @@ export function MessageTraceEventBlocks({ activeThinkBlock, messageStreaming, autoCollapseReady, + runDurationMS, }: { events: ChatTraceEvent[]; activeToolBlock?: ChatTraceBlock; activeThinkBlock?: ChatTraceBlock; messageStreaming?: boolean; autoCollapseReady?: boolean; + runDurationMS?: number; }) { + const labels = useProcessTraceLabels(); const displayEvents = React.useMemo(() => buildTraceDisplayEvents(traceEvents), [traceEvents]); - const { toolEvents, thinkBlock } = React.useMemo( - () => splitTraceDisplayEvents(displayEvents, activeThinkBlock), - [activeThinkBlock, displayEvents], + const groups = React.useMemo( + () => groupTraceDisplayEvents(displayEvents, activeThinkBlock, activeToolBlock), + [activeThinkBlock, activeToolBlock, displayEvents], + ); + const groupToolSteps = React.useMemo( + () => groups.map((group) => buildToolGroupSteps(group.toolEvents, group.toolBlock, labels)), + [groups, labels], ); - if (toolEvents.length === 0 && !activeToolBlock && !thinkBlock) { + const items = React.useMemo(() => { + const list: TraceTimelineItem[] = []; + // 聚合工具块会包含全部轮次的调用，跨组按调用 ID 去重，避免每个轮次重复出现。 + const seenToolKeys = new Set(); + groups.forEach((group, index) => { + const thinkBlock = mergeThinkTraceBlock(group.thinkEvents, group.thinkBlock); + if (thinkBlock) { + list.push({ + kind: "think", + key: `${group.key}:think`, + block: thinkBlock, + streaming: Boolean(messageStreaming && thinkBlock.status === "streaming"), + durationMS: thinkEventDurationMS(group.thinkEvents) ?? thinkBlockDurationMS(thinkBlock), + }); + } + groupToolSteps[index].forEach((step, stepIndex) => { + const toolKey = step.toolCallID?.trim() + ? `id:${step.toolCallID.trim()}` + : `fb:${step.label}:${step.toolInput?.trim() || ""}`; + if (toolKey && seenToolKeys.has(toolKey)) { + return; + } + if (toolKey) { + seenToolKeys.add(toolKey); + } + list.push({ kind: "tool", key: `${group.key}:${step.key}:${stepIndex}`, step }); + }); + }); + return list; + }, [groupToolSteps, groups, messageStreaming]); + + const [accordionValue, setAccordionValue] = React.useState(() => (messageStreaming ? "message-trace-timeline" : "")); + const wasStreamingRef = React.useRef(Boolean(messageStreaming)); + + React.useEffect(() => { + if (messageStreaming) { + setAccordionValue("message-trace-timeline"); + wasStreamingRef.current = true; + return; + } + if (wasStreamingRef.current && autoCollapseReady) { + setAccordionValue(""); + } + if (autoCollapseReady) { + wasStreamingRef.current = false; + } + }, [autoCollapseReady, messageStreaming]); + + if (items.length === 0) { return null; } - const toolSeq = firstTraceSeq(displayEvents, "tool"); - const thinkSeq = firstTraceSeq(displayEvents, "think"); - const shouldRenderThinkFirst = Boolean( - thinkBlock && - ( - (toolEvents.length === 0 && !activeToolBlock) || - thinkSeq === undefined || - toolSeq === undefined || - thinkSeq <= toolSeq - ), - ); - const toolTrace = ( - item.event.status === "streaming" || hasActiveToolTraceCalls(item.event.payloadJson)) - ), - )} - autoCollapseReady={autoCollapseReady || Boolean(thinkBlock)} - /> - ); - const thinkTrace = thinkBlock ? ( - - ) : null; + const allToolSteps = groupToolSteps.flat(); + const toolSummary = summarizeToolChainSteps(allToolSteps); + const thinkRounds = items.filter((item) => item.kind === "think").length; + const thinkStreaming = items.some((item) => item.kind === "think" && item.streaming); + const toolsStreaming = groups.some((group) => isGroupToolStreaming(group, Boolean(messageStreaming))); + const durationText = formatRunDuration(runDurationMS); + + const kindChips = toolSummary.kinds.map((kind) => `${kind.label} (${kind.count})`).join(labels.run.listSeparator); + const subtitleParts: string[] = []; + if (toolSummary.total > 0) { + subtitleParts.push(labels.run.toolCallsSummary(toolSummary.total, kindChips)); + } + if (thinkRounds > 0) { + subtitleParts.push(labels.run.thinkRounds(thinkRounds)); + } + const subtitle = subtitleParts.join(labels.run.labelSeparator); + + const resolvedTitle = messageStreaming + ? thinkStreaming + ? labels.think.titleActive + : toolsStreaming + ? labels.tool.chain.titleActive + : labels.think.titleActive + : labels.run.summarySteps(groups.length); + const title = durationText ? `${resolvedTitle}${labels.run.durationSuffix(durationText)}` : resolvedTitle; + const open = accordionValue === "message-trace-timeline"; return ( - <> - {shouldRenderThinkFirst ? thinkTrace : toolTrace} - {shouldRenderThinkFirst ? toolTrace : thinkTrace} - +
    + setAccordionValue(value || "")} + className="w-full" + > + + +
    +
    + } + className={cn( + "inline-flex min-h-0 w-auto text-[13px] font-medium transition-colors", + !messageStreaming && "text-muted-foreground group-hover:text-foreground", + )} + > + {title} + +
    + {subtitle ? ( +
    {subtitle}
    + ) : null} +
    + +     + + + +     +     +
    ); } - export function MessageUpstreamThink({ block, streaming, diff --git a/frontend/features/chat/components/message/message-tool-trace.tsx b/frontend/features/chat/components/message/message-tool-trace.tsx index 307282c61..478c8c56a 100644 --- a/frontend/features/chat/components/message/message-tool-trace.tsx +++ b/frontend/features/chat/components/message/message-tool-trace.tsx @@ -2,6 +2,8 @@ import * as React from "react"; +import { Check, Copy, X } from "lucide-react"; + import { ChevronDown } from "@/components/animate-ui/icons/chevron-down"; import { Accordion, @@ -9,7 +11,9 @@ import { AccordionItem, AccordionTrigger, } from "@/components/ui/accordion"; +import { Spinner } from "@/components/ui/spinner"; import { Marker, MarkerContent } from "@/components/ui/marker"; +import { useCopyAction } from "@/shared/components/copy-action"; import type { ChatTraceBlock } from "@/features/chat/types/messages"; import { useProcessTraceLabels, @@ -633,7 +637,7 @@ function ToolTraceStructuredContent({ ); } -type ToolChainStep = { +export type ToolChainStep = { key: string; label: string; detail: string; @@ -702,7 +706,34 @@ function dedupeToolChainSteps(steps: ToolChainStep[]): ToolChainStep[] { return result; } -function buildToolChainSteps(events: TraceDisplayEvent[], labels: ProcessTraceLabels): ToolChainStep[] { +export function buildToolGroupSteps( + toolEvents: TraceDisplayEvent[], + toolBlock: ChatTraceBlock | undefined, + labels: ProcessTraceLabels, +): ToolChainStep[] { + return dedupeToolChainSteps([ + ...buildToolChainSteps(toolEvents, labels), + ...buildToolChainStepsFromBlock(toolBlock, labels), + ]); +} + +export type ToolTraceSummary = { + total: number; + kinds: Array<{ label: string; count: number }>; +}; + +export function summarizeToolChainSteps(steps: ToolChainStep[]): ToolTraceSummary { + const kinds = new Map(); + for (const step of steps) { + kinds.set(step.label, (kinds.get(step.label) ?? 0) + 1); + } + return { + total: steps.length, + kinds: Array.from(kinds.entries()).map(([label, count]) => ({ label, count })), + }; +} + +export function buildToolChainSteps(events: TraceDisplayEvent[], labels: ProcessTraceLabels): ToolChainStep[] { return events.flatMap((item, eventIndex) => { const event = item.event; if (item.kind !== "tool") { @@ -717,6 +748,7 @@ function buildToolChainSteps(events: TraceDisplayEvent[], labels: ProcessTraceLa label: labels.tool.names.generic, detail: event.contentMarkdown?.trim() || event.summary?.trim() || event.title?.trim() || "", failed: event.status === "error", + toolStatus: event.status?.trim(), }, ]; } @@ -777,6 +809,47 @@ function buildToolChainStepsFromBlock(block: ChatTraceBlock | undefined, labels: }); } +function ToolTraceRowBody({ + step, + open, + canExpand, + onToggle, + labels, +}: { + step: ToolChainStep; + open: boolean; + canExpand: boolean; + onToggle: () => void; + labels: ProcessTraceLabels; +}) { + if (step.toolCall) { + return ( + + ); + } + return ( + + {step.latencyMS && step.latencyMS > 0 ? {step.latencyMS}ms : null} + {step.latencyMS && step.latencyMS > 0 && step.detail ? {labels.tool.detail.latencySeparator} : null} + {step.detail ? {step.detail} : null} + + ); +} + function ToolChainRows({ steps, labels }: { steps: ToolChainStep[]; labels: ProcessTraceLabels }) { const [expanded, setExpanded] = React.useState>(() => new Set()); @@ -817,49 +890,23 @@ function ToolChainRows({ steps, labels }: { steps: ToolChainStep[]; labels: Proc
    - {step.toolCall ? ( - - setExpanded((current) => { - const next = new Set(current); - if (next.has(step.key)) { - next.delete(step.key); - } else { - next.add(step.key); - } - return next; - }) - } - /> - ) : ( - - setExpanded((current) => { - const next = new Set(current); - if (next.has(step.key)) { - next.delete(step.key); - } else { - next.add(step.key); - } - return next; - }) - } - > - {step.latencyMS && step.latencyMS > 0 ? {step.latencyMS}ms : null} - {step.latencyMS && step.latencyMS > 0 && step.detail ? {labels.tool.detail.latencySeparator} : null} - {step.detail ? {step.detail} : null} - - )} + + setExpanded((current) => { + const next = new Set(current); + if (next.has(step.key)) { + next.delete(step.key); + } else { + next.add(step.key); + } + return next; + }) + } + labels={labels} + />
    ); @@ -868,6 +915,229 @@ function ToolChainRows({ steps, labels }: { steps: ToolChainStep[]; labels: Proc ); } +function isToolStepActive(step: ToolChainStep): boolean { + return isToolTraceStatusActive(step.toolCall?.status) || isToolTraceStatusActive(step.toolStatus); +} + +function isToolStepDone(step: ToolChainStep): boolean { + const status = step.toolCall?.status?.trim() || step.toolStatus?.trim() || ""; + return status === "success" || status === "completed" || status === "reused"; +} + +function ToolStepStatusIcon({ step }: { step: ToolChainStep }) { + if (step.failed) { + return ; + } + if (isToolStepActive(step)) { + return ; + } + if (isToolStepDone(step)) { + return ; + } + return ; +} + +function formatArgumentValue(value: unknown): string { + if (Array.isArray(value)) { + return value.map((item) => (typeof item === "object" ? JSON.stringify(item) : String(item))).join(", "); + } + if (typeof value === "object" && value !== null) { + return Object.entries(value) + .map(([key, item]) => `${key}: ${typeof item === "object" ? JSON.stringify(item) : String(item)}`) + .join(", "); + } + return String(value); +} + +function ToolArgumentCopyButton({ value, labels }: { value: string; labels: ProcessTraceLabels }) { + const { copy, isCopied } = useCopyAction({ + messages: { copied: labels.tool.detail.copied, failed: labels.tool.detail.copyFailed }, + }); + const done = isCopied(value); + return ( + + ); +} + +function ToolArgumentsCard({ call, labels }: { call: ToolTraceCall; labels: ProcessTraceLabels }) { + const input = toolInputPayload(call); + if (!isRecord(input) || Object.keys(input).length === 0) { + return null; + } + return ( +
    +
    {labels.tool.detail.argumentsTitle}
    +
    + {Object.entries(input).map(([key, value]) => { + const text = formatArgumentValue(value); + return ( +
    + {key} + {text} + +
    + ); + })} +
    +
    + ); +} + +function ToolResultCard({ call, labels }: { call: ToolTraceCall; labels: ProcessTraceLabels }) { + const failedStatus = call.status === "error" || call.status === "failed"; + const kind = resolveNativeToolKind(call); + const output = toolOutputPayload(call); + let content: React.ReactNode = null; + + if (failedStatus) { + const errorText = call.error?.trim(); + if (errorText) { + content = {errorText}; + } + } else if (kind === "web_search") { + const urls = collectToolStrings(output, ["url", "uri", "image_url", "retrievedUrl"]); + const responseText = geminiWebSearchSummary(output) + || formatToolPayload(call.output_detail) + || formatToolPayload(call.output) + || formatToolPayload(call.output_text) + || formatToolPayload(call.output_preview); + if (urls.length > 0 || responseText) { + content = ( + <> + {urls.length > 0 ? : null} + {responseText ? {responseText} : null} + + ); + } + } else if (kind === "code_interpreter") { + const logs = collectToolStrings(output, ["logs", "stdout", "stderr", "text", "output"]).join("\n\n"); + const artifactURLs = collectToolStrings(output, ["url", "uri", "image_url"]); + if (logs || artifactURLs.length > 0) { + content = ( + <> + {logs ? {logs} : null} + {artifactURLs.length > 0 ? : null} + + ); + } + } else if (kind === "image_generation") { + const urls = collectToolImageSources(output); + if (urls.length > 0) { + content = ; + } + } else if (kind === "shell") { + const stdout = toolOutputText(call, ["stdout", "output"]); + const stderr = toolOutputText(call, ["stderr", "error"]); + const exitCode = isRecord(output) ? readNumber(output.exit_code) ?? readNumber(output.code) : null; + if (stdout || stderr || exitCode !== null) { + content = ( + <> + {stdout ? {stdout} : null} + {stderr ? {stderr} : null} + {exitCode !== null ?
    exit code: {exitCode}
    : null} + + ); + } + } else { + const text = formatToolPayload(call.output_detail) + || formatToolPayload(call.output) + || formatToolPayload(call.output_text) + || formatToolPayload(call.output_preview); + if (text) { + content = {text}; + } + } + + if (!content) { + return null; + } + return ( +
    +
    {labels.tool.detail.resultTitle}
    +
    {content}
    +
    + ); +} + +function ToolCallDetailCard({ step, labels }: { step: ToolChainStep; labels: ProcessTraceLabels }) { + const call = step.toolCall; + return ( +
    + {call ? ( + <> + + + + ) : step.detail ? ( +
    {step.detail}
    + ) : null} +
    + ); +} + +export function AgentToolStepRow({ step, labels }: { step: ToolChainStep; labels: ProcessTraceLabels }) { + const [open, setOpen] = React.useState(false); + const failed = step.failed; + const statusText = toolStatusLabel(step.toolCall?.status ?? step.toolStatus, labels); + const expandable = Boolean(step.toolCall || step.detail); + + return ( +
  • +
    +
    + +
    + +
    + {open ? ( +
    + +
    + ) : null} +
    • + ); +} + export function MessageToolChainTrace({ events, activeToolBlock, diff --git a/frontend/features/chat/hooks/use-process-trace-labels.ts b/frontend/features/chat/hooks/use-process-trace-labels.ts index 63b70401e..20a1f23e5 100644 --- a/frontend/features/chat/hooks/use-process-trace-labels.ts +++ b/frontend/features/chat/hooks/use-process-trace-labels.ts @@ -65,6 +65,11 @@ export type ProcessTraceLabels = { error: string; expand: string; collapse: string; + argumentsTitle: string; + resultTitle: string; + copy: string; + copied: string; + copyFailed: string; sourceFallback: (index: number) => string; generatedImageAlt: (index: number) => string; query: string; @@ -106,6 +111,17 @@ export type ProcessTraceLabels = { titleDone: string; subtitleActive: string; subtitleDone: string; + rowActive: string; + rowDone: string; + duration: (seconds: string) => string; + }; + run: { + summarySteps: (count: number) => string; + durationSuffix: (duration: string) => string; + toolCallsSummary: (count: number, kinds: string) => string; + thinkRounds: (count: number) => string; + listSeparator: string; + labelSeparator: string; }; promptTrace: { modes: { @@ -218,6 +234,11 @@ export function useProcessTraceLabels(): ProcessTraceLabels { error: t("tool.detail.error"), expand: t("tool.detail.expand"), collapse: t("tool.detail.collapse"), + argumentsTitle: t("tool.detail.argumentsTitle"), + resultTitle: t("tool.detail.resultTitle"), + copy: t("tool.detail.copy"), + copied: t("tool.detail.copied"), + copyFailed: t("tool.detail.copyFailed"), sourceFallback: (index: number) => t("tool.detail.sourceFallback", { index }), generatedImageAlt: (index: number) => t("tool.detail.generatedImageAlt", { index }), query: t("tool.detail.query"), @@ -259,6 +280,17 @@ export function useProcessTraceLabels(): ProcessTraceLabels { titleDone: t("think.titleDone"), subtitleActive: t("think.subtitleActive"), subtitleDone: t("think.subtitleDone"), + rowActive: t("think.rowActive"), + rowDone: t("think.rowDone"), + duration: (seconds: string) => t("think.duration", { seconds }), + }, + run: { + summarySteps: (count: number) => t("run.summarySteps", { count }), + durationSuffix: (duration: string) => t("run.durationSuffix", { duration }), + toolCallsSummary: (count: number, kinds: string) => t("run.toolCallsSummary", { count, kinds }), + thinkRounds: (count: number) => t("run.thinkRounds", { count }), + listSeparator: t("run.listSeparator"), + labelSeparator: t("run.labelSeparator"), }, promptTrace: { modes: { diff --git a/frontend/features/chat/model/message-submit.ts b/frontend/features/chat/model/message-submit.ts index 2461c991a..042a79533 100644 --- a/frontend/features/chat/model/message-submit.ts +++ b/frontend/features/chat/model/message-submit.ts @@ -98,6 +98,7 @@ function toPendingTraceBlock(block: TraceBlockDTO | undefined) { stage: block.stage, roundID: block.roundID, parentEventID: block.parentEventID, + startedAt: block.startedAt, updatedAt: block.updatedAt, payloadJson: block.payloadJSON, }; diff --git a/frontend/features/chat/model/upstream-think-store.ts b/frontend/features/chat/model/upstream-think-store.ts index 659b3b3cd..2d5de9cfe 100644 --- a/frontend/features/chat/model/upstream-think-store.ts +++ b/frontend/features/chat/model/upstream-think-store.ts @@ -32,14 +32,18 @@ function mergeContent(previous: string, event: UpstreamThinkDeltaEvent) { function mergeUpstreamThinkBlock(current: ChatTraceBlock | undefined, event: UpstreamThinkDeltaEvent): ChatTraceBlock { const contentMarkdown = mergeContent(current?.contentMarkdown ?? "", event); + const roundID = event.roundID || current?.roundID; + // 轮次切换后重新开始计时，避免上一轮的 startedAt 被带到本轮导致耗时变成会话累计值。 + const roundChanged = Boolean(roundID && current?.roundID && roundID !== current.roundID); return { title: event.title?.trim() || current?.title || "", summary: event.summary?.trim() || current?.summary || "", contentMarkdown, status: event.status || current?.status || "streaming", stage: event.stage || current?.stage || "think", - roundID: event.roundID || current?.roundID, + roundID, parentEventID: current?.parentEventID, + startedAt: !current || roundChanged ? nowISO() : current?.startedAt ?? nowISO(), updatedAt: nowISO(), payloadJson: current?.payloadJson, }; diff --git a/frontend/features/chat/types/messages.ts b/frontend/features/chat/types/messages.ts index d0264b124..1f1d5144b 100644 --- a/frontend/features/chat/types/messages.ts +++ b/frontend/features/chat/types/messages.ts @@ -46,6 +46,7 @@ export type ChatTraceBlock = { stage?: string; roundID?: string; parentEventID?: string; + startedAt?: string; updatedAt?: string; payloadJson?: string; }; diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index 3b130d329..ec7f8c038 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -621,6 +621,11 @@ "error": "Error", "expand": "Expand", "collapse": "Collapse", + "argumentsTitle": "Parameters", + "resultTitle": "Result", + "copy": "Copy", + "copied": "Copied", + "copyFailed": "Copy failed", "sourceFallback": "Source {index}", "generatedImageAlt": "Generated image {index}", "query": "Query", @@ -661,7 +666,18 @@ "titleActive": "Thinking", "titleDone": "Thinking complete", "subtitleActive": "Parsing the request and organizing key context", - "subtitleDone": "Reasoning and context integration complete" + "subtitleDone": "Reasoning and context integration complete", + "rowActive": "Thinking deeply", + "rowDone": "Deeply thought", + "duration": "took {seconds}" + }, + "run": { + "summarySteps": "Ran {count} steps", + "durationSuffix": "({duration})", + "toolCallsSummary": "{count} calls: {kinds}", + "thinkRounds": "thought in {count} rounds", + "listSeparator": ", ", + "labelSeparator": " · " }, "promptTrace": { "modes": { diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index c304a6059..4b3e4cfa0 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -621,6 +621,11 @@ "error": "错误", "expand": "展开", "collapse": "收起", + "argumentsTitle": "参数列表", + "resultTitle": "返回结果", + "copy": "复制", + "copied": "已复制", + "copyFailed": "复制失败", "sourceFallback": "来源 {index}", "generatedImageAlt": "生成图片 {index}", "query": "关键词", @@ -661,7 +666,18 @@ "titleActive": "正在思考", "titleDone": "思考完成", "subtitleActive": "正在解析问题并梳理关键信息", - "subtitleDone": "已理解并完成相关信息整合与推理" + "subtitleDone": "已理解并完成相关信息整合与推理", + "rowActive": "正在深度思考", + "rowDone": "已深度思考", + "duration": "用时 {seconds}" + }, + "run": { + "summarySteps": "共运行 {count} 步", + "durationSuffix": "（{duration}）", + "toolCallsSummary": "{count} 次调用：{kinds}", + "thinkRounds": "思考 {count} 轮", + "listSeparator": "、", + "labelSeparator": " · " }, "promptTrace": { "modes": { From 5a163f1aa4ed08a5b01489e2307589797e4bbd5c Mon Sep 17 00:00:00 2001 From: liyp Date: Sat, 22 Aug 2026 14:18:47 +0800 Subject: [PATCH 04/54] fix: keep project dialog buttons reachable for long prompts The system prompt textarea could grow unboundedly (field-sizing: content) and push the footer out of the clipped dialog. Make the dialog a flex column with a scrollable body, and cap the textarea's height dynamically to the available space so the save button always stays visible. Closes #630 --- .../components/navigation/project-dialog.tsx | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/frontend/features/layouts/components/navigation/project-dialog.tsx b/frontend/features/layouts/components/navigation/project-dialog.tsx index 0814fad19..4b1723c66 100644 --- a/frontend/features/layouts/components/navigation/project-dialog.tsx +++ b/frontend/features/layouts/components/navigation/project-dialog.tsx @@ -75,6 +75,9 @@ export function ProjectDialog({ const open = Boolean(draft); const nameInputID = React.useId(); const systemPromptInputID = React.useId(); + const bodyRef = React.useRef(null); + const systemPromptRef = React.useRef(null); + const [systemPromptMaxHeight, setSystemPromptMaxHeight] = React.useState(undefined); React.useEffect(() => { if (!draft) { @@ -149,6 +152,29 @@ export function ProjectDialog({ }; }, [open, setDraft, t]); + // 输入框随内容增高的上限：滚动区当前可见高度减去其顶部偏移， + // 保证下方选项区和保存按钮始终不被顶出可视范围；窗口或区域尺寸变化时重新测量。 + React.useLayoutEffect(() => { + const body = bodyRef.current; + const textarea = systemPromptRef.current; + if (!open || !body || !textarea) { + return; + } + const measure = () => { + const available = body.clientHeight - (textarea.offsetTop - body.offsetTop) - 8; + const nextMaxHeight = Math.max(0, available); + setSystemPromptMaxHeight((previous) => (previous === nextMaxHeight ? previous : nextMaxHeight)); + }; + measure(); + const resizeObserver = new ResizeObserver(measure); + resizeObserver.observe(body); + window.addEventListener("resize", measure); + return () => { + resizeObserver.disconnect(); + window.removeEventListener("resize", measure); + }; + }, [open]); + const handleSubmit = React.useCallback>( async (event) => { event.preventDefault(); @@ -169,14 +195,14 @@ export function ProjectDialog({ return ( - +
    {stableDraft?.publicID ? t("editTitle") : t("createTitle")} {stableDraft?.publicID ? t("editDescription") : t("createDescription")} -
    +