diff --git a/web/src/components/ai/generation-tool-card.tsx b/web/src/components/ai/generation-tool-card.tsx new file mode 100644 index 000000000..a165ff9da --- /dev/null +++ b/web/src/components/ai/generation-tool-card.tsx @@ -0,0 +1,22 @@ +import { useEffect, useState, type ReactNode } from "react"; +import { ChevronDown } from "lucide-react"; + +export type GenerationToolStatus = "running" | "completed" | "error" | "cancelled"; + +export function GenerationToolCard({ status, isBulk = false, heading, children }: { status: GenerationToolStatus; isBulk?: boolean; heading: ReactNode; children: ReactNode }) { + const [open, setOpen] = useState(status !== "completed" || !isBulk); + + useEffect(() => { + if (status !== "completed") setOpen(true); + }, [status]); + + return <> +
+ {heading} + +
+ {open ? children : null} + ; +} diff --git a/web/src/components/ai/message-reasoning.tsx b/web/src/components/ai/message-reasoning.tsx new file mode 100644 index 000000000..3f74c923e --- /dev/null +++ b/web/src/components/ai/message-reasoning.tsx @@ -0,0 +1,41 @@ +import { useEffect, useRef, useState } from "react"; +import { ChevronDown, Sparkles } from "lucide-react"; + +import { AIMessageMarkdown } from "@/components/ai/ai-message-markdown"; + +export function MessageReasoning({ reasoning, isStreaming }: { reasoning: string; isStreaming: boolean }) { + const [open, setOpen] = useState(isStreaming); + const [durationSeconds, setDurationSeconds] = useState(); + const startedAtRef = useRef(undefined); + const autoClosedRef = useRef(false); + + useEffect(() => { + if (!isStreaming) return; + startedAtRef.current ??= Date.now(); + setOpen(true); + }, [isStreaming]); + + useEffect(() => { + if (isStreaming || !startedAtRef.current) return; + setDurationSeconds(Math.max(1, Math.ceil((Date.now() - startedAtRef.current) / 1000))); + startedAtRef.current = undefined; + }, [isStreaming]); + + useEffect(() => { + if (isStreaming || !open || autoClosedRef.current || durationSeconds === undefined) return; + const timer = window.setTimeout(() => { + setOpen(false); + autoClosedRef.current = true; + }, 1000); + return () => window.clearTimeout(timer); + }, [durationSeconds, isStreaming, open]); + + return
+ + {open ?
{reasoning}
: null} +
; +} diff --git a/web/src/components/canvas/canvas-resource-mention-textarea.tsx b/web/src/components/canvas/canvas-resource-mention-textarea.tsx index 10e8d7106..39aea8355 100644 --- a/web/src/components/canvas/canvas-resource-mention-textarea.tsx +++ b/web/src/components/canvas/canvas-resource-mention-textarea.tsx @@ -255,6 +255,7 @@ export const CanvasResourceMentionTextarea = forwardRef) => { + if (event.key === "Enter" && (event.nativeEvent.isComposing || composingRef.current)) return; if (mention && candidates.length) { if (event.key === "ArrowDown") { event.preventDefault(); @@ -339,7 +340,16 @@ export const CanvasResourceMentionTextarea = forwardRef { + composingRef.current = true; + props.onCompositionStart?.(event); + }} + onCompositionEnd={(event) => { + composingRef.current = false; + props.onCompositionEnd?.(event); + }} onKeyDown={(event) => { + if (event.key === "Enter" && (event.nativeEvent.isComposing || composingRef.current)) return; if (mention && candidates.length) { if (event.key === "ArrowDown") { event.preventDefault(); diff --git a/web/src/pages/create/index.tsx b/web/src/pages/create/index.tsx index 4e43be9ac..517450c11 100644 --- a/web/src/pages/create/index.tsx +++ b/web/src/pages/create/index.tsx @@ -4,6 +4,8 @@ import { ArrowDown, ArrowUp, Check, ChevronDown, Clapperboard, Clock3, Copy, Dow import { Link } from "react-router"; import { AIMessageMarkdown } from "@/components/ai/ai-message-markdown"; +import { GenerationToolCard, type GenerationToolStatus } from "@/components/ai/generation-tool-card"; +import { MessageReasoning } from "@/components/ai/message-reasoning"; import { AssetLibraryPickerModal, type AssetLibraryPickerItem } from "@/components/assets/asset-library-picker-modal"; import { CanvasResourceMentionTextarea } from "@/components/canvas/canvas-resource-mention-textarea"; import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; @@ -45,6 +47,7 @@ type CreationMessage = { role: "user" | "assistant"; mode?: CreationMode; content: string; + reasoning?: string; createdAt: string; status?: CreationStatus; model?: string; @@ -487,7 +490,10 @@ export default function CreatePage() { ? await buildTextMessageContent(item) : item.content, }))); - await requestImageQuestion(requestConfig, history, (text) => updateOriginAssistant((item) => ({ ...item, content: text })), { signal: requestLifecycle.signal }); + await requestImageQuestion(requestConfig, history, (text) => updateOriginAssistant((item) => ({ ...item, content: text })), { + signal: requestLifecycle.signal, + onReasoning: (reasoning) => updateOriginAssistant((item) => ({ ...item, reasoning })), + }); } else if (mode === "image") { const taskCount = Math.max(1, Math.min(imageProfile.maxOutputs, Math.floor(Number(count) || 1))); const settled = await runGenerationOperationOnce(retryContext?.clientOperationId, () => runBackendGenerationTaskBatch({ @@ -918,9 +924,10 @@ function CreationMessageView({ item, modelName, onRetryFailure, onCreateVariant, if (item.role === "user") return ; const mode = item.mode || "text"; const stateLabel = item.status === "pending" ? "生成中" : item.status === "cancelled" ? "已停止" : item.status === "error" ? "生成失败" : ""; + const heading = <>{mode === "image" ? "图像生成" : mode === "video" ? "视频生成" : "影策 AI"}{mode !== "text" ? {item.status === "pending" ? `影策正在生成${mode === "video" ? "视频" : "图像"}……` : item.status === "done" ? `你的${mode === "video" ? "视频" : "图像"}已创建` : null} : null}{modelName ? {modelName} : null}{item.createdAt ? : null}{stateLabel ? {stateLabel} : null}; + const toolStatus: GenerationToolStatus = item.status === "pending" ? "running" : item.status === "error" ? "error" : item.status === "cancelled" ? "cancelled" : "completed"; return
-
{mode === "image" ? "图像生成" : mode === "video" ? "视频生成" : "影策 AI"}{mode !== "text" ? {item.status === "pending" ? `影策正在生成${mode === "video" ? "视频" : "图像"}……` : item.status === "done" ? `你的${mode === "video" ? "视频" : "图像"}已创建` : null} : null}{modelName ? {modelName} : null}{item.createdAt ? : null}{stateLabel ? {stateLabel} : null}
- {mode === "text" ?
{item.content ? {item.content} : 正在生成…}
: } + {mode === "text" ? <>
{heading}
{item.reasoning ? : null}
{item.content ? {item.content} : 正在生成…}
: 1} heading={heading}>} {item.error && mode === "text" ?
{generationErrorMessage(item.error)}
: null}
; } diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index 802fdc9d8..2870f2414 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -45,6 +45,7 @@ export type ResponseFunctionTool = { export type ToolResponseResult = { content: string; toolCalls: ResponseToolCall[]; + reasoning?: string; }; type ToolChoice = "auto" | "required" | { type: "function"; name: string }; @@ -67,16 +68,16 @@ type ResponseApiPayload = { code?: number; msg?: string; }; -type ResponseStreamState = { buffer: string; text: string; payload?: ResponseApiPayload; error?: string }; +type ResponseStreamState = { buffer: string; text: string; reasoning: string; payload?: ResponseApiPayload; error?: string }; type ChatCompletionToolCall = { id?: string; type?: "function"; function?: { name?: string; arguments?: string } }; type ChatCompletionPayload = { - choices?: Array<{ message?: { content?: string | null; tool_calls?: ChatCompletionToolCall[] } }>; + choices?: Array<{ message?: { content?: string | null; reasoning_content?: string | null; tool_calls?: ChatCompletionToolCall[] } }>; error?: { message?: string }; code?: number; msg?: string; }; type ChatCompletionStreamToolCall = { id: string; name: string; arguments: string }; -type ChatCompletionStreamState = { buffer: string; text: string; toolCalls: Map; error?: string }; +type ChatCompletionStreamState = { buffer: string; text: string; reasoning: string; toolCalls: Map; error?: string }; type ImageApiResponse = { data?: Array>; @@ -86,6 +87,7 @@ type ImageApiResponse = { }; type GeminiPart = { text?: string; + thought?: boolean; inlineData?: { mimeType?: string; data?: string }; inline_data?: { mime_type?: string; mimeType?: string; data?: string }; fileData?: { mimeType?: string; fileUri?: string }; @@ -101,8 +103,8 @@ type GeminiPayload = { error?: { message?: string }; promptFeedback?: { blockReason?: string }; }; -type GeminiStreamState = { buffer: string; text: string; toolCalls: ResponseToolCall[]; error?: string }; -type RequestOptions = { signal?: AbortSignal; promptCacheKey?: string }; +type GeminiStreamState = { buffer: string; text: string; reasoning: string; toolCalls: ResponseToolCall[]; error?: string }; +type RequestOptions = { signal?: AbortSignal; promptCacheKey?: string; onReasoning?: (text: string) => void }; const QUALITY_BASE: Record = { low: 1024, @@ -427,7 +429,7 @@ function parseChatCompletionPayload(payload: ChatCompletionPayload): ToolRespons function: { name: call.function?.name || "", arguments: call.function?.arguments || "{}" }, })) .filter((call) => call.id && call.function.name); - return { content: message?.content || "", toolCalls }; + return { content: message?.content || "", toolCalls, ...(message?.reasoning_content ? { reasoning: message.reasoning_content } : {}) }; } function parseToolResponse(payload: ResponseApiPayload): ToolResponseResult { @@ -495,7 +497,7 @@ async function readJsonPayload(response: Response, fallback: string): Promise } } -function consumeResponseStreamBlock(block: string, state: ResponseStreamState, onDelta?: (text: string) => void) { +function consumeResponseStreamBlock(block: string, state: ResponseStreamState, onDelta?: (text: string) => void, onReasoning?: (text: string) => void) { const data = block .split(/\r?\n/) .filter((line) => line.startsWith("data:")) @@ -515,6 +517,14 @@ function consumeResponseStreamBlock(block: string, state: ResponseStreamState, o state.text = event.text; onDelta?.(state.text); } + if (["response.reasoning.delta", "response.reasoning_text.delta", "response.reasoning_summary_text.delta"].includes(type) && typeof event.delta === "string") { + state.reasoning += event.delta; + onReasoning?.(state.reasoning); + } + if (["response.reasoning.done", "response.reasoning_text.done", "response.reasoning_summary_text.done"].includes(type) && !state.reasoning && typeof event.text === "string") { + state.reasoning = event.text; + onReasoning?.(state.reasoning); + } if (type === "response.completed" && isRecord(event.response)) { state.payload = event.response as ResponseApiPayload; } else if (Array.isArray(event.output)) { @@ -522,17 +532,17 @@ function consumeResponseStreamBlock(block: string, state: ResponseStreamState, o } } -function consumeResponseStreamText(state: ResponseStreamState, text: string, onDelta?: (text: string) => void, flush = false) { +function consumeResponseStreamText(state: ResponseStreamState, text: string, onDelta?: (text: string) => void, onReasoning?: (text: string) => void, flush = false) { state.buffer += text; for (;;) { const match = state.buffer.match(/\r?\n\r?\n/); if (!match) break; const index = match.index ?? 0; - consumeResponseStreamBlock(state.buffer.slice(0, index), state, onDelta); + consumeResponseStreamBlock(state.buffer.slice(0, index), state, onDelta, onReasoning); state.buffer = state.buffer.slice(index + match[0].length); } if (flush && state.buffer.trim()) { - consumeResponseStreamBlock(state.buffer, state, onDelta); + consumeResponseStreamBlock(state.buffer, state, onDelta, onReasoning); state.buffer = ""; } } @@ -555,22 +565,22 @@ async function requestStreamingResponse(config: AiConfig, body: Record void) { +function consumeChatCompletionStreamBlock(block: string, state: ChatCompletionStreamState, onDelta?: (text: string) => void, onReasoning?: (text: string) => void) { const data = block .split(/\r?\n/) .filter((line) => line.startsWith("data:")) @@ -589,6 +599,11 @@ function consumeChatCompletionStreamBlock(block: string, state: ChatCompletionSt state.text += delta.content; onDelta?.(state.text); } + const reasoningDelta = stringValue(delta.reasoning_content) || stringValue(delta.reasoning) || stringValue(delta.reasoning_text); + if (reasoningDelta) { + state.reasoning += reasoningDelta; + onReasoning?.(state.reasoning); + } const chunks = Array.isArray(delta.tool_calls) ? delta.tool_calls : []; chunks.forEach((value, fallbackIndex) => { if (!isRecord(value)) return; @@ -603,17 +618,17 @@ function consumeChatCompletionStreamBlock(block: string, state: ChatCompletionSt }); } -function consumeChatCompletionStreamText(state: ChatCompletionStreamState, text: string, onDelta?: (text: string) => void, flush = false) { +function consumeChatCompletionStreamText(state: ChatCompletionStreamState, text: string, onDelta?: (text: string) => void, onReasoning?: (text: string) => void, flush = false) { state.buffer += text; for (;;) { const match = state.buffer.match(/\r?\n\r?\n/); if (!match) break; const index = match.index ?? 0; - consumeChatCompletionStreamBlock(state.buffer.slice(0, index), state, onDelta); + consumeChatCompletionStreamBlock(state.buffer.slice(0, index), state, onDelta, onReasoning); state.buffer = state.buffer.slice(index + match[0].length); } if (flush && state.buffer.trim()) { - consumeChatCompletionStreamBlock(state.buffer, state, onDelta); + consumeChatCompletionStreamBlock(state.buffer, state, onDelta, onReasoning); state.buffer = ""; } } @@ -629,24 +644,28 @@ async function requestStreamingChatCompletion(config: AiConfig, body: Record(response, "请求失败")); + if (!response.body || !contentType.includes("text/event-stream")) { + const result = parseChatCompletionPayload(await readJsonPayload(response, "请求失败")); + if (result.reasoning) options?.onReasoning?.(result.reasoning); + return result; + } const reader = response.body.getReader(); const decoder = new TextDecoder(); - const state: ChatCompletionStreamState = { buffer: "", text: "", toolCalls: new Map() }; + const state: ChatCompletionStreamState = { buffer: "", text: "", reasoning: "", toolCalls: new Map() }; for (;;) { const { done, value } = await reader.read(); if (done) break; - consumeChatCompletionStreamText(state, decoder.decode(value, { stream: true }), onDelta); + consumeChatCompletionStreamText(state, decoder.decode(value, { stream: true }), onDelta, options?.onReasoning); if (state.error) throw new Error(state.error); } - consumeChatCompletionStreamText(state, decoder.decode(), onDelta, true); + consumeChatCompletionStreamText(state, decoder.decode(), onDelta, options?.onReasoning, true); if (state.error) throw new Error(state.error); const toolCalls = Array.from(state.toolCalls.entries()) .sort(([left], [right]) => left - right) .map(([, call]) => ({ id: call.id, type: "function" as const, function: { name: call.name, arguments: call.arguments || "{}" } })) .filter((call) => call.id && call.function.name); - return { content: state.text, toolCalls }; + return { content: state.text, toolCalls, ...(state.reasoning ? { reasoning: state.reasoning } : {}) }; } function toGeminiBody(config: AiConfig, messages: ResponseInputMessage[], extra?: Record) { @@ -733,39 +752,41 @@ async function requestGeminiStreamingResponse(config: AiConfig, body: Record void, flush = false) { +function consumeGeminiStreamText(state: GeminiStreamState, text: string, onDelta?: (text: string) => void, onReasoning?: (text: string) => void, flush = false) { state.buffer += text; for (;;) { const match = state.buffer.match(/\r?\n\r?\n/); if (!match) break; const index = match.index ?? 0; - consumeGeminiStreamBlock(state.buffer.slice(0, index), state, onDelta); + consumeGeminiStreamBlock(state.buffer.slice(0, index), state, onDelta, onReasoning); state.buffer = state.buffer.slice(index + match[0].length); } if (flush && state.buffer.trim()) { - consumeGeminiStreamBlock(state.buffer, state, onDelta); + consumeGeminiStreamBlock(state.buffer, state, onDelta, onReasoning); state.buffer = ""; } } -function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDelta?: (text: string) => void) { +function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDelta?: (text: string) => void, onReasoning?: (text: string) => void) { const data = block .split(/\r?\n/) .filter((line) => line.startsWith("data:")) @@ -774,6 +795,10 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel .trim(); if (!data || data === "[DONE]") return; const result = parseGeminiToolResponse(JSON.parse(data) as GeminiPayload); + if (result.reasoning) { + state.reasoning += result.reasoning; + onReasoning?.(state.reasoning); + } if (result.content) { state.text += result.content; onDelta?.(state.text); @@ -784,7 +809,8 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult { validateGeminiPayload(payload); const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || []; - const content = parts.map((part) => part.text || "").join(""); + const content = parts.filter((part) => !part.thought).map((part) => part.text || "").join(""); + const reasoning = parts.filter((part) => part.thought).map((part) => part.text || "").join(""); const toolCalls = parts .map((part) => part.functionCall) .filter((call): call is NonNullable => Boolean(call?.name)) @@ -798,7 +824,7 @@ function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult { ...(thoughtSignature ? { thoughtSignature } : {}), }; }); - return { content, toolCalls }; + return { content, toolCalls, ...(reasoning ? { reasoning } : {}) }; } async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) { diff --git a/web/test/text-reasoning-stream.test.ts b/web/test/text-reasoning-stream.test.ts new file mode 100644 index 000000000..20111434c --- /dev/null +++ b/web/test/text-reasoning-stream.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { GenerationToolCard } from "../src/components/ai/generation-tool-card"; +import type { ModelProtocol } from "../src/lib/model-protocols"; +import { requestImageQuestion } from "../src/services/api/image"; +import { createModelChannel, defaultConfig, type AiConfig } from "../src/stores/use-config-store"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function reasoningConfig(protocol: ModelProtocol | undefined, apiFormat: "openai" | "gemini" = "openai"): AiConfig { + const channel = createModelChannel({ + id: "reasoning", + name: "Reasoning", + baseUrl: "https://reasoning.example/v1", + apiKey: "synthetic-test-key", + apiFormat, + interfaceType: apiFormat === "gemini" ? undefined : protocol, + models: ["reasoner"], + modelCosts: [{ model: "reasoner", capability: "text", ...(protocol ? { protocol } : {}), billingMode: "fixed_request", unitPriceMicrocredits: 0 }], + }); + return { ...defaultConfig, channels: [channel], model: "reasoning::reasoner", textModel: "reasoning::reasoner" }; +} + +function streamResponse(events: unknown[]) { + const body = `${events.map((event) => `data: ${JSON.stringify(event)}`).join("\n\n")}\n\n`; + return new Response(body, { headers: { "Content-Type": "text/event-stream" } }); +} + +async function requestWithReasoning(config: AiConfig, events: unknown[]) { + globalThis.fetch = (async () => streamResponse(events)) as typeof fetch; + let content = ""; + let reasoning = ""; + const result = await requestImageQuestion(config, [{ role: "user", content: "请回答" }], (value) => { content = value; }, { onReasoning: (value) => { reasoning = value; } }); + return { result, content, reasoning }; +} + +describe("text reasoning streams", () => { + test("separates Responses reasoning summaries from answer text", async () => { + const output = await requestWithReasoning(reasoningConfig("openai-response"), [ + { type: "response.reasoning_summary_text.delta", delta: "先分析" }, + { type: "response.output_text.delta", delta: "最终答案" }, + ]); + + expect(output).toEqual({ result: "最终答案", content: "最终答案", reasoning: "先分析" }); + }); + + test("separates Chat Completions reasoning_content from answer text", async () => { + const output = await requestWithReasoning(reasoningConfig("chat-completion"), [ + { choices: [{ delta: { reasoning_content: "推理过程" } }] }, + { choices: [{ delta: { content: "回答内容" } }] }, + ]); + + expect(output).toEqual({ result: "回答内容", content: "回答内容", reasoning: "推理过程" }); + }); + + test("does not duplicate Gemini thought parts into visible content", async () => { + const output = await requestWithReasoning(reasoningConfig(undefined, "gemini"), [ + { candidates: [{ content: { parts: [{ text: "内部思考", thought: true }, { text: "可见回答" }] } }] }, + ]); + + expect(output).toEqual({ result: "可见回答", content: "可见回答", reasoning: "内部思考" }); + }); + + test("keeps single results open and collapses completed bulk results on restore", () => { + const single = renderToStaticMarkup(React.createElement(GenerationToolCard, { status: "completed", heading: "图像生成" }, React.createElement("span", null, "单图结果"))); + const bulk = renderToStaticMarkup(React.createElement(GenerationToolCard, { status: "completed", isBulk: true, heading: "图像生成" }, React.createElement("span", null, "批量结果"))); + + expect(single).toContain("单图结果"); + expect(bulk).not.toContain("批量结果"); + }); +});