diff --git a/web/src/components/canvas/canvas-resource-mention-textarea.tsx b/web/src/components/canvas/canvas-resource-mention-textarea.tsx index 03cecbc93..bd540b0bb 100644 --- a/web/src/components/canvas/canvas-resource-mention-textarea.tsx +++ b/web/src/components/canvas/canvas-resource-mention-textarea.tsx @@ -36,11 +36,12 @@ type Props = Omit, "onChange" | "val onSubmit?: () => void; containerClassName?: string; highlightLabels?: boolean; + mentionMenuWidth?: number; onContentSizeChange?: (height: number) => void; }; export const CanvasResourceMentionTextarea = forwardRef(function CanvasResourceMentionTextarea( - { value, references, onChange, onSubmit, onKeyDown, className, containerClassName, style, highlightLabels = true, onContentSizeChange, ...props }, + { value, references, onChange, onSubmit, onKeyDown, className, containerClassName, style, highlightLabels = true, mentionMenuWidth = 256, onContentSizeChange, ...props }, forwardedRef, ) { const theme = canvasThemes[useThemeStore((state) => state.theme)]; @@ -182,7 +183,7 @@ export const CanvasResourceMentionTextarea = forwardRef : null; + const menu = mention && candidates.length && menuAnchor ? : null; if (useRichEditor) { return ( @@ -388,11 +389,11 @@ function createInlinePreview(reference: CanvasResourceReference) { return fallback; } -function MentionMenu({ anchor, references, activeIndex, theme, onSelect }: { anchor: HTMLElement; references: CanvasResourceReference[]; activeIndex: number; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; onSelect: (reference: CanvasResourceReference) => void }) { +function MentionMenu({ anchor, references, activeIndex, theme, preferredWidth, onSelect }: { anchor: HTMLElement; references: CanvasResourceReference[]; activeIndex: number; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; preferredWidth: number; onSelect: (reference: CanvasResourceReference) => void }) { const selectedRef = useRef(false); const rect = anchor.getBoundingClientRect(); const boundary = anchor.closest(".ant-modal-content")?.getBoundingClientRect() || { left: 8, top: 8, right: window.innerWidth - 8, bottom: window.innerHeight - 8 }; - const menuWidth = 256; + const menuWidth = Math.min(preferredWidth, Math.max(0, boundary.right - boundary.left - 16)); const maxMenuHeight = 224; const gap = 6; const left = clamp(rect.left, boundary.left + 8, boundary.right - menuWidth - 8); @@ -411,8 +412,8 @@ function MentionMenu({ anchor, references, activeIndex, theme, onSelect }: { anc return createPortal(
event.stopPropagation()} @@ -436,7 +437,7 @@ function MentionMenu({ anchor, references, activeIndex, theme, onSelect }: { anc > - {reference.label} + {reference.label} {reference.kind !== "skill" ? {reference.text || reference.title} : null} diff --git a/web/src/pages/create/creation-references.ts b/web/src/pages/create/creation-references.ts index 2a32bb5ba..d91360bc4 100644 --- a/web/src/pages/create/creation-references.ts +++ b/web/src/pages/create/creation-references.ts @@ -1,27 +1,19 @@ import { buildSkillMentionReferences, renderSkillPrompt } from "@/lib/canvas/canvas-skill-mentions"; import { canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; import type { Skill } from "@/services/api/skills"; -import type { Asset } from "@/stores/use-asset-store"; import type { ReferenceImage } from "@/types/image"; -import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; - -type CreationMode = "text" | "image" | "video"; -type MentionableAsset = Extract; export type CreationReference = CanvasResourceReference & { - asset?: MentionableAsset; attachmentId?: string; }; -export function buildCreationMentionReferences(assets: Asset[], skills: Skill[], mode: CreationMode, attachments: ReferenceImage[] = [], snapshots: CreationReference[] = []) { +export function buildCreationMentionReferences(skills: Skill[], attachments: ReferenceImage[] = [], snapshots: CreationReference[] = []) { const attachmentReferences = attachments.map(attachmentReference); - const assetReferences = assets - .filter((asset): asset is MentionableAsset => isMentionableAsset(asset) && supportsMode(asset.kind, mode)) - .map(assetReference); const skillReferences = buildSkillMentionReferences(skills) as CreationReference[]; - const current = [...attachmentReferences, ...assetReferences, ...skillReferences]; + const current = [...attachmentReferences, ...skillReferences]; const currentIDs = new Set(current.map((reference) => reference.id)); - return [...current, ...snapshots.filter((reference) => !currentIDs.has(reference.id))].map((reference) => ({ ...reference, active: true })); + const restored = snapshots.filter((reference) => reference.kind === "skill" && !currentIDs.has(reference.id)); + return [...current, ...restored].map((reference) => ({ ...reference, active: true })); } export function selectedCreationReferences(prompt: string, references: CreationReference[]) { @@ -39,7 +31,6 @@ export function expandCreationPrompt(prompt: string, references: CreationReferen const contexts: string[] = []; const mediaMappings: string[] = []; const attachmentPositions = new Map(attachments.map((attachment, index) => [attachment.id, index + 1])); - const counts = { image: attachments.length, video: 0, audio: 0 }; references.forEach((reference) => { if (reference.kind === "skill" && reference.skill) { contexts.push(renderSkillPrompt(reference.skill)); @@ -50,113 +41,28 @@ export function expandCreationPrompt(prompt: string, references: CreationReferen mediaMappings.push(`- @${reference.label}：参考图片 ${position || 1}`); return; } - const asset = reference.asset; - if (!asset) return; - if (asset.kind === "text") { - contexts.push(`【引用资源：${reference.label}】\n${asset.data.content}`); - return; - } - counts[asset.kind] += 1; - const label = asset.kind === "image" ? "参考图片" : asset.kind === "video" ? "参考视频" : "参考音频"; - mediaMappings.push(`- @${reference.label}：${label} ${counts[asset.kind]}`); }); if (mediaMappings.length) contexts.push(`【资源对应关系】\n${mediaMappings.join("\n")}`); return [...contexts, `【创作要求】\n${visiblePrompt}`].filter(Boolean).join("\n\n"); } -export function creationReferenceImages(references: CreationReference[]): ReferenceImage[] { - return references.flatMap((reference) => { - const asset = reference.asset; - if (!asset || asset.kind !== "image") return []; - return [{ - id: asset.id, - name: asset.title, - type: asset.data.mimeType || "image/png", - dataUrl: asset.data.dataUrl, - url: asset.data.dataUrl, - storageKey: asset.data.storageKey, - }]; - }); -} - -export function creationReferenceVideos(references: CreationReference[]): ReferenceVideo[] { - return references.flatMap((reference) => { - const asset = reference.asset; - if (!asset || asset.kind !== "video") return []; - return [{ - id: asset.id, - name: asset.title, - type: asset.data.mimeType || "video/mp4", - url: asset.data.url, - storageKey: asset.data.storageKey, - bytes: asset.data.bytes, - width: asset.data.width, - height: asset.data.height, - durationMs: asset.data.durationMs, - }]; - }); -} - -export function creationReferenceAudios(references: CreationReference[]): ReferenceAudio[] { - return references.flatMap((reference) => { - const asset = reference.asset; - if (!asset || asset.kind !== "audio") return []; - return [{ - id: asset.id, - name: asset.title, - type: asset.data.mimeType || "audio/mpeg", - url: asset.data.url, - storageKey: asset.data.storageKey, - durationMs: asset.data.durationMs, - }]; - }); -} - export function creationReferenceMetadata(references: CreationReference[]) { return { - assetIds: references.flatMap((reference) => reference.asset ? [reference.asset.id] : []), skillIds: references.flatMap((reference) => reference.skill?.skill_id ? [reference.skill.skill_id] : []), }; } -function assetReference(asset: MentionableAsset): CreationReference { - const kindLabel = asset.kind === "image" ? "图片资源" : asset.kind === "video" ? "视频资源" : asset.kind === "audio" ? "音频资源" : "文本资源"; - const previewUrl = asset.coverUrl || (asset.kind === "image" ? asset.data.dataUrl : asset.kind === "video" ? asset.data.url : ""); - return { - id: `asset:${asset.id}`, - nodeId: `asset:${asset.id}`, - kind: asset.kind, - label: asset.title.trim() || kindLabel, - title: kindLabel, - previewUrl: previewUrl || undefined, - storageKey: asset.kind === "text" ? undefined : asset.data.storageKey, - text: asset.kind === "text" ? asset.data.content : undefined, - active: true, - asset, - }; -} - function attachmentReference(attachment: ReferenceImage, index: number): CreationReference { - const label = attachment.name.replace(/\.[^.]+$/, "").trim() || `上传图片${index + 1}`; return { id: `upload:${attachment.id}`, nodeId: `upload:${attachment.id}`, kind: "image", - label, - title: "本次上传", + label: `图片${index + 1}`, + title: "当前参考内容", previewUrl: attachment.dataUrl || attachment.url, storageKey: attachment.storageKey, active: true, attachmentId: attachment.id, }; } - -function isMentionableAsset(asset: Asset): asset is MentionableAsset { - return asset.kind === "text" || asset.kind === "image" || asset.kind === "video" || asset.kind === "audio"; -} - -function supportsMode(kind: MentionableAsset["kind"], mode: CreationMode) { - if (kind === "text" || kind === "image") return true; - return mode === "video"; -} diff --git a/web/src/pages/create/index.tsx b/web/src/pages/create/index.tsx index 87040a8fa..23c2c6b55 100644 --- a/web/src/pages/create/index.tsx +++ b/web/src/pages/create/index.tsx @@ -13,10 +13,9 @@ import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options"; import { runBackendGenerationTask, runBackendGenerationTaskBatch } from "@/services/api/generation-task"; import { requestImageQuestion } from "@/services/api/image"; import { listAddedSkills, type Skill } from "@/services/api/skills"; -import { useAssetStore } from "@/stores/use-asset-store"; import { modelDisplayName, modelOptionName, selectableModelsByCapability, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store"; import type { ReferenceImage } from "@/types/image"; -import { buildCreationMentionReferences, creationReferenceAudios, creationReferenceImages, creationReferenceMetadata, creationReferenceVideos, displayCreationPrompt, expandCreationPrompt, selectedCreationReferences, type CreationReference } from "./creation-references"; +import { buildCreationMentionReferences, creationReferenceMetadata, displayCreationPrompt, expandCreationPrompt, selectedCreationReferences, type CreationReference } from "./creation-references"; type CreationMode = "text" | "image" | "video"; type CreationStatus = "streaming" | "pending" | "done" | "error" | "cancelled"; @@ -70,7 +69,6 @@ export default function CreatePage() { const { message: toast } = App.useApp(); const config = useEffectiveConfig(); const updateConfig = useConfigStore((state) => state.updateConfig); - const assets = useAssetStore((state) => state.assets); const [conversations, setConversations] = useState([]); const [activeId, setActiveId] = useState(""); const [hydrated, setHydrated] = useState(false); @@ -97,7 +95,7 @@ export default function CreatePage() { [activeId, conversations], ); const selectedModel = mode === "text" ? config.textModel : mode === "image" ? config.imageModel : config.videoModel; - const mentionReferences = useMemo(() => buildCreationMentionReferences(assets, addedSkills, mode, attachments, draftReferences), [addedSkills, assets, attachments, draftReferences, mode]); + const mentionReferences = useMemo(() => buildCreationMentionReferences(addedSkills, attachments, draftReferences), [addedSkills, attachments, draftReferences]); const isEmpty = !activeConversation?.messages.length; useEffect(() => { @@ -153,10 +151,6 @@ export default function CreatePage() { }, [updateActive]); const selectMode = (next: CreationMode) => { - if (next !== "video") { - const unsupported = mentionReferences.filter((reference) => reference.kind === "video" || reference.kind === "audio"); - if (unsupported.length) setPrompt((current) => removeReferenceTokens(current, unsupported)); - } setMode(next); const nextModels = selectableModelsByCapability(config, next); const current = next === "text" ? config.textModel : next === "image" ? config.imageModel : config.videoModel; @@ -195,9 +189,6 @@ export default function CreatePage() { const settings = { ratio, seconds, quality, videoQuality, count }; const references = selectedCreationReferences(text, mentionReferences); const expandedPrompt = expandCreationPrompt(text, references, attachments); - const referencedImages = creationReferenceImages(references); - const referencedVideos = creationReferenceVideos(references); - const referencedAudios = creationReferenceAudios(references); const referenceMetadata = creationReferenceMetadata(references); followLatestMessageRef.current = true; const userMessage = newMessage("user", text, { mode, model: selectedModel, attachments, references, settings }); @@ -230,7 +221,7 @@ export default function CreatePage() { mode: "image", prompt: expandedPrompt, config: { ...requestConfig, count: "1" }, - referenceImages: [...attachments, ...referencedImages], + referenceImages: attachments, signal: controller.signal, metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, ...referenceMetadata }, count: taskCount, @@ -249,11 +240,9 @@ export default function CreatePage() { mode: "video", prompt: expandedPrompt, config: requestConfig, - referenceImages: [...attachments, ...referencedImages], - referenceVideos: referencedVideos, - referenceAudios: referencedAudios, + referenceImages: attachments, signal: controller.signal, - metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: attachments.length || referencedImages.length ? "image_to_video" : "text_to_video", ...referenceMetadata }, + metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: attachments.length ? "image_to_video" : "text_to_video", ...referenceMetadata }, }); if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频"); const videoUrl = result.video.dataUrl; @@ -476,7 +465,7 @@ function CreationComposer(props: ComposerProps) {
- + {props.attachments.length ?
{props.attachments.map((item) =>
{item.name}
)}
: null}
@@ -564,7 +553,7 @@ function conversationPreviewMessage(conversation: CreationConversation) { function buildTextMessageContent(item: CreationMessage) { const content = expandCreationPrompt(item.content, item.references || [], item.attachments || []); - const images = [...(item.attachments || []), ...creationReferenceImages(item.references || [])]; + const images = item.attachments || []; if (!images.length) return content; return [{ type: "text" as const, text: content }, ...images.map((image) => ({ type: "image_url" as const, image_url: { url: image.dataUrl || image.url || "" } }))]; } diff --git a/web/src/services/api/generation-task.ts b/web/src/services/api/generation-task.ts index 9de4455a6..b46d95d46 100644 --- a/web/src/services/api/generation-task.ts +++ b/web/src/services/api/generation-task.ts @@ -105,9 +105,9 @@ async function createAndWaitGenerationTask({ projectId, mode, prompt, config, re } async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAudio) { - if (resourceIdFromStorageKey(media.storageKey)) return { ...media, dataUrl: "" }; + if (resourceIdFromStorageKey(media.storageKey)) return backendMediaReference(media, { storageKey: media.storageKey }); const url = media.url || ""; - if (/^https?:\/\//i.test(url)) return media; + if (/^https?:\/\//i.test(url)) return backendMediaReference(media, { url }); let blob: Blob | null = null; if (media.storageKey) blob = await getMediaBlob(media.storageKey); if (!blob && (url.startsWith("blob:") || url.startsWith("data:"))) blob = await (await fetch(url)).blob(); @@ -115,25 +115,52 @@ async function prepareBackendMediaReference(media: ReferenceVideo | ReferenceAud try { const kind: "video" | "audio" | "file" = blob.type.startsWith("video/") ? "video" : blob.type.startsWith("audio/") ? "audio" : "file"; const resource = await uploadResourceFile(blob, kind, { fileName: media.name, width: "width" in media ? media.width : undefined, height: "height" in media ? media.height : undefined, durationMs: media.durationMs }); - return { ...media, url: resource.publicUrl || `/api/resources/${resource.id}/file`, storageKey: resourceStorageKey(resource.id), dataUrl: "", type: resource.mimeType || media.type || blob.type }; + return backendMediaReference(media, { storageKey: resourceStorageKey(resource.id), type: resource.mimeType || media.type || blob.type }); } catch (error) { throw new Error(error instanceof Error ? `参考媒体上传失败：${error.message}` : "参考媒体上传失败"); } } async function prepareBackendImageReference(image: ReferenceImage) { - if (resourceIdFromStorageKey(image.storageKey)) return { ...image, dataUrl: "" }; - if (/^https?:\/\//i.test(image.dataUrl)) return { ...image, url: image.url || image.dataUrl, dataUrl: "" }; - const blob = image.storageKey ? await getImageBlob(image.storageKey) : image.dataUrl ? await (await fetch(image.dataUrl)).blob() : null; + if (resourceIdFromStorageKey(image.storageKey)) return backendImageReference(image, { storageKey: image.storageKey }); + const sourceUrl = image.url || image.dataUrl; + if (/^https?:\/\//i.test(sourceUrl)) return backendImageReference(image, { url: sourceUrl }); + const blob = image.storageKey ? await getImageBlob(image.storageKey) : sourceUrl ? await (await fetch(sourceUrl)).blob() : null; if (!blob) throw new Error("参考图片尚未保存，请重新上传后再生成"); try { const resource = await uploadResourceFile(blob, "image", { fileName: image.name }); - return { ...image, dataUrl: "", url: resource.publicUrl || `/api/resources/${resource.id}/file`, storageKey: resourceStorageKey(resource.id), type: resource.mimeType || image.type || blob.type }; + return backendImageReference(image, { storageKey: resourceStorageKey(resource.id), type: resource.mimeType || image.type || blob.type }); } catch (error) { throw new Error(error instanceof Error ? `参考图片上传失败：${error.message}` : "参考图片上传失败"); } } +// 任务输入只允许后端协议字段，避免把 previewUrl 等页面态 Data URL 带入强校验写路径。 +function backendImageReference(image: ReferenceImage, override: Partial): ReferenceImage { + return { + id: image.id, + name: image.name, + type: override.type || image.type, + dataUrl: "", + url: override.url, + storageKey: override.storageKey, + }; +} + +function backendMediaReference(media: T, override: Partial): T { + return { + id: media.id, + name: media.name, + type: override.type || media.type, + url: override.url || "", + storageKey: override.storageKey, + ...("bytes" in media && media.bytes ? { bytes: media.bytes } : {}), + ...("width" in media && media.width ? { width: media.width } : {}), + ...("height" in media && media.height ? { height: media.height } : {}), + ...(media.durationMs ? { durationMs: media.durationMs } : {}), + } as T; +} + export function backendProviderConfig(config: AiConfig) { const requestConfig = resolveModelRequestConfig(config, config.model); return {