feat(canvas): 画布插件与视觉 - 合并 PR #289 并保留任务计费边界
This commit is contained in:
ddCat
@@ -1,5 +1,6 @@
|
||||
import type { UploadedFile } from "@/services/file-storage";
|
||||
import type { UploadedImage } from "@/services/image-storage";
|
||||
import type { ExternalAssetPickerReference } from "@/lib/plugins/plugin-types";
|
||||
import type { Asset, AudioAsset, ImageAsset, NewAsset } from "@/stores/use-asset-store";
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||
@@ -192,6 +193,49 @@ export function creationAttachmentFromAudioAsset(asset: AudioAsset): CreationAtt
|
||||
};
|
||||
}
|
||||
|
||||
export function creationAttachmentFromExternalAsset(reference: ExternalAssetPickerReference): CreationAttachment {
|
||||
const item = reference.item;
|
||||
const url = item.fileUrl || "";
|
||||
if (!url) throw new Error("“" + item.title + "”暂时无法读取，请先在 Eagle 中确认文件可用");
|
||||
const id = "external:" + reference.sourceId + ":" + item.id;
|
||||
const type = item.mimeType || (item.kind === "image" ? "image/png" : item.kind === "video" ? "video/mp4" : "audio/mpeg");
|
||||
if (item.kind === "image") {
|
||||
return {
|
||||
id,
|
||||
name: item.title || "素材图片",
|
||||
type,
|
||||
dataUrl: url,
|
||||
url,
|
||||
bytes: item.bytes,
|
||||
width: item.width,
|
||||
height: item.height,
|
||||
previewUrl: item.thumbnailUrl || url,
|
||||
};
|
||||
}
|
||||
if (item.kind === "video") {
|
||||
return {
|
||||
id,
|
||||
name: item.title || "素材视频",
|
||||
type,
|
||||
url,
|
||||
bytes: item.bytes,
|
||||
width: item.width,
|
||||
height: item.height,
|
||||
previewUrl: item.thumbnailUrl || url,
|
||||
};
|
||||
}
|
||||
if (item.kind === "audio") {
|
||||
return {
|
||||
id,
|
||||
name: item.title || "素材音频",
|
||||
type,
|
||||
url,
|
||||
bytes: item.bytes,
|
||||
previewUrl: item.thumbnailUrl || url,
|
||||
};
|
||||
}
|
||||
throw new Error("“" + item.title + "”不是可用于创作参考的媒体文件");
|
||||
}
|
||||
export function creationImageAsset({ title, uploaded, metadata }: { title: string; uploaded: UploadedImage; metadata?: Record<string, unknown> }): NewAsset {
|
||||
return {
|
||||
kind: "image",
|
||||
|
||||
@@ -16,6 +16,7 @@ import { createGenerationBatchRetryContexts, createGenerationRetryContext, runGe
|
||||
import { createClientId } from "@/lib/client-id";
|
||||
import { generationErrorCode, generationErrorMessage } from "@/lib/generation-error";
|
||||
import { useCopyText } from "@/hooks/use-copy-text";
|
||||
import { useExternalAssetSources } from "@/hooks/use-external-asset-sources";
|
||||
import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSize, imageResolutionChoices, imageResolutionOption, imageSizeForResolution, supportsImageResolutionPresets, type ImageResolutionChoice } from "@/lib/image-resolution-tiers";
|
||||
import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options";
|
||||
import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities";
|
||||
@@ -37,7 +38,7 @@ import { modelDisplayName, modelOptionName, resolveModelChannel, selectableModel
|
||||
import { useAssetStore, type Asset } from "@/stores/use-asset-store";
|
||||
import { useUserStore } from "@/stores/use-user-store";
|
||||
import { buildCreationMentionReferences, creationReferenceMetadata, displayCreationPrompt, expandCreationPrompt, reconcileCreationAttachmentLimit, removeCreationReferenceTokens, selectedCreationReferences, type CreationReference } from "./creation-references";
|
||||
import { creationAttachmentFromAsset, creationAttachmentFromAudio, creationAttachmentFromAudioAsset, creationAttachmentFromDocument, creationAttachmentFromImage, creationAttachmentFromVideo, creationAttachmentFromVideoAsset, creationAttachmentKind, creationAudioAsset, creationFileAccepted, creationImageAsset, creationMediaAspectRatio, creationUploadAccept, creationVideoAsset, splitCreationAttachments, type CreationAttachment } from "./creation-assets";
|
||||
import { creationAttachmentFromAsset, creationAttachmentFromAudio, creationAttachmentFromAudioAsset, creationAttachmentFromDocument, creationAttachmentFromExternalAsset, creationAttachmentFromImage, creationAttachmentFromVideo, creationAttachmentFromVideoAsset, creationAttachmentKind, creationAudioAsset, creationFileAccepted, creationImageAsset, creationMediaAspectRatio, creationUploadAccept, creationVideoAsset, splitCreationAttachments, type CreationAttachment } from "./creation-assets";
|
||||
|
||||
type CreationMode = "text" | "image" | "video";
|
||||
type CreationViewMode = "chat" | "storyboard";
|
||||
@@ -151,6 +152,7 @@ export default function CreatePage() {
|
||||
const [composingNextShot, setComposingNextShot] = useState(false);
|
||||
const [historyOpen, setHistoryOpen] = useState(false);
|
||||
const [libraryOpen, setLibraryOpen] = useState(false);
|
||||
const externalAssetSources = useExternalAssetSources(libraryOpen);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const composerFocusRef = useRef<HTMLTextAreaElement>(null);
|
||||
@@ -189,6 +191,13 @@ export default function CreatePage() {
|
||||
const imageProfile = useMemo(() => modelCapabilityConfigFor(config, selectedModel).image!, [config, selectedModel]);
|
||||
const videoProfile = useMemo(() => modelCapabilityConfigFor(config, selectedModel).video!, [config, selectedModel]);
|
||||
const maxReferences = mode === "video" ? videoProfile.operations.includes("image_to_video") ? videoProfile.references.maxImages : 0 : mode === "image" ? imageProfile.references.maxImages : 6;
|
||||
const referenceImageSize = useMemo(() => {
|
||||
const imageAttachments = attachments.filter(isImageAttachment);
|
||||
if (imageAttachments.length !== 1) return undefined;
|
||||
const { width, height } = imageAttachments[0];
|
||||
if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;
|
||||
return { width, height };
|
||||
}, [attachments]);
|
||||
const mentionReferences = useMemo(() => buildCreationMentionReferences(addedSkills, attachments, draftReferences), [addedSkills, attachments, draftReferences]);
|
||||
const isEmpty = !activeConversation?.messages.length;
|
||||
const pendingTaskKey = useMemo(() => pendingCreationTaskKey(conversations), [conversations]);
|
||||
@@ -344,17 +353,27 @@ export default function CreatePage() {
|
||||
}
|
||||
};
|
||||
|
||||
const libraryItems = useMemo<AssetLibraryPickerItem[]>(() => assets
|
||||
.filter((asset): asset is Extract<Asset, { kind: "image" | "video" | "audio" }> => asset.kind === "image" || asset.kind === "video" || asset.kind === "audio")
|
||||
.map((asset) => ({
|
||||
id: asset.id,
|
||||
title: asset.title,
|
||||
category: asset.category || "other",
|
||||
kindLabel: asset.kind === "video" ? "视频" : asset.kind === "audio" ? "音频" : "图片",
|
||||
asset,
|
||||
searchText: asset.tags.join(" "),
|
||||
disabledReason: mode === "image" && asset.kind !== "image" ? "图片创作仅支持参考图" : undefined,
|
||||
})), [assets, mode]);
|
||||
const externalLibraryItems = useMemo<AssetLibraryPickerItem[]>(
|
||||
() => externalAssetSources.items.map((item) => ({
|
||||
...item,
|
||||
disabledReason: mode === "image" && item.external?.item.kind !== "image" ? "图片创作仅支持参考图" : undefined,
|
||||
})),
|
||||
[externalAssetSources.items, mode],
|
||||
);
|
||||
const libraryItems = useMemo<AssetLibraryPickerItem[]>(() => [
|
||||
...assets
|
||||
.filter((asset): asset is Extract<Asset, { kind: "image" | "video" | "audio" }> => asset.kind === "image" || asset.kind === "video" || asset.kind === "audio")
|
||||
.map((asset) => ({
|
||||
id: asset.id,
|
||||
title: asset.title,
|
||||
category: asset.category || "other",
|
||||
kindLabel: asset.kind === "video" ? "视频" : asset.kind === "audio" ? "音频" : "图片",
|
||||
asset,
|
||||
searchText: asset.tags.join(" "),
|
||||
disabledReason: mode === "image" && asset.kind !== "image" ? "图片创作仅支持参考图" : undefined,
|
||||
})),
|
||||
...externalLibraryItems,
|
||||
], [assets, externalLibraryItems, mode]);
|
||||
const uploadCreationAsset = async (file: File) => {
|
||||
if (file.type.startsWith("video/")) {
|
||||
const uploaded = await uploadMediaFile(file, "create-upload");
|
||||
@@ -420,12 +439,14 @@ export default function CreatePage() {
|
||||
event.target.value = "";
|
||||
};
|
||||
|
||||
const handleLibrarySelect = (selected: Asset[]) => {
|
||||
const next = selected.flatMap((asset): CreationAttachment[] => {
|
||||
if (asset.kind === "image") return [creationAttachmentFromAsset(asset)];
|
||||
if (asset.kind === "video" && mode !== "image") return [creationAttachmentFromVideoAsset(asset)];
|
||||
if (asset.kind === "audio" && mode !== "image") return [creationAttachmentFromAudioAsset(asset)];
|
||||
return [];
|
||||
const handleLibrarySelect = (selectedIds: string[]) => {
|
||||
const next = selectedIds.flatMap((id): CreationAttachment[] => {
|
||||
const asset = assets.find((item) => item.id === id);
|
||||
if (asset?.kind === "image") return [creationAttachmentFromAsset(asset)];
|
||||
if (asset?.kind === "video" && mode !== "image") return [creationAttachmentFromVideoAsset(asset)];
|
||||
if (asset?.kind === "audio" && mode !== "image") return [creationAttachmentFromAudioAsset(asset)];
|
||||
const external = libraryItems.find((item) => item.id === id)?.external;
|
||||
return external ? [creationAttachmentFromExternalAsset(external)] : [];
|
||||
});
|
||||
if (!next.length) return;
|
||||
setAttachments((current) => [...current.filter((item) => !next.some((candidate) => candidate.id === item.id)), ...next].slice(0, maxReferences));
|
||||
@@ -787,6 +808,7 @@ export default function CreatePage() {
|
||||
setPrompt,
|
||||
busy,
|
||||
attachments,
|
||||
referenceImageSize,
|
||||
maxReferences,
|
||||
references: mentionReferences,
|
||||
onRemoveAttachment: removeAttachment,
|
||||
@@ -885,11 +907,12 @@ export default function CreatePage() {
|
||||
<AssetLibraryPickerModal
|
||||
open={libraryOpen}
|
||||
items={libraryItems}
|
||||
categoryLabels={creationAssetCategoryLabels}
|
||||
initialSelectedIds={attachments.filter((item) => item.id.startsWith("asset:")).map((item) => item.id.slice(6))}
|
||||
upload={{ accept: creationUploadAccept(mode), description: mode === "text" ? "支持图片、视频、音频和常用文档；媒体会保存到素材库" : `支持图片${mode === "video" ? "、视频和音频" : ""}，上传后保存到素材库`, onUpload: uploadLibraryAssets }}
|
||||
categoryLabels={{ ...creationAssetCategoryLabels, ...externalAssetSources.categoryLabels }}
|
||||
folders={externalAssetSources.folders}
|
||||
initialSelectedIds={attachments.flatMap((item) => item.id.startsWith("asset:") ? [item.id.slice(6)] : item.id.startsWith("external:") ? [item.id] : [])}
|
||||
upload={{ accept: creationUploadAccept(mode), description: mode === "text" ? "支持图片、视频、音频和常用文档；媒体会保存到素材库" : `支持图片${mode === "video" ? "、视频和音频" : ""}，上传后保存到素材库`, onUpload: uploadLibraryAssets, external: { accept: "image/*", description: "写入当前 Eagle 文件夹；Eagle 当前支持图片文件", onUpload: (files, folderId) => externalAssetSources.uploadExternalFiles(files, folderId) } }}
|
||||
onClose={() => setLibraryOpen(false)}
|
||||
onConfirm={(ids) => handleLibrarySelect(assets.filter((asset) => ids.includes(asset.id)))}
|
||||
onConfirm={handleLibrarySelect}
|
||||
/>
|
||||
</>;
|
||||
}
|
||||
@@ -1051,6 +1074,7 @@ type ComposerProps = {
|
||||
setPrompt: (value: string) => void;
|
||||
busy: boolean;
|
||||
attachments: CreationAttachment[];
|
||||
referenceImageSize?: { width: number; height: number };
|
||||
maxReferences: number;
|
||||
references: CreationReference[];
|
||||
onRemoveAttachment: (id: string) => void;
|
||||
@@ -1183,6 +1207,11 @@ function GenerationSettingsMenu(props: ComposerProps) {
|
||||
? Array.from(new Set(imageResolutionOptions.filter((item) => !activeImageResolution || item.tier === activeImageResolution.tier).map((item) => item.ratio)))
|
||||
: mergedProfile.size.values.length ? mergedProfile.size.values : ratioOptions.map((item) => item.value);
|
||||
const ratios = props.mode === "video" ? props.videoProfile.ratios : imageRatios;
|
||||
const referenceImageSize = props.mode === "image" && mergedProfile.size.allowCustom ? props.referenceImageSize : undefined;
|
||||
const referenceImageSizeValue = referenceImageSize ? String(referenceImageSize.width) + "x" + String(referenceImageSize.height) : "";
|
||||
const referenceImageSizeLabel = referenceImageSize ? String(referenceImageSize.width) + " × " + String(referenceImageSize.height) : "";
|
||||
const referenceImageSizeRatio = referenceImageSize ? String(referenceImageSize.width) + ":" + String(referenceImageSize.height) : "";
|
||||
const referenceImageSizeSelected = Boolean(referenceImageSizeValue && props.ratio === referenceImageSizeValue);
|
||||
const resolutions = props.mode === "video" ? props.videoProfile.resolutions.map((value) => ({ value: value.replace(/p$/i, ""), label: videoResolutionLabel(value) })) : resolutionOptions;
|
||||
const selectImageRatio = (nextRatio: string) => {
|
||||
if (!usesImageResolutionPicker || activeImageResolutionChoice === "auto") {
|
||||
@@ -1199,15 +1228,20 @@ function GenerationSettingsMenu(props: ComposerProps) {
|
||||
const nextSize = imageSizeForResolution(imageResolutionOptions, choice, activeImageRatio) || imageResolutionOptions.find((item) => item.tier === choice)?.size;
|
||||
if (nextSize) props.setRatio(nextSize);
|
||||
};
|
||||
const selectReferenceImageSize = () => {
|
||||
if (!referenceImageSizeValue) return;
|
||||
props.setRatio(referenceImageSizeValue);
|
||||
setCustomRatioOpen(false);
|
||||
};
|
||||
const videoResolutionSupported = props.mode === "video" && resolutions.length > 0;
|
||||
const imageSummary = [
|
||||
...(mergedProfile.size.parameter !== "none" ? [usesImageResolutionPicker ? formatImageResolutionSize(props.ratio, imageResolutionOptions) : props.ratio] : []),
|
||||
...(mergedProfile.size.parameter !== "none" ? [referenceImageSizeSelected ? referenceImageSizeLabel : usesImageResolutionPicker ? formatImageResolutionSize(props.ratio, imageResolutionOptions) : props.ratio] : []),
|
||||
...(props.imageProfile.quality.supported ? [qualityLabel] : []),
|
||||
...(props.imageProfile.maxOutputs > 1 ? [props.count] : []),
|
||||
].join(" · ");
|
||||
const summary = props.mode === "video" ? [props.ratio, ...(videoResolutionSupported ? [videoResolutionLabel(props.videoQuality)] : [])].join(" · ") : imageSummary;
|
||||
const panel = <div className="creation-parameter-menu">
|
||||
{props.mode === "video" || mergedProfile.size.parameter !== "none" ? <SettingSection title="画幅" value={props.mode === "image" && usesImageResolutionPicker ? activeImageRatio : props.ratio}><div className="creation-parameter-content"><div className="creation-choice-grid is-ratio">{ratios.map((value) => { const selected = props.mode === "image" && usesImageResolutionPicker ? value === activeImageRatio : value === props.ratio; return <button key={value} type="button" aria-pressed={selected} className={selected ? "is-selected" : ""} onClick={() => { if (props.mode === "image") selectImageRatio(value); else props.setRatio(value); setCustomRatioOpen(false); }}><span className="creation-ratio-preview"><span style={ratioPreviewStyle(value)} /></span><span>{value}</span></button>; })}</div>{props.mode !== "video" && mergedProfile.size.allowCustom && (customRatioOpen ? <label className="creation-custom-value"><span>宽 x 高</span><input value={props.ratio} onFocus={(event) => event.currentTarget.select()} onChange={(event) => props.setRatio(event.target.value)} placeholder="1920x1080 或 2:1" aria-label="自定义图片尺寸或比例" /></label> : <button type="button" className="creation-custom-trigger" onClick={() => setCustomRatioOpen(true)}><Plus />输入自定义尺寸</button>)}</div></SettingSection> : null}
|
||||
{props.mode === "video" || mergedProfile.size.parameter !== "none" ? <SettingSection title="画幅" value={referenceImageSizeSelected ? referenceImageSizeLabel : props.mode === "image" && usesImageResolutionPicker ? activeImageRatio : props.ratio}><div className="creation-parameter-content"><div className="creation-choice-grid is-ratio">{referenceImageSizeValue ? <button type="button" aria-pressed={referenceImageSizeSelected} aria-label={"使用参考图尺寸 " + referenceImageSizeLabel} title={"使用参考图尺寸 " + referenceImageSizeLabel} className={"creation-reference-size-choice" + (referenceImageSizeSelected ? " is-selected" : "")} onClick={selectReferenceImageSize}><span className="creation-ratio-preview"><span style={ratioPreviewStyle(referenceImageSizeRatio)} /></span><span>参考图</span></button> : null}{ratios.map((value) => { const selected = props.mode === "image" && usesImageResolutionPicker ? value === activeImageRatio : value === props.ratio; return <button key={value} type="button" aria-pressed={selected} className={selected ? "is-selected" : ""} onClick={() => { if (props.mode === "image") selectImageRatio(value); else props.setRatio(value); setCustomRatioOpen(false); }}><span className="creation-ratio-preview"><span style={ratioPreviewStyle(value)} /></span><span>{value}</span></button>; })}</div>{props.mode !== "video" && mergedProfile.size.allowCustom && (customRatioOpen ? <label className="creation-custom-value"><span>宽 x 高</span><input value={props.ratio} onFocus={(event) => event.currentTarget.select()} onChange={(event) => props.setRatio(event.target.value)} placeholder="1920x1080 或 2:1" aria-label="自定义图片尺寸或比例" /></label> : <button type="button" className="creation-custom-trigger" onClick={() => setCustomRatioOpen(true)}><Plus />输入自定义尺寸</button>)}</div></SettingSection> : null}
|
||||
{props.mode === "video" ? (videoResolutionSupported ? <SettingSection title="清晰度" value={videoResolutionLabel(props.videoQuality)}><div className="creation-choice-grid is-resolution">{resolutions.map((option) => <button key={option.value} type="button" aria-pressed={option.value === props.videoQuality} className={option.value === props.videoQuality ? "is-selected" : ""} onClick={() => props.setVideoQuality(option.value)}>{option.label}</button>)}</div></SettingSection> : null) : <>
|
||||
{imageResolutionChoiceOptions.length ? <SettingSection title="分辨率" value={activeImageResolutionChoice === "auto" ? "自动" : activeImageResolutionChoice.toUpperCase()}><div className="creation-choice-grid is-resolution">{imageResolutionChoiceOptions.map((choice) => <button key={choice} type="button" aria-pressed={choice === activeImageResolutionChoice} className={choice === activeImageResolutionChoice ? "is-selected" : ""} onClick={() => selectImageResolution(choice)}>{choice === "auto" ? "自动" : choice.toUpperCase()}</button>)}</div></SettingSection> : null}
|
||||
{props.imageProfile.quality.supported ? <SettingSection title={activeQualityOptions.some((item) => item.value === "1k" || item.value === "2k") ? "分辨率" : "图片质量"} value={qualityLabel}><div className="creation-choice-grid is-quality">{activeQualityOptions.map((option) => <button key={option.value} type="button" aria-pressed={option.value === props.quality} className={option.value === props.quality ? "is-selected" : ""} onClick={() => props.setQuality(option.value)}><span>{option.label}</span><small>{option.description}</small></button>)}</div></SettingSection> : null}
|
||||
@@ -1509,7 +1543,7 @@ function isVideoAttachment(attachment: CreationAttachment): attachment is Creati
|
||||
return creationAttachmentKind(attachment) === "video";
|
||||
}
|
||||
|
||||
function isImageAttachment(attachment: CreationAttachment): attachment is CreationAttachment & { dataUrl: string } {
|
||||
function isImageAttachment(attachment: CreationAttachment): attachment is CreationAttachment & { dataUrl: string; width?: number; height?: number } {
|
||||
return creationAttachmentKind(attachment) === "image";
|
||||
}
|
||||
|
||||
@@ -1607,7 +1641,8 @@ function formatConversationTime(value: string) {
|
||||
|
||||
function ratioPreviewStyle(value: string) {
|
||||
const [width, height] = value.replace("x", ":").split(":").map(Number);
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return { width: 14, height: 14 };
|
||||
const scale = Math.min(28 / width, 20 / height);
|
||||
return { width: Math.max(8, width * scale), height: Math.max(8, height * scale) };
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) return { width: 10, height: 10 };
|
||||
// 画幅容器的可用空间是 14×10；同时计算宽高，避免 CSS 的 max-width/max-height 把宽银幕比例压扁。
|
||||
const scale = Math.min(14 / width, 10 / height);
|
||||
return { width: Math.max(4, Math.round(width * scale)), height: Math.max(4, Math.round(height * scale)) };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user