From 6a7605ec335034168127bd1112edc6f9d2a4d737 Mon Sep 17 00:00:00 2001 From: Rou <76741805+Rou325@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:28:19 +0800 Subject: [PATCH] =?UTF-8?q?fix(media,canvas):=20=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=8F=82=E8=80=83=E7=B4=A0=E6=9D=90=E4=BC=98?= =?UTF-8?q?=E5=85=88=20URL=EF=BC=8C=E5=B9=B6=E5=90=8C=E6=AD=A5=E6=B8=85?= =?UTF-8?q?=E7=90=86=E5=9B=BE=E7=89=87=E5=BC=95=E7=94=A8=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E8=AF=8D=20(#434)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(media): 对象存储参考素材 - 协议支持 URL 时优先发送签名链接 开启对象存储后，支持 URL 的上游协议优先发送 OSS/COS/七牛/S3 签名链接；必须传字节的协议仍走 Base64 或 multipart。本地资源回退字节，签发失败直接报错，不自动重发生成。 * fix(canvas): 图片引用清理 - 取消引用时同步移除提示词引用 --- backend/internal/service/provider.go | 97 ++++++++++++++----- backend/internal/service/provider_test.go | 28 ++++++ backend/internal/service/resource_test.go | 52 +++++++++- docs/content/docs/progress/pending-test.mdx | 8 ++ .../lib/canvas/canvas-resource-references.ts | 94 ++++++++++++++++++ web/src/pages/canvas/project.tsx | 22 +++-- .../canvas/use-canvas-node-operations.ts | 23 +++-- .../canvas-node-generation-mentions.test.ts | 12 +++ web/test/canvas-resource-references.test.ts | 44 ++++++++- 9 files changed, 339 insertions(+), 41 deletions(-) diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index f1b1f099c..d0278ad35 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -371,7 +371,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } // 工作流参数由工作流字段定义校验，普通模型能力配置不能覆盖它们。 if resumedProviderRequestID(ctx) == "" { - if err := s.hydrateGenerationMedia(userID, &input, false); err != nil { + if err := s.hydrateGenerationMedia(userID, &input, providerMediaHydrationPolicy{}); err != nil { return nil, err } } @@ -408,11 +408,8 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } } if resumedProviderRequestID(ctx) == "" { - requirePublicURL := input.Config.InterfaceType == "newapi-channel-1" || input.Config.InterfaceType == "newapi-channel-2" || input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || input.Config.InterfaceType == string(model.ChannelInterfaceMiniMaxVideo) - if adapter, ok := protocolAdapterForContext(ctx, input.Config.InterfaceType); ok { - requirePublicURL = requirePublicURL || adapter.Metadata().RequiresPublicMediaURLs - } - if err := s.hydrateGenerationMedia(userID, &input, requirePublicURL); err != nil { + mediaPolicy := providerMediaHydrationPolicyFor(ctx, input) + if err := s.hydrateGenerationMedia(userID, &input, mediaPolicy); err != nil { return nil, err } if err := s.prepareArkPrivateAssetReferences(ctx, userID, &input); err != nil { @@ -445,6 +442,51 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } } +type providerMediaHydrationPolicy struct { + requireURL bool + preferURL bool +} + +func providerMediaHydrationPolicyFor(ctx context.Context, input canvasGenerationInput) providerMediaHydrationPolicy { + policy := providerMediaHydrationPolicy{preferURL: providerPrefersMediaURLs(input.Config.InterfaceType, input)} + switch strings.TrimSpace(input.Config.InterfaceType) { + case string(model.ChannelInterfaceNewAPIVideo), string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), string(model.ChannelInterfaceVolcengineArkVideo), string(model.ChannelInterfaceMiniMaxVideo): + policy.requireURL = true + policy.preferURL = true + } + if adapter, ok := protocolAdapterForContext(ctx, input.Config.InterfaceType); ok && adapter.Metadata().RequiresPublicMediaURLs { + policy.requireURL = true + policy.preferURL = true + } + if input.Mask != nil { + policy.requireURL = false + policy.preferURL = false + } + return policy +} + +// providerPrefersMediaURLs lists protocols whose media fields accept a remote +// URL. Byte-oriented protocols deliberately remain on the existing data path. +func providerPrefersMediaURLs(interfaceType string, input canvasGenerationInput) bool { + if input.Mask != nil { + // OpenAI image edits and similar multipart requests require file bytes. + return false + } + switch strings.TrimSpace(interfaceType) { + case string(model.ChannelInterfaceChatCompletion), string(model.ChannelInterfaceOpenAIResponse), string(model.ChannelInterfaceClaudeAPI), + string(model.ChannelInterfaceGrokImage), string(model.ChannelInterfaceVolcengineArkImage), + string(model.ChannelInterfaceXAIVideo), string(model.ChannelInterfaceNovitaVideo), + string(model.ChannelInterfaceMiniMaxVideo), string(model.ChannelInterfaceNewAPIVideo), + string(model.ChannelInterfaceNewAPIChannel1), string(model.ChannelInterfaceNewAPIChannel2), + string(model.ChannelInterfaceVolcengineArkVideo): + return true + } + if isGrokVideoConfig(input.Config) || isSeedanceVideoConfig(input.Config) || isArkPlanVideoConfig(input.Config) { + return true + } + return false +} + func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { if adapter, ok := agentProtocolAdapterForContext(ctx, input.Config.InterfaceType); ok { return runDeclarativeAgentTask(ctx, input, adapter) @@ -1320,40 +1362,41 @@ func metadataStringValues(value any) map[string]string { return values } -func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationInput, requirePublicURL bool) error { +func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationInput, policy providerMediaHydrationPolicy) error { groups := [][]providerMedia{input.ReferenceImages, input.ReferenceVideos, input.ReferenceAudios} for _, group := range groups { for index := range group { - if err := s.hydrateProviderMedia(userID, &group[index], requirePublicURL); err != nil { + if err := s.hydrateProviderMedia(userID, &group[index], policy); err != nil { return err } } } if input.Mask != nil { - return s.hydrateProviderMedia(userID, input.Mask, requirePublicURL) + return s.hydrateProviderMedia(userID, input.Mask, policy) } return nil } -func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requirePublicURL bool) error { +func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, policy providerMediaHydrationPolicy) error { if !strings.HasPrefix(media.StorageKey, "resource:") { - if requirePublicURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") { + if policy.requireURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") { return errors.New("当前 JSON 视频协议的参考素材不能使用内嵌数据，请先上传到对象存储或提供公网素材地址") } return nil } resourceID := strings.TrimPrefix(media.StorageKey, "resource:") - if requirePublicURL { - resource, err := s.repo.ResourceForUser(userID, resourceID) - if err != nil { - return fmt.Errorf("读取任务参考资源失败：%w", err) - } - if resource.Status != "ready" { - return errors.New("任务参考资源尚未上传完成") - } + resource, err := s.repo.ResourceForUser(userID, resourceID) + if err != nil { + return fmt.Errorf("读取任务参考资源失败：%w", err) + } + if resource.Status != "ready" { + return errors.New("任务参考资源尚未上传完成") + } + useObjectURL := policy.requireURL || (policy.preferURL && resourceUsesObjectStorage(resource)) + if useObjectURL { signedURL, err := s.directResourceURL(resource, time.Now().Add(providerResourceURLTTL)) if err != nil { - return fmt.Errorf("生成 JSON 视频协议参考素材地址失败：%w", err) + return fmt.Errorf("生成参考素材地址失败：%w", err) } media.URL = signedURL media.DataURL = "" @@ -1372,17 +1415,17 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ return fmt.Errorf("读取任务参考资源失败：%w", err) } defer body.Close() - policy, err := s.RuntimePolicy() + runtimePolicy, err := s.RuntimePolicy() if err != nil { return err } - resourceLimit := megabytes(policy.Resource.ResourceUploadMB) + resourceLimit := megabytes(runtimePolicy.Resource.ResourceUploadMB) data, err := io.ReadAll(io.LimitReader(body, resourceLimit+1)) if err != nil { return err } if int64(len(data)) > resourceLimit { - return fmt.Errorf("任务参考资源超过 %dMB", policy.Resource.ResourceUploadMB) + return fmt.Errorf("任务参考资源超过 %dMB", runtimePolicy.Resource.ResourceUploadMB) } mimeType := normalizedMediaMimeType(firstNonEmpty(media.MimeType, resource.MimeType), data) media.DataURL = dataURL(mimeType, data) @@ -1394,6 +1437,14 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ return nil } +func resourceUsesObjectStorage(resource *model.Resource) bool { + if resource == nil { + return false + } + provider := strings.ToLower(strings.TrimSpace(resource.Provider)) + return provider != "" && provider != "local" +} + func normalizedMediaMimeType(declared string, data []byte) string { declared = strings.TrimSpace(strings.Split(declared, ";")[0]) if declared != "" && declared != "application/octet-stream" { diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index a82288e62..78036047d 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -23,6 +23,34 @@ import ( const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8=" const testGeminiReferenceImageDataURL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" +func TestProviderMediaHydrationPolicyPrefersObjectURLs(t *testing.T) { + prefer := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceGrokImage)}}) + if prefer.requireURL || !prefer.preferURL { + t.Fatalf("grok image policy = %#v", prefer) + } + required := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceMiniMaxVideo)}}) + if !required.requireURL || !required.preferURL { + t.Fatalf("minimax policy = %#v", required) + } + bytesOnly := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceGeminiImage)}}) + if bytesOnly.requireURL || bytesOnly.preferURL { + t.Fatalf("gemini image policy = %#v", bytesOnly) + } + masked := providerMediaHydrationPolicyFor(context.Background(), canvasGenerationInput{Config: providerConfig{InterfaceType: string(model.ChannelInterfaceOpenAIImage)}, Mask: &providerMedia{ID: "mask"}}) + if masked.requireURL || masked.preferURL { + t.Fatalf("openai mask policy = %#v", masked) + } + if resourceUsesObjectStorage(&model.Resource{Provider: "aliyun"}) != true { + t.Fatal("aliyun should use object storage URLs") + } + if resourceUsesObjectStorage(&model.Resource{Provider: "tencent"}) != true || resourceUsesObjectStorage(&model.Resource{Provider: "qiniu"}) != true || resourceUsesObjectStorage(&model.Resource{Provider: "s3"}) != true { + t.Fatal("cos/kodo/s3 should use object storage URLs") + } + if resourceUsesObjectStorage(&model.Resource{Provider: "local"}) { + t.Fatal("local resources should fall back to protocol-compatible bytes") + } +} + func TestProviderRequestErrorDetails(t *testing.T) { tests := []struct { name string diff --git a/backend/internal/service/resource_test.go b/backend/internal/service/resource_test.go index ea1c9c02c..937cb177f 100644 --- a/backend/internal/service/resource_test.go +++ b/backend/internal/service/resource_test.go @@ -11,6 +11,7 @@ import ( "net/url" "os" + "path/filepath" "strconv" "strings" "testing" @@ -735,13 +736,13 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedOSSURL(t *testing.T) { t.Fatal(err) } media := providerMedia{StorageKey: "resource:resource-1", DataURL: "data:image/png;base64,old"} - if err := svc.hydrateProviderMedia("user-1", &media, true); err != nil { + if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{requireURL: true}); err != nil { t.Fatalf("hydrateProviderMedia() error = %v", err) } if !strings.HasPrefix(media.URL, "https://private-bucket.oss-cn-test.aliyuncs.com/") || media.DataURL != "" || !strings.Contains(media.URL, "Signature=") { t.Fatalf("media = %#v", media) } - if err := svc.hydrateProviderMedia("other-user", &providerMedia{StorageKey: "resource:resource-1"}, true); err == nil { + if err := svc.hydrateProviderMedia("other-user", &providerMedia{StorageKey: "resource:resource-1"}, providerMediaHydrationPolicy{requireURL: true}); err == nil { t.Fatal("hydrateProviderMedia() allowed another user's resource") } } @@ -760,7 +761,7 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedLocalURL(t *testing.T) { t.Fatal(err) } media := providerMedia{StorageKey: "resource:resource-local"} - if err := svc.hydrateProviderMedia("user-1", &media, true); err != nil { + if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{requireURL: true}); err != nil { t.Fatalf("hydrateProviderMedia() error = %v", err) } if !strings.HasPrefix(media.URL, server.URL+"/api/public/resources/resource-local/file/resource-local.png?") || !strings.Contains(media.URL, "signature=") || media.DataURL != "" { @@ -772,6 +773,51 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedLocalURL(t *testing.T) { } } +func TestHydratePreferredURLUsesObjectStorageAndFallsBackLocal(t *testing.T) { + svc := newResourceTestService(t) + settingJSON, _ := json.Marshal(ossSettingValue{ + Enabled: true, Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + AccessKeyID: "access-id", AccessKeySecret: "secret-value", + }) + if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil { + t.Fatal(err) + } + objectResource := model.Resource{ + ID: "resource-oss", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, + Provider: "aliyun", Endpoint: "https://oss-cn-test.aliyuncs.com", Bucket: "private-bucket", + ObjectKey: "users/user-1/image/prefer.png", MimeType: "image/png", + } + if err := svc.repo.CreateResource(&objectResource); err != nil { + t.Fatal(err) + } + media := providerMedia{StorageKey: "resource:resource-oss", DataURL: "data:image/png;base64,old"} + if err := svc.hydrateProviderMedia("user-1", &media, providerMediaHydrationPolicy{preferURL: true}); err != nil { + t.Fatalf("hydrateProviderMedia(prefer object) error = %v", err) + } + if !strings.HasPrefix(media.URL, "https://private-bucket.oss-cn-test.aliyuncs.com/") || media.DataURL != "" || !strings.Contains(media.URL, "Signature=") { + t.Fatalf("object media = %#v", media) + } + + localDir := filepath.Join(svc.dataDir, "resources", "users", "user-1", "image") + if err := os.MkdirAll(localDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(localDir, "local.png"), []byte("png-bytes"), 0o644); err != nil { + t.Fatal(err) + } + localResource := model.Resource{ID: "resource-local-bytes", UserID: "user-1", Kind: "image", Status: model.ResourceStatusReady, Provider: "local", ObjectKey: "users/user-1/image/local.png", MimeType: "image/png"} + if err := svc.repo.CreateResource(&localResource); err != nil { + t.Fatal(err) + } + localMedia := providerMedia{StorageKey: "resource:resource-local-bytes"} + if err := svc.hydrateProviderMedia("user-1", &localMedia, providerMediaHydrationPolicy{preferURL: true}); err != nil { + t.Fatalf("hydrateProviderMedia(prefer local) error = %v", err) + } + if localMedia.URL != "" || !strings.HasPrefix(localMedia.DataURL, "data:image/png;base64,") { + t.Fatalf("local media = %#v", localMedia) + } +} + func TestPublicResourceSignatureRejectsExpiredAndAlteredLinks(t *testing.T) { svc := newResourceTestService(t) expires := strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10) diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 67fd0b4e1..da44587b5 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -16,6 +16,14 @@ description: 当前项目已实现功能的手工验证清单与未确认边界 - [x] 回归测试在修复前复现相同保存校验错误，修复后通过；相关能力匹配、线路覆盖和 MiniMax 专项测试通过。模拟 HTTP 请求验证 2K 与内部 1440p 均发送 2K，未声明的 4K 仍被拒绝，图片 quality 参数不受视频别名影响。 - [ ] 在实际 `/admin/channels` 配置 768P / 2K 价格档后，于 `/admin/models` 保存包含 2K 的模型能力，确认创作端可选并正确报价；未调用真实收费生成接口或部署服务。 +## 对象存储优先发送媒体 URL + +- 开启对象存储后，参考图、视频、音频发给上游时：协议支持 URL 就优先发对象存储签名链接；协议必须传文件字节时仍走 Base64 或 multipart。不会在上游失败后再自动重发一次生成。 +- 强制 URL：NewAPI 系列、火山方舟视频、MiniMax 视频，以及声明需要公网媒体 URL 的插件协议。优先 URL：chat-completion、openai-response、claude-api、Grok 图片、火山方舟图片、xAI/Novita/Grok/Seedance 视频等。Gemini 图片、Gemini Veo、带蒙版的 OpenAI 图片编辑继续走字节。 +- 本地资源在“优先 URL”协议下仍读成 Base64；对象存储资源签发失败、资源不存在或未就绪会直接报错，不会伪装成“不支持 URL”。存储桶不必公开读，使用短期签名链接即可。 +- [x] 后端专项测试覆盖协议策略、OSS 强制 URL、本地强制 URL，以及 preferURL 时对象存储走签名 URL、本地回退 Base64。 +- [ ] 用户浏览器验收：开启 OSS/COS/七牛/S3 后，图生图或视频参考素材应发对象存储 URL；Gemini 或蒙版编辑仍发文件字节。签名失败不能变成静默回退 Base64。 + ## 管理后台系统性能 - 管理后台「系统配置」新增独立「系统性能」页面，集中展示主机与 Go 进程、数据目录所在磁盘、系统/进程内存、数据库连接池与 Schema、PostgreSQL 统计、Redis 运行状态和运行时缓存 key 数量；连接地址、账号、密码、DSN 与 Redis URL 不返回前端。 diff --git a/web/src/lib/canvas/canvas-resource-references.ts b/web/src/lib/canvas/canvas-resource-references.ts index b75782a8f..1d6786685 100644 --- a/web/src/lib/canvas/canvas-resource-references.ts +++ b/web/src/lib/canvas/canvas-resource-references.ts @@ -51,6 +51,100 @@ export function normalizeCanvasNodeMentionTokens(prompt: string, references: Can }, prompt); } +const CANVAS_RESOURCE_MENTION_BOUNDARY = /(?:$|\s|[,.!?;:，。！？；：、)\]}】）])/; + +function canvasResourceReferenceMentionTokens(reference: CanvasResourceReference) { + const tokens = [canvasResourceMentionToken(reference), `@${reference.label}`]; + if (reference.nodeId && !reference.assetId && reference.kind !== "skill") tokens.push(canvasNodeMentionToken(reference.nodeId)); + return [...new Set(tokens.filter(Boolean))]; +} + +function removeCanvasResourceReferenceTokens(prompt: string, references: CanvasResourceReference[]) { + return compactRemovedCanvasMentionPrompt(references.reduce((value, reference) => { + let next = value; + for (const token of canvasResourceReferenceMentionTokens(reference)) { + next = removeCanvasMentionToken(next, token); + } + return next; + }, prompt)); +} + +function rewriteCanvasPromptAfterReferenceChange(prompt: string, previousReferences: CanvasResourceReference[], nextReferences: CanvasResourceReference[]) { + const previousCanvasReferences = canvasNodeBoundReferences(previousReferences); + const nextCanvasReferences = canvasNodeBoundReferences(nextReferences); + if (!previousCanvasReferences.length || !canvasReferenceIdentityChanged(previousCanvasReferences, nextCanvasReferences)) return prompt; + + const nextByNodeId = new Map(nextCanvasReferences.map((reference) => [reference.nodeId, reference])); + let value = prompt; + previousCanvasReferences.forEach((reference) => { + const nodeToken = canvasNodeMentionToken(reference.nodeId); + canvasResourceReferenceMentionTokens(reference).forEach((token) => { + if (token !== nodeToken) value = replaceCanvasMentionToken(value, token, nodeToken); + }); + }); + previousCanvasReferences.filter((reference) => !nextByNodeId.has(reference.nodeId)).forEach((reference) => { + value = removeCanvasMentionToken(value, canvasNodeMentionToken(reference.nodeId)); + }); + value = normalizeCanvasNodeMentionTokens(value, nextCanvasReferences); + return value === prompt ? prompt : compactRemovedCanvasMentionPrompt(value); +} + +export function applyCanvasConnectionPromptSync(previousNodes: CanvasNodeData[], previousConnections: CanvasConnection[], nextNodes: CanvasNodeData[], nextConnections: CanvasConnection[]) { + const previousMap = buildCanvasNodeMentionReferenceMap(previousNodes, previousConnections, nextNodes); + const nextMap = buildCanvasNodeMentionReferenceMap(nextNodes, nextConnections, nextNodes); + let changed = false; + const mapped = nextNodes.map((node) => { + const previousPrompt = node.metadata?.composerContent ?? node.metadata?.prompt ?? ""; + const nextPrompt = rewriteCanvasPromptAfterReferenceChange(previousPrompt, previousMap.get(node.id) || [], nextMap.get(node.id) || []); + if (nextPrompt === previousPrompt) return node; + changed = true; + return writeCanvasNodePrompt(node, nextPrompt); + }); + return changed ? mapped : nextNodes; +} + +function canvasNodeBoundReferences(references: CanvasResourceReference[]) { + return references.filter((reference) => reference.nodeId && !reference.assetId && reference.kind !== "skill"); +} + +function canvasReferenceIdentityChanged(previousReferences: CanvasResourceReference[], nextReferences: CanvasResourceReference[]) { + if (previousReferences.length !== nextReferences.length) return true; + const nextLabelByNodeId = new Map(nextReferences.map((reference) => [reference.nodeId, reference.label])); + return previousReferences.some((reference) => nextLabelByNodeId.get(reference.nodeId) !== reference.label); +} + +function writeCanvasNodePrompt(node: CanvasNodeData, prompt: string) { + const hasExistingContent = (node.type === CanvasNodeType.Text && Boolean(node.metadata?.content?.trim())) || (node.type === CanvasNodeType.Image && Boolean(node.metadata?.content)); + const promptTemplateMetadata = node.metadata?.promptTemplateOperation + ? { promptTemplateOperation: undefined, promptTemplateVariables: undefined } + : {}; + return { + ...node, + metadata: hasExistingContent + ? { ...node.metadata, ...promptTemplateMetadata, composerContent: prompt } + : { ...node.metadata, ...promptTemplateMetadata, prompt, composerContent: prompt }, + }; +} + +function removeCanvasMentionToken(value: string, token: string) { + return replaceCanvasMentionToken(value, token, ""); +} + +function replaceCanvasMentionToken(value: string, token: string, replacement: string) { + if (!token) return value; + const escapedToken = token.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return value.replace(new RegExp(`${escapedToken}(?=${CANVAS_RESOURCE_MENTION_BOUNDARY.source})`, "gu"), replacement); +} + +function compactRemovedCanvasMentionPrompt(value: string) { + return value + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .replace(/[ \t]{2,}/g, " ") + .replace(/^[ \t]+|[ \t]+$/g, "") + .replace(/^\n+|\n+$/g, ""); +} + export function buildAssetMentionReferences(assets: Asset[]): CanvasResourceReference[] { return assets.flatMap((asset): CanvasResourceReference[] => { if (asset.kind === "model") return []; diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index eb140f733..0829294a9 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -70,7 +70,7 @@ import { CanvasVersionCompareModal } from "@/components/canvas/canvas-version-co import { CanvasLocalAgentPanel } from "@/components/canvas/canvas-local-agent-panel"; import { useFocusMode } from "@/hooks/use-focus-mode"; import { useCanvasAgentStore } from "@/stores/canvas/use-canvas-agent-store"; -import { getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; +import { applyCanvasConnectionPromptSync, getContextResourceNodes, normalizeCanvasNodeMentionTokens, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; import { CanvasConnectionCreateMenu, CanvasNodePanelOverlay } from "@/components/canvas/canvas-workspace-overlays"; import { CanvasOverlayLayerContainer, CanvasOverlayLayerProvider } from "@/components/canvas/canvas-overlay-layer"; import { CanvasLeaferGraphicsLayer } from "@/components/canvas/canvas-leafer-graphics-layer"; @@ -1173,20 +1173,28 @@ function InfiniteCanvasPage() { const referenceNodeId = reference.nodeId; if (!referenceNodeId) return; // 生成节点可能通过配置节点接收参考，只移除参考来源边，保留目标到配置节点的主链。 - const configNodeId = connectionsRef.current.find((connection) => { + const previousNodes = nodesRef.current; + const previousConnections = connectionsRef.current; + const configNodeId = previousConnections.find((connection) => { if (connection.fromNodeId !== targetNodeId) return false; - return nodesRef.current.find((node) => node.id === connection.toNodeId)?.type === CanvasNodeType.Config; + return previousNodes.find((node) => node.id === connection.toNodeId)?.type === CanvasNodeType.Config; })?.toNodeId; const removedConnectionIds = new Set( - connectionsRef.current + previousConnections .filter((connection) => connection.fromNodeId === referenceNodeId && (connection.toNodeId === targetNodeId || connection.toNodeId === configNodeId)) .map((connection) => connection.id), ); if (!removedConnectionIds.size) return; - connectionsRef.current = connectionsRef.current.filter((connection) => !removedConnectionIds.has(connection.id)); - setConnections(connectionsRef.current); + const nextConnections = previousConnections.filter((connection) => !removedConnectionIds.has(connection.id)); + const nextNodes = applyCanvasConnectionPromptSync(previousNodes, previousConnections, previousNodes, nextConnections); + if (nextNodes !== previousNodes) { + nodesRef.current = nextNodes; + setNodes(nextNodes); + } + connectionsRef.current = nextConnections; + setConnections(nextConnections); setSelectedConnectionId((current) => current && removedConnectionIds.has(current) ? null : current); - }, [connectionsRef, nodesRef, setConnections, setSelectedConnectionId]); + }, [connectionsRef, nodesRef, setConnections, setNodes, setSelectedConnectionId]); const handleProjectFolderInsert = useCallback((folderId: string) => { const folder = linkedProjectQuery.data?.assetFolders.find((item) => item.id === folderId); diff --git a/web/src/pages/canvas/use-canvas-node-operations.ts b/web/src/pages/canvas/use-canvas-node-operations.ts index 4c1e65ba6..3431f14ec 100644 --- a/web/src/pages/canvas/use-canvas-node-operations.ts +++ b/web/src/pages/canvas/use-canvas-node-operations.ts @@ -6,6 +6,7 @@ import { nanoid } from "nanoid"; import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; import { FOLDER_COLLAPSED_HEIGHT, FOLDER_COLLAPSED_WIDTH, FRAME_HEADER_HEIGHT, getFrameChildIds, getFrameChildren, isFrameNode } from "@/lib/canvas/canvas-frame"; import { alignCanvasNodes, layoutCanvasAuto, layoutCanvasFlow, layoutCanvasNodes, nextCanvasVersionLabel, type CanvasAlignmentMode } from "@/lib/canvas/canvas-layout"; +import { applyCanvasConnectionPromptSync } from "@/lib/canvas/canvas-resource-references"; import { createCanvasNode, isHiddenBatchChild, removeCanvasNodes } from "@/lib/canvas/canvas-project-domain"; import { isolateCopiedNodeMetadata, nextCopiedNodeTitle } from "@/lib/canvas/canvas-node-copy"; import { CanvasNodeType, type CanvasConnection, type CanvasFolderStyle, type CanvasFolderTheme, type CanvasNodeData, type CanvasNodeMetadata, type CanvasNodeTypeId, type ContextMenuState, type Position } from "@/types/canvas"; @@ -325,20 +326,28 @@ export function useCanvasNodeOperations({ const deleteNodes = useCallback((ids: Set) => { if (!ids.size) return; - const result = removeCanvasNodes(nodesRef.current, ids); - const removedNodes = nodesRef.current.filter((node) => result.removedIds.has(node.id)); - const nextConnections = connectionsRef.current.filter((connection) => !result.removedIds.has(connection.fromNodeId) && !result.removedIds.has(connection.toNodeId)); - commitNodes(result.nodes); + const previousNodes = nodesRef.current; + const previousConnections = connectionsRef.current; + const result = removeCanvasNodes(previousNodes, ids); + const removedNodes = previousNodes.filter((node) => result.removedIds.has(node.id)); + const nextConnections = previousConnections.filter((connection) => !result.removedIds.has(connection.fromNodeId) && !result.removedIds.has(connection.toNodeId)); + const nextNodes = applyCanvasConnectionPromptSync(previousNodes, previousConnections, result.nodes, nextConnections); + commitNodes(nextNodes); commitConnections(nextConnections); selectNodes(new Set()); - onNodesDeleted(result.removedIds, result.nodes, removedNodes); + onNodesDeleted(result.removedIds, nextNodes, removedNodes); }, [commitConnections, commitNodes, connectionsRef, nodesRef, onNodesDeleted, selectNodes]); const deleteConnection = useCallback((connectionId: string) => { - commitConnections(connectionsRef.current.filter((connection) => connection.id !== connectionId)); + const previousNodes = nodesRef.current; + const previousConnections = connectionsRef.current; + const nextConnections = previousConnections.filter((item) => item.id !== connectionId); + const nextNodes = applyCanvasConnectionPromptSync(previousNodes, previousConnections, previousNodes, nextConnections); + if (nextNodes !== previousNodes) commitNodes(nextNodes); + commitConnections(nextConnections); setSelectedConnectionId((current) => current === connectionId ? null : current); setContextMenu((current) => current?.type === "connection" && current.connectionId === connectionId ? null : current); - }, [commitConnections, connectionsRef, setContextMenu, setSelectedConnectionId]); + }, [commitConnections, commitNodes, connectionsRef, nodesRef, setContextMenu, setSelectedConnectionId]); const duplicateNode = useCallback((nodeId: string, duplicateMode: "variant" | "copy" = "variant") => { const source = nodesRef.current.find((node) => node.id === nodeId); diff --git a/web/test/canvas-node-generation-mentions.test.ts b/web/test/canvas-node-generation-mentions.test.ts index 255afba3d..2612d2fde 100644 --- a/web/test/canvas-node-generation-mentions.test.ts +++ b/web/test/canvas-node-generation-mentions.test.ts @@ -93,4 +93,16 @@ describe("canvas node generation position mentions", () => { expect(context.prompt).toBe("让 @图片1 进入画面"); expect(context.prompt).not.toContain("@[node:"); }); + + test("取消引用后的提示词不再因为悬挂 @图片1 阻断生成", () => { + const target = targetNode(); + const image = node("image-a", CanvasNodeType.Image, "data:image/png;base64,a"); + const { applyCanvasConnectionPromptSync } = require("../src/lib/canvas/canvas-resource-references") as typeof import("../src/lib/canvas/canvas-resource-references"); + const [nextTarget] = applyCanvasConnectionPromptSync([image, target], [connection(image.id)], [image, target], []).filter((item: typeof target) => item.id === target.id); + const context = buildNodeGenerationContext(nextTarget.id, [image, nextTarget], [], nextTarget.metadata?.composerContent || "", []); + + expect(nextTarget.metadata?.composerContent).toBe("让 进入画面"); + expect(context.referenceImages).toEqual([]); + expect(context.prompt).toBe("让 进入画面"); + }); }); diff --git a/web/test/canvas-resource-references.test.ts b/web/test/canvas-resource-references.test.ts index c13d1195f..564aa1422 100644 --- a/web/test/canvas-resource-references.test.ts +++ b/web/test/canvas-resource-references.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { buildAssetMentionReferences, buildCanvasNodeMentionReferenceMap, buildNodeMentionReferences, buildOrderedCanvasResourceReferences, canvasResourceMentionToken, collectUpstreamVideoNodes } from "../src/lib/canvas/canvas-resource-references"; +import { applyCanvasConnectionPromptSync, buildAssetMentionReferences, buildCanvasNodeMentionReferenceMap, buildNodeMentionReferences, buildOrderedCanvasResourceReferences, canvasResourceMentionToken, collectUpstreamVideoNodes } from "../src/lib/canvas/canvas-resource-references"; import { canvasNodeToAsset } from "../src/lib/canvas/canvas-node-asset"; import { buildNodeGenerationInputs } from "../src/components/canvas/canvas-node-generation"; import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas"; @@ -205,3 +205,45 @@ describe("canvas resource mention slots", () => { })).toBe("@[asset:asset-a]"); }); }); + +describe("remove canvas resource mention tokens", () => { + test("取消引用后会清掉对应的 @图片N", () => { + const image = imageNode("image-a"); + const target = { + ...videoNode("target"), + metadata: { composerContent: "@图片1 生成角色设定图：保持同一角色身份。" }, + }; + const previousConnections = [connection(image.id, target.id)]; + const [nextTarget] = applyCanvasConnectionPromptSync([image, target], previousConnections, [image, target], []).filter((node) => node.id === target.id); + + expect(nextTarget.metadata?.composerContent).toBe("生成角色设定图：保持同一角色身份。"); + expect(nextTarget.metadata?.composerContent).not.toContain("@图片1"); + }); + + test("同时存在节点 token 时一并清掉", () => { + const image = imageNode("image-a"); + const target = { + ...videoNode("target"), + metadata: { composerContent: "让 @图片1 和 @[node:image-a] 一起进入画面" }, + }; + const [nextTarget] = applyCanvasConnectionPromptSync([image, target], [connection(image.id, target.id)], [image, target], []).filter((node) => node.id === target.id); + expect(nextTarget.metadata?.composerContent).toBe("让 和 一起进入画面"); + expect(nextTarget.metadata?.composerContent).not.toContain("@图片1"); + expect(nextTarget.metadata?.composerContent).not.toContain("@[node:image-a]"); + }); + + test("多图时只清被移除的那张，并把剩余引用重新编号", () => { + const imageA = imageNode("image-a"); + const imageB = imageNode("image-b"); + const target = { + ...videoNode("target"), + metadata: { composerContent: "比较 @图片1 和 @图片2" }, + }; + const previousConnections = [connection(imageA.id, target.id), connection(imageB.id, target.id)]; + const nextConnections = [connection(imageB.id, target.id)]; + const [nextTarget] = applyCanvasConnectionPromptSync([imageA, imageB, target], previousConnections, [imageA, imageB, target], nextConnections).filter((node) => node.id === target.id); + + expect(nextTarget.metadata?.composerContent).toBe("比较 和 @图片1"); + }); +}); +