tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video): 火山方舟 - 解耦可信素材并修复音频组合参考 (#302)

Browse Source
This commit is contained in:
MasterChiefCN
2026-08-24 19:42:30 +08:00
committed by GitHub
parent b809d1f835
commit 76035451f1
6 changed files with 63 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
backend/internal/service/provider_ark_private_assets.go
View File

@@ -60,15 +60,19 @@ func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID s
if !hasOwnedReference {
return nil
}
if taskID := taskExecutionID(ctx); taskID != "" {
_ = s.repo.UpdateTaskProgress(taskID, "同步方舟可信素材", 36)
}
settingRecord, setting, err := s.readArkPrivateAssetSetting()
if err != nil {
return err
}
if !setting.Enabled || setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
return errors.New("方舟可信素材库尚未配置，请由管理员在方舟素材库设置中填写启用的 IAM AK/SK")
automaticSyncEnabled, err := arkPrivateAssetAutomaticSyncEnabled(setting)
if err != nil {
return err
}
if !automaticSyncEnabled {
return nil
}
if taskID := taskExecutionID(ctx); taskID != "" {
_ = s.repo.UpdateTaskProgress(taskID, "同步方舟可信素材", 36)
}
for index := range input.ReferenceImages {
reference := &input.ReferenceImages[index]
@@ -106,6 +110,18 @@ func isArkPrivateAssetVideoConfig(config providerConfig) bool {
return config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || isArkPlanVideoConfig(config)
}
func arkPrivateAssetAutomaticSyncEnabled(setting arkPrivateAssetSettingValue) (bool, error) {
// 可信素材同步是可选增强能力。管理员未启用时保持原始参考 URL，
// 继续执行常规火山方舟视频请求。
if !setting.Enabled {
return false, nil
}
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
return false, errors.New("方舟可信素材库尚未配置，请由管理员在方舟素材库设置中填写启用的 IAM AK/SK")
}
return true, nil
}
func arkPrivateAssetResourceID(reference providerMedia) string {
if !strings.HasPrefix(reference.StorageKey, "resource:") {
return ""

12
backend/internal/service/provider_ark_private_assets_test.go
View File

@@ -96,6 +96,18 @@ func TestArkPrivateAssetControlPlaneDisablesGenerationAnalytics(t *testing.T) {
}
}
func TestArkPrivateAssetAutomaticSyncIsOptionalWhenSettingDisabled(t *testing.T) {
enabled, err := arkPrivateAssetAutomaticSyncEnabled(arkPrivateAssetSettingValue{})
if err != nil || enabled {
t.Fatalf("arkPrivateAssetAutomaticSyncEnabled() = %v, %v; want false, nil", enabled, err)
}
_, err = arkPrivateAssetAutomaticSyncEnabled(arkPrivateAssetSettingValue{Enabled: true})
if err == nil || !strings.Contains(err.Error(), "尚未配置") {
t.Fatalf("enabled incomplete setting error = %v, want configuration error", err)
}
}
func TestArkPrivateAssetSettingsEncryptSecret(t *testing.T) {
svc := &Service{dataDir: t.TempDir()}
value, err := arkPrivateAssetSettingFromRequest(ArkPrivateAssetSettingRequest{

6
web/src/lib/model-selection.ts
View File

@@ -319,9 +319,9 @@ export function modelGroupReferenceLimits(config: AiConfig, selected: string, ca
export function inferVideoOperation(input: ModelInputSummary) {
const visualInputCount = input.imageCount + input.characterCount;
// 音频参考有独立的协议能力；不能先归类为 reference_to_video，
// 否则只声明 audio_to_video 的细分模型永远无法被兼容路由选中。
if (input.audioCount > 0) return "audio_to_video";
// 纯音频参考使用独立能力；音频与图片、角色或视频组合时属于全模态参考，
// 不能把组合请求误路由到只支持 audio_to_video 的细分模型。
if (input.audioCount > 0) return visualInputCount > 0 || input.videoCount > 0 ? "reference_to_video" : "audio_to_video";
if (input.videoCount > 0 || visualInputCount > 2) return "reference_to_video";
if (visualInputCount > 0) return "image_to_video";
return "text_to_video";

13
web/src/pages/create/index.tsx
View File

@@ -20,7 +20,7 @@ import { useExternalAssetSources } from "@/hooks/use-external-asset-sources";
import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSize, imageResolutionChoices, imageResolutionOption, imageSizeForResolution, supportsImageResolutionPresets, type ImageResolutionChoice } from "@/lib/image-resolution-tiers";
import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options";
import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities";
import { resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection";
import { inferVideoOperation, resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection";
import { backendModelRuntimeRequired, isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task";
import { requestImageQuestion, type AiTextContentPart } from "@/services/api/image";
import { listAddedSkills, type Skill } from "@/services/api/skills";
@@ -487,6 +487,13 @@ export default function CreatePage() {
const references = selectedCreationReferences(text, mentionReferences);
// 后端对图片和视频使用不同的参考字段；这里先拆分，避免媒体类型在写入任务时被误判。
const { referenceImages, referenceVideos, referenceAudios } = splitCreationAttachments(attachments);
const videoOperation = inferVideoOperation({
textCount: text ? 1 : 0,
imageCount: referenceImages.length,
videoCount: referenceVideos.length,
audioCount: referenceAudios.length,
characterCount: 0,
});
const expandedPrompt = expandCreationPrompt(text, references, attachments);
const referenceMetadata = creationReferenceMetadata(references);
followLatestMessageRef.current = true;
@@ -627,7 +634,7 @@ export default function CreatePage() {
referenceVideos,
referenceAudios,
signal: requestLifecycle.signal,
metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: referenceAudios.length && !referenceImages.length && !referenceVideos.length ? "audio_to_video" : attachments.length ? "image_to_video" : "text_to_video", ...referenceMetadata },
metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: videoOperation, ...referenceMetadata },
onTaskUpdate: bindTask,
...retryContext,
}));
@@ -647,7 +654,7 @@ export default function CreatePage() {
return;
}
const message = generationErrorMessage(error);
updateOriginAssistant((item) => ({ ...item, status: "error", error: message, generationErrorCode: item.generationErrorCode || generationErrorCode(error), generationOperation: item.generationOperation || (mode === "video" ? (attachments.length ? "image_to_video" : "text_to_video") : mode), createdAt: assistantMessage.createdAt, content: "生成失败" }));
updateOriginAssistant((item) => ({ ...item, status: "error", error: message, generationErrorCode: item.generationErrorCode || generationErrorCode(error), generationOperation: item.generationOperation || (mode === "video" ? videoOperation : mode), createdAt: assistantMessage.createdAt, content: "生成失败" }));
} finally {
requestLifecycle.release();
releaseRetryLock();

8
web/test/canvas-model-policy.test.ts
View File

@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy";
import { assertCanvasImageReferenceLimit, buildGenerationConfig, canvasImageReferenceLimitError, resolveCanvasGenerationModel } from "../src/lib/canvas/canvas-project-generation";
import { defaultModelCapabilityConfig } from "../src/lib/model-capabilities";
import { groupModelsByDisplayName, modelCompatibilityError, modelGroupReferenceLimits, resolveCompatibleModel, resolveModelGenerationDefaults } from "../src/lib/model-selection";
import { groupModelsByDisplayName, inferVideoOperation, modelCompatibilityError, modelGroupReferenceLimits, resolveCompatibleModel, resolveModelGenerationDefaults } from "../src/lib/model-selection";
import { defaultConfig, normalizeModelOptionValue, type AiConfig, type ModelChannel } from "../src/stores/use-config-store";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
@@ -167,6 +167,12 @@ describe("逻辑模型选择", () => {
expect(modelCompatibilityError(config, "relay::cinema-image", requirements)).toContain("参考音频");
});
test("音频仅在单独参考时归类为音频生视频", () => {
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("audio_to_video");
expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 1, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
});
test("逻辑视频模型将 720 与 720p 视为同一分辨率", () => {
const model = "cinema-720p";
const channel: ModelChannel = {

11
web/test/generation-task-local.test.ts
View File

@@ -1318,6 +1318,15 @@ test("remote image video and audio references keep Backend parity without Dreami
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-video-output" } },
operation: "reference_to_video",
},
{
mode: "video" as const,
references: {
referenceImages: [{ id: "remote-image-audio-image-0001", name: "reference.png", type: "image/png", dataUrl: "", storageKey: "resource:remote-image-audio-image-0001" }],
referenceAudios: [{ id: "remote-image-audio-audio-0001", name: "reference.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-image-audio-audio-0001" }],
},
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-image-audio-output" } },
operation: "reference_to_video",
},
{
mode: "audio" as const,
references: { referenceAudios: [{ id: "remote-audio-0001", name: "audio.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-audio-0001" }] },
@@ -1677,7 +1686,7 @@ test("Create audio upload converts, previews, removes, and submits through the s
operation?: string;
input?: { referenceAudios?: Array<Record<string, unknown>> };
};
expect(submitted.operation).toBe("reference_to_video");
expect(submitted.operation).toBe("audio_to_video");
expect(submitted.input?.referenceAudios).toEqual([
{
id: attachment.id,