fix(video): 火山方舟 - 解耦可信素材并修复音频组合参考 (#302)
This commit is contained in:
1 parent
a2b44d940d
commit
e56a33008a
6 files changed
+63
-13
No files matched your search
@@ -60,15 +60,19 @@ func (s *Service) prepareArkPrivateAssetReferences(ctx context.Context, userID s
|
|||||||
if !hasOwnedReference {
|
if !hasOwnedReference {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if taskID := taskExecutionID(ctx); taskID != "" {
|
|
||||||
_ = s.repo.UpdateTaskProgress(taskID, "同步方舟可信素材", 36)
|
|
||||||
}
|
|
||||||
settingRecord, setting, err := s.readArkPrivateAssetSetting()
|
settingRecord, setting, err := s.readArkPrivateAssetSetting()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !setting.Enabled || setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
automaticSyncEnabled, err := arkPrivateAssetAutomaticSyncEnabled(setting)
|
||||||
return errors.New("方舟可信素材库尚未配置,请由管理员在方舟素材库设置中填写启用的 IAM AK/SK")
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !automaticSyncEnabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if taskID := taskExecutionID(ctx); taskID != "" {
|
||||||
|
_ = s.repo.UpdateTaskProgress(taskID, "同步方舟可信素材", 36)
|
||||||
}
|
}
|
||||||
for index := range input.ReferenceImages {
|
for index := range input.ReferenceImages {
|
||||||
reference := &input.ReferenceImages[index]
|
reference := &input.ReferenceImages[index]
|
||||||
@@ -106,6 +110,18 @@ func isArkPrivateAssetVideoConfig(config providerConfig) bool {
|
|||||||
return config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || isArkPlanVideoConfig(config)
|
return config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) || isArkPlanVideoConfig(config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func arkPrivateAssetAutomaticSyncEnabled(setting arkPrivateAssetSettingValue) (bool, error) {
|
||||||
|
// 可信素材同步是可选增强能力。管理员未启用时保持原始参考 URL,
|
||||||
|
// 继续执行常规火山方舟视频请求。
|
||||||
|
if !setting.Enabled {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if setting.AccessKeyID == "" || setting.AccessKeySecret == "" {
|
||||||
|
return false, errors.New("方舟可信素材库尚未配置,请由管理员在方舟素材库设置中填写启用的 IAM AK/SK")
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
func arkPrivateAssetResourceID(reference providerMedia) string {
|
func arkPrivateAssetResourceID(reference providerMedia) string {
|
||||||
if !strings.HasPrefix(reference.StorageKey, "resource:") {
|
if !strings.HasPrefix(reference.StorageKey, "resource:") {
|
||||||
return ""
|
return ""
|
||||||
|
|||||||
@@ -96,6 +96,18 @@ func TestArkPrivateAssetControlPlaneDisablesGenerationAnalytics(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArkPrivateAssetAutomaticSyncIsOptionalWhenSettingDisabled(t *testing.T) {
|
||||||
|
enabled, err := arkPrivateAssetAutomaticSyncEnabled(arkPrivateAssetSettingValue{})
|
||||||
|
if err != nil || enabled {
|
||||||
|
t.Fatalf("arkPrivateAssetAutomaticSyncEnabled() = %v, %v; want false, nil", enabled, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = arkPrivateAssetAutomaticSyncEnabled(arkPrivateAssetSettingValue{Enabled: true})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "尚未配置") {
|
||||||
|
t.Fatalf("enabled incomplete setting error = %v, want configuration error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestArkPrivateAssetSettingsEncryptSecret(t *testing.T) {
|
func TestArkPrivateAssetSettingsEncryptSecret(t *testing.T) {
|
||||||
svc := &Service{dataDir: t.TempDir()}
|
svc := &Service{dataDir: t.TempDir()}
|
||||||
value, err := arkPrivateAssetSettingFromRequest(ArkPrivateAssetSettingRequest{
|
value, err := arkPrivateAssetSettingFromRequest(ArkPrivateAssetSettingRequest{
|
||||||
|
|||||||
@@ -319,9 +319,9 @@ export function modelGroupReferenceLimits(config: AiConfig, selected: string, ca
|
|||||||
|
|
||||||
export function inferVideoOperation(input: ModelInputSummary) {
|
export function inferVideoOperation(input: ModelInputSummary) {
|
||||||
const visualInputCount = input.imageCount + input.characterCount;
|
const visualInputCount = input.imageCount + input.characterCount;
|
||||||
// 音频参考有独立的协议能力;不能先归类为 reference_to_video,
|
// 纯音频参考使用独立能力;音频与图片、角色或视频组合时属于全模态参考,
|
||||||
// 否则只声明 audio_to_video 的细分模型永远无法被兼容路由选中。
|
// 不能把组合请求误路由到只支持 audio_to_video 的细分模型。
|
||||||
if (input.audioCount > 0) return "audio_to_video";
|
if (input.audioCount > 0) return visualInputCount > 0 || input.videoCount > 0 ? "reference_to_video" : "audio_to_video";
|
||||||
if (input.videoCount > 0 || visualInputCount > 2) return "reference_to_video";
|
if (input.videoCount > 0 || visualInputCount > 2) return "reference_to_video";
|
||||||
if (visualInputCount > 0) return "image_to_video";
|
if (visualInputCount > 0) return "image_to_video";
|
||||||
return "text_to_video";
|
return "text_to_video";
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import { useExternalAssetSources } from "@/hooks/use-external-asset-sources";
|
|||||||
import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSize, imageResolutionChoices, imageResolutionOption, imageSizeForResolution, supportsImageResolutionPresets, type ImageResolutionChoice } from "@/lib/image-resolution-tiers";
|
import { buildImageResolutionOptions, formatImageResolutionSize, imageRatioForSize, imageResolutionChoices, imageResolutionOption, imageSizeForResolution, supportsImageResolutionPresets, type ImageResolutionChoice } from "@/lib/image-resolution-tiers";
|
||||||
import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options";
|
import { VIDEO_RESOLUTION_OPTIONS } from "@/lib/video-generation-options";
|
||||||
import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities";
|
import { modelCapabilityConfigFor, normalizeImageValue, normalizeVideoValue, videoDurationAllowed, videoDurationOptions, type ImageCapabilityConfig, type VideoCapabilityConfig } from "@/lib/model-capabilities";
|
||||||
import { resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection";
|
import { inferVideoOperation, resolveCompatibleModel, mergedImageCapabilityConfig, type ModelRequirements } from "@/lib/model-selection";
|
||||||
import { backendModelRuntimeRequired, isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task";
|
import { backendModelRuntimeRequired, isGenerationTaskCancelled, runBackendGenerationTask, runBackendGenerationTaskBatch, type BackendGenerationResult } from "@/services/api/generation-task";
|
||||||
import { requestImageQuestion, type AiTextContentPart } from "@/services/api/image";
|
import { requestImageQuestion, type AiTextContentPart } from "@/services/api/image";
|
||||||
import { listAddedSkills, type Skill } from "@/services/api/skills";
|
import { listAddedSkills, type Skill } from "@/services/api/skills";
|
||||||
@@ -487,6 +487,13 @@ export default function CreatePage() {
|
|||||||
const references = selectedCreationReferences(text, mentionReferences);
|
const references = selectedCreationReferences(text, mentionReferences);
|
||||||
// 后端对图片和视频使用不同的参考字段;这里先拆分,避免媒体类型在写入任务时被误判。
|
// 后端对图片和视频使用不同的参考字段;这里先拆分,避免媒体类型在写入任务时被误判。
|
||||||
const { referenceImages, referenceVideos, referenceAudios } = splitCreationAttachments(attachments);
|
const { referenceImages, referenceVideos, referenceAudios } = splitCreationAttachments(attachments);
|
||||||
|
const videoOperation = inferVideoOperation({
|
||||||
|
textCount: text ? 1 : 0,
|
||||||
|
imageCount: referenceImages.length,
|
||||||
|
videoCount: referenceVideos.length,
|
||||||
|
audioCount: referenceAudios.length,
|
||||||
|
characterCount: 0,
|
||||||
|
});
|
||||||
const expandedPrompt = expandCreationPrompt(text, references, attachments);
|
const expandedPrompt = expandCreationPrompt(text, references, attachments);
|
||||||
const referenceMetadata = creationReferenceMetadata(references);
|
const referenceMetadata = creationReferenceMetadata(references);
|
||||||
followLatestMessageRef.current = true;
|
followLatestMessageRef.current = true;
|
||||||
@@ -627,7 +634,7 @@ export default function CreatePage() {
|
|||||||
referenceVideos,
|
referenceVideos,
|
||||||
referenceAudios,
|
referenceAudios,
|
||||||
signal: requestLifecycle.signal,
|
signal: requestLifecycle.signal,
|
||||||
metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: referenceAudios.length && !referenceImages.length && !referenceVideos.length ? "audio_to_video" : attachments.length ? "image_to_video" : "text_to_video", ...referenceMetadata },
|
metadata: { source: "create-page", conversationId: activeConversation.id, messageId: assistantMessage.id, videoEditOperation: videoOperation, ...referenceMetadata },
|
||||||
onTaskUpdate: bindTask,
|
onTaskUpdate: bindTask,
|
||||||
...retryContext,
|
...retryContext,
|
||||||
}));
|
}));
|
||||||
@@ -647,7 +654,7 @@ export default function CreatePage() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const message = generationErrorMessage(error);
|
const message = generationErrorMessage(error);
|
||||||
updateOriginAssistant((item) => ({ ...item, status: "error", error: message, generationErrorCode: item.generationErrorCode || generationErrorCode(error), generationOperation: item.generationOperation || (mode === "video" ? (attachments.length ? "image_to_video" : "text_to_video") : mode), createdAt: assistantMessage.createdAt, content: "生成失败" }));
|
updateOriginAssistant((item) => ({ ...item, status: "error", error: message, generationErrorCode: item.generationErrorCode || generationErrorCode(error), generationOperation: item.generationOperation || (mode === "video" ? videoOperation : mode), createdAt: assistantMessage.createdAt, content: "生成失败" }));
|
||||||
} finally {
|
} finally {
|
||||||
requestLifecycle.release();
|
requestLifecycle.release();
|
||||||
releaseRetryLock();
|
releaseRetryLock();
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
|
|||||||
import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy";
|
import { canvasConnectionError } from "../src/lib/canvas/canvas-connection-policy";
|
||||||
import { assertCanvasImageReferenceLimit, buildGenerationConfig, canvasImageReferenceLimitError, resolveCanvasGenerationModel } from "../src/lib/canvas/canvas-project-generation";
|
import { assertCanvasImageReferenceLimit, buildGenerationConfig, canvasImageReferenceLimitError, resolveCanvasGenerationModel } from "../src/lib/canvas/canvas-project-generation";
|
||||||
import { defaultModelCapabilityConfig } from "../src/lib/model-capabilities";
|
import { defaultModelCapabilityConfig } from "../src/lib/model-capabilities";
|
||||||
import { groupModelsByDisplayName, modelCompatibilityError, modelGroupReferenceLimits, resolveCompatibleModel, resolveModelGenerationDefaults } from "../src/lib/model-selection";
|
import { groupModelsByDisplayName, inferVideoOperation, modelCompatibilityError, modelGroupReferenceLimits, resolveCompatibleModel, resolveModelGenerationDefaults } from "../src/lib/model-selection";
|
||||||
import { defaultConfig, normalizeModelOptionValue, type AiConfig, type ModelChannel } from "../src/stores/use-config-store";
|
import { defaultConfig, normalizeModelOptionValue, type AiConfig, type ModelChannel } from "../src/stores/use-config-store";
|
||||||
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
|
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "../src/types/canvas";
|
||||||
|
|
||||||
@@ -167,6 +167,12 @@ describe("逻辑模型选择", () => {
|
|||||||
expect(modelCompatibilityError(config, "relay::cinema-image", requirements)).toContain("参考音频");
|
expect(modelCompatibilityError(config, "relay::cinema-image", requirements)).toContain("参考音频");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("音频仅在单独参考时归类为音频生视频", () => {
|
||||||
|
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("audio_to_video");
|
||||||
|
expect(inferVideoOperation({ textCount: 1, imageCount: 1, videoCount: 0, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
|
||||||
|
expect(inferVideoOperation({ textCount: 1, imageCount: 0, videoCount: 1, audioCount: 1, characterCount: 0 })).toBe("reference_to_video");
|
||||||
|
});
|
||||||
|
|
||||||
test("逻辑视频模型将 720 与 720p 视为同一分辨率", () => {
|
test("逻辑视频模型将 720 与 720p 视为同一分辨率", () => {
|
||||||
const model = "cinema-720p";
|
const model = "cinema-720p";
|
||||||
const channel: ModelChannel = {
|
const channel: ModelChannel = {
|
||||||
|
|||||||
@@ -1318,6 +1318,15 @@ test("remote image video and audio references keep Backend parity without Dreami
|
|||||||
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-video-output" } },
|
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-video-output" } },
|
||||||
operation: "reference_to_video",
|
operation: "reference_to_video",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
mode: "video" as const,
|
||||||
|
references: {
|
||||||
|
referenceImages: [{ id: "remote-image-audio-image-0001", name: "reference.png", type: "image/png", dataUrl: "", storageKey: "resource:remote-image-audio-image-0001" }],
|
||||||
|
referenceAudios: [{ id: "remote-image-audio-audio-0001", name: "reference.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-image-audio-audio-0001" }],
|
||||||
|
},
|
||||||
|
result: { mode: "video", video: { dataUrl: "opaque://video", storageKey: "resource:remote-image-audio-output" } },
|
||||||
|
operation: "reference_to_video",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
mode: "audio" as const,
|
mode: "audio" as const,
|
||||||
references: { referenceAudios: [{ id: "remote-audio-0001", name: "audio.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-audio-0001" }] },
|
references: { referenceAudios: [{ id: "remote-audio-0001", name: "audio.mp3", type: "audio/mpeg", url: "", storageKey: "resource:remote-audio-0001" }] },
|
||||||
@@ -1677,7 +1686,7 @@ test("Create audio upload converts, previews, removes, and submits through the s
|
|||||||
operation?: string;
|
operation?: string;
|
||||||
input?: { referenceAudios?: Array<Record<string, unknown>> };
|
input?: { referenceAudios?: Array<Record<string, unknown>> };
|
||||||
};
|
};
|
||||||
expect(submitted.operation).toBe("reference_to_video");
|
expect(submitted.operation).toBe("audio_to_video");
|
||||||
expect(submitted.input?.referenceAudios).toEqual([
|
expect(submitted.input?.referenceAudios).toEqual([
|
||||||
{
|
{
|
||||||
id: attachment.id,
|
id: attachment.id,
|
||||||
|
|||||||
Reference in new issue
Block a user