fix(canvas): 分镜与视频模型 - 按端点协议适配并修复分镜生成
* fix(backend): 分镜生成 - 自动修复携带原始剧情并支持有界重试 * fix(canvas): 视频模型 - 按端点协议拉取模型并适配分镜视频节点 * fix(canvas): 视频模型 - 保留已有协议并刷新分镜节点模型 * fix(canvas): 引用图预览 - 修正提示面板代码格式 --------- Co-authored-by: sikongyue <3155157671@qq.com> Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
10 files changed
+195
-65
No files matched your search
@@ -26,7 +26,7 @@ func RegisterChannelModelRoutes(r *gin.RouterGroup, svc *service.Service) {
|
||||
return
|
||||
}
|
||||
// handler 不接触上游响应细节,密钥校验和模型目录解析统一收敛到 service。
|
||||
models, err := svc.FetchChannelModels(c.Request.Context(), user, input)
|
||||
models, err := svc.FetchChannelModelCatalog(c.Request.Context(), user, input)
|
||||
if err != nil {
|
||||
failService(c, err)
|
||||
return
|
||||
|
||||
@@ -28,11 +28,38 @@ type channelModelsPayload struct {
|
||||
}
|
||||
|
||||
type channelModelItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SupportedEndpointTypes []string `json:"supported_endpoint_types"`
|
||||
}
|
||||
|
||||
func (s *Service) FetchChannelModels(ctx context.Context, actor *model.User, input ChannelModelsRequest) ([]string, error) {
|
||||
items, err := s.FetchChannelModelCatalog(ctx, actor, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := make(map[string]bool, len(items))
|
||||
models := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
name := strings.TrimSpace(item.ID)
|
||||
if name == "" || seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
models = append(models, name)
|
||||
}
|
||||
sort.Strings(models)
|
||||
return models, nil
|
||||
}
|
||||
|
||||
// ChannelModelCatalogItem 是前端自定义渠道拉取模型目录后的最小合同;
|
||||
// supportedEndpointTypes 由上游 /models 返回,前端据此推导能力与协议,不依赖模型名猜测。
|
||||
type ChannelModelCatalogItem struct {
|
||||
ID string `json:"id"`
|
||||
SupportedEndpointTypes []string `json:"supportedEndpointTypes,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) FetchChannelModelCatalog(ctx context.Context, actor *model.User, input ChannelModelsRequest) ([]ChannelModelCatalogItem, error) {
|
||||
if actor == nil || strings.TrimSpace(actor.ID) == "" {
|
||||
return nil, Unauthorized("请先登录")
|
||||
}
|
||||
@@ -99,17 +126,33 @@ func (s *Service) FetchChannelModels(ctx context.Context, actor *model.User, inp
|
||||
items = payload.Models
|
||||
}
|
||||
seen := make(map[string]bool, len(items))
|
||||
models := make([]string, 0, len(items))
|
||||
catalog := make([]ChannelModelCatalogItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
name := strings.TrimPrefix(strings.TrimSpace(firstNonEmpty(item.ID, item.Name)), "models/")
|
||||
if name == "" || seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
models = append(models, name)
|
||||
catalog = append(catalog, ChannelModelCatalogItem{ID: name, SupportedEndpointTypes: normalizeCatalogEndpointTypes(item.SupportedEndpointTypes)})
|
||||
}
|
||||
sort.Strings(models)
|
||||
return models, nil
|
||||
sort.Slice(catalog, func(left int, right int) bool {
|
||||
return catalog[left].ID < catalog[right].ID
|
||||
})
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func normalizeCatalogEndpointTypes(values []string) []string {
|
||||
seen := make(map[string]bool, len(values))
|
||||
normalized := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
item := strings.TrimSpace(value)
|
||||
if item == "" || seen[item] {
|
||||
continue
|
||||
}
|
||||
seen[item] = true
|
||||
normalized = append(normalized, item)
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func channelModelsUpstreamError(err error) error {
|
||||
|
||||
@@ -26,7 +26,7 @@ func defaultPromptDefinitions() []PromptOperationDefinition {
|
||||
Operation: promptOperationStoryboardRepair, Label: "分镜修复", Category: "分镜", OutputType: "json", SchemaKey: "storyboard-plan/v3",
|
||||
Description: "修复模型返回的分镜结构、字段和镜头复杂度。",
|
||||
Variables: []PromptTemplateVariable{{Label: "校验错误", Placeholder: "{{校验错误}}"}, {Label: "项目画风", Placeholder: "{{项目画风}}"}},
|
||||
DefaultContent: `你是影视分镜 JSON 修复导演。修复结构时必须保留剧情信息,通过拆镜或重新分配内容解决复杂度超限,不得用删除关键剧情掩盖错误。保持原项目的视觉媒介、角色身份、服装、道具和连续性,不要擅自改写画风。只修复校验错误和由此引发的镜头组织问题。`,
|
||||
DefaultContent: `你是影视分镜 JSON 修复导演。修复结构时必须保留剧情信息,通过拆镜或重新分配内容解决复杂度超限,不得用删除关键剧情掩盖错误。若校验错误是台词/旁白超长,必须拆镜或精简为单镜头可念完的台词,保留关键情节,不要把超长 dialogue 原样放回。保持原项目的视觉媒介、角色身份、服装、道具和连续性,不要擅自改写画风。只修复校验错误和由此引发的镜头组织问题。`,
|
||||
},
|
||||
{
|
||||
Operation: promptOperationStoryboardFirstFrame, Label: "分镜首帧", Category: "生成", OutputType: "text",
|
||||
@@ -213,6 +213,9 @@ func storyboardProtectedContext(values map[string]string) string {
|
||||
|
||||
func storyboardRepairProtectedContext(values map[string]string) string {
|
||||
return strings.Join([]string{
|
||||
"【剧情】\n" + values["剧情"],
|
||||
"【用户本次要求】\n" + values["用户要求"],
|
||||
"【当前画布资产】\n" + values["画布资产"],
|
||||
"【原始校验错误】\n" + values["校验错误"],
|
||||
"【当前项目画风】\n" + values["项目画风"],
|
||||
"【当前角色版本】\n" + values["角色版本"],
|
||||
@@ -226,6 +229,7 @@ func storyboardExecutionContract(durationRule string, countRule string) string {
|
||||
- ` + durationRule + `
|
||||
- ` + countRule + `
|
||||
- 单镜头最多 2 名主要角色、1 个主运镜、1 条主要动作链、3 个 timeBeats 和 3 个 mustHave;超限必须拆镜或在固定镜头数内重新分配。
|
||||
- dialogue 只写本镜头实际念出的台词或简短旁白,字数上限按 1 秒最多约 5 个中文字符计算(至少 24 字);超长台词/旁白必须拆镜或精简,不得用 dialogue 承载长段叙述。
|
||||
- characterIds 只能引用当前角色版本中的 assetId;没有角色时返回空数组。
|
||||
- styleGuide 最多 120 个中文字符;visualPrompt 只描述首帧,videoPrompt 只描述运动和结尾状态。
|
||||
- 画幅比例由视频节点参数控制,提示词不得写入具体比例,也不要讨论画幅配置。
|
||||
|
||||
@@ -1145,22 +1145,9 @@ func (s *Service) processAgentStoryboardTask(ctx context.Context, task model.Tas
|
||||
err = validateStoryboardPlan(plan, 0, 0, input.Characters)
|
||||
}
|
||||
if err != nil {
|
||||
_ = s.repo.UpdateTaskProgress(task.ID, "修复分镜结构", 55)
|
||||
repairPrompt, promptErr := s.buildStoryboardRepairPrompt(task.UserID, err, input, text)
|
||||
if promptErr != nil {
|
||||
return nil, nil, promptErr
|
||||
}
|
||||
repaired, repairErr := runTextTask(withProviderRequestKind(ctx, "repair"), canvasGenerationInput{Mode: "text", Prompt: repairPrompt, Config: config, StreamText: true})
|
||||
if repairErr != nil {
|
||||
return nil, nil, fmt.Errorf("分镜结构修复失败:%w", repairErr)
|
||||
}
|
||||
repairedText, _ := repaired["text"].(string)
|
||||
plan, err = parseAgentStoryboardPlan(repairedText)
|
||||
if err == nil {
|
||||
err = validateStoryboardPlan(plan, 0, 0, input.Characters)
|
||||
}
|
||||
plan, err = s.repairStoryboardPlan(ctx, task, input, config, text, err, 0, 0)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("分镜模型结构修复后仍不合法:%w", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return s.buildAgentStoryboardResult(task, plan, assets, input.ProjectStyle)
|
||||
@@ -1201,22 +1188,9 @@ func (s *Service) processStoryboardRowsTask(ctx context.Context, task model.Task
|
||||
err = validateStoryboardPlan(plan, input.ShotDuration, input.ShotCount, input.Characters)
|
||||
}
|
||||
if err != nil {
|
||||
_ = s.repo.UpdateTaskProgress(task.ID, "修复分镜结构", 55)
|
||||
repairPrompt, promptErr := s.buildStoryboardRepairPrompt(task.UserID, err, input, text)
|
||||
if promptErr != nil {
|
||||
return nil, nil, promptErr
|
||||
}
|
||||
repaired, repairErr := runTextTask(withProviderRequestKind(ctx, "repair"), canvasGenerationInput{Mode: "text", Prompt: repairPrompt, Config: config, StreamText: true})
|
||||
if repairErr != nil {
|
||||
return nil, nil, fmt.Errorf("分镜结构修复失败:%w", repairErr)
|
||||
}
|
||||
repairedText, _ := repaired["text"].(string)
|
||||
plan, err = parseAgentStoryboardPlan(repairedText)
|
||||
if err == nil {
|
||||
err = validateStoryboardPlan(plan, input.ShotDuration, input.ShotCount, input.Characters)
|
||||
}
|
||||
plan, err = s.repairStoryboardPlan(ctx, task, input, config, text, err, input.ShotDuration, input.ShotCount)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("分镜模型结构修复后仍不合法:%w", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
rows := make([]map[string]any, 0, len(plan.Shots))
|
||||
@@ -1251,6 +1225,35 @@ func (s *Service) processStoryboardRowsTask(ctx context.Context, task model.Task
|
||||
return map[string]interface{}{"title": plan.Title, "rows": rows}, nil, nil
|
||||
}
|
||||
|
||||
const maxStoryboardRepairAttempts = 2
|
||||
|
||||
func (s *Service) repairStoryboardPlan(ctx context.Context, task model.Task, input agentStoryboardInput, config providerConfig, originalText string, validationErr error, shotDuration int, shotCount int) (agentStoryboardPlan, error) {
|
||||
currentText := originalText
|
||||
currentErr := validationErr
|
||||
for attempt := 1; attempt <= maxStoryboardRepairAttempts; attempt++ {
|
||||
_ = s.repo.UpdateTaskProgress(task.ID, "修复分镜结构", 55+attempt*10)
|
||||
repairPrompt, promptErr := s.buildStoryboardRepairPrompt(task.UserID, task.Prompt, currentErr, input, currentText)
|
||||
if promptErr != nil {
|
||||
return agentStoryboardPlan{}, promptErr
|
||||
}
|
||||
repaired, repairErr := runTextTask(withProviderRequestKind(ctx, "repair"), canvasGenerationInput{Mode: "text", Prompt: repairPrompt, Config: config, StreamText: true})
|
||||
if repairErr != nil {
|
||||
return agentStoryboardPlan{}, fmt.Errorf("分镜结构修复失败:%w", repairErr)
|
||||
}
|
||||
repairedText, _ := repaired["text"].(string)
|
||||
plan, parseErr := parseAgentStoryboardPlan(repairedText)
|
||||
if parseErr == nil {
|
||||
parseErr = validateStoryboardPlan(plan, shotDuration, shotCount, input.Characters)
|
||||
}
|
||||
if parseErr == nil {
|
||||
return plan, nil
|
||||
}
|
||||
currentText = repairedText
|
||||
currentErr = parseErr
|
||||
}
|
||||
return agentStoryboardPlan{}, fmt.Errorf("分镜模型结构修复后仍不合法:%w", currentErr)
|
||||
}
|
||||
|
||||
func providerConfigReady(config providerConfig) bool {
|
||||
return strings.TrimSpace(config.Model) != "" && (strings.TrimSpace(config.ChannelID) != "" || (strings.TrimSpace(config.BaseURL) != "" && strings.TrimSpace(config.APIKey) != ""))
|
||||
}
|
||||
|
||||
@@ -15,8 +15,8 @@ func (s *Service) buildAgentStoryboardPlannerPrompt(userID string, brief string,
|
||||
return removeFixedMediaRestrictions(compiled.Content), nil
|
||||
}
|
||||
|
||||
func (s *Service) buildStoryboardRepairPrompt(userID string, validationErr error, input agentStoryboardInput, original string) (string, error) {
|
||||
values := storyboardPromptValues("", input.Requirements, input.CanvasAssets, input.ProjectStyle, input.Characters, input.ShotDuration, input.ShotCount)
|
||||
func (s *Service) buildStoryboardRepairPrompt(userID string, brief string, validationErr error, input agentStoryboardInput, original string) (string, error) {
|
||||
values := storyboardPromptValues(brief, input.Requirements, input.CanvasAssets, input.ProjectStyle, input.Characters, input.ShotDuration, input.ShotCount)
|
||||
values["校验错误"] = validationErr.Error()
|
||||
values["原始输出"] = original
|
||||
compiled, err := s.compilePrompt(userID, promptOperationStoryboardRepair, values)
|
||||
|
||||
Reference in new issue
Block a user