tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #8 from CyrusAuyeung/agent/add-sub2api-grok-video

Browse Source 
fix(media): Grok 视频 - 支持 xAI 与 Sub2API 生成协议
This commit is contained in:
ddcat666
2026-07-22 21:39:59 +08:00
committed by GitHub
parent e6182fa388 2c9bba20b3
commit f668b54439
13 changed files with 129 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/internal/handler/security.go
View File

@@ -93,7 +93,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
return requestPath == "/responses"
case model.ChannelInterfaceOpenAIImage:
return requestPath == "/images/generations" || requestPath == "/images/edits"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2:
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
return false
default:
return true

10
backend/internal/handler/security_test.go
View File

@@ -48,10 +48,12 @@ func TestAuthorizeSystemProxyRestrictsConfiguredInterfaceType(t *testing.T) {
}
}
func TestAuthorizeSystemProxyBlocksNewAPIChannel2DirectBrowserProxy(t *testing.T) {
func TestAuthorizeSystemProxyBlocksBackendOnlyVideoInterfaces(t *testing.T) {
body := []byte(`{"model":"grok-image-video"}`)
channel := &model.ModelChannel{APIFormat: "openai", InterfaceType: model.ChannelInterfaceNewAPIChannel2, ModelsJSON: `["grok-image-video"]`}
if err := authorizeSystemProxy(channel, http.MethodPost, "/video/generations", "application/json", body); err == nil {
t.Fatal("authorizeSystemProxy() error = nil for backend-only video interface")
for _, interfaceType := range []model.ChannelInterfaceType{model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo} {
channel := &model.ModelChannel{APIFormat: "openai", InterfaceType: interfaceType, ModelsJSON: `["grok-image-video"]`}
if err := authorizeSystemProxy(channel, http.MethodPost, "/video/generations", "application/json", body); err == nil {
t.Fatalf("authorizeSystemProxy() error = nil for backend-only interface %q", interfaceType)
}
}
}

1
backend/internal/model/models.go
View File

@@ -54,6 +54,7 @@ const (
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1"
ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2"
ChannelInterfaceXAIVideo ChannelInterfaceType = "xai-video"
ApiCallStatusSucceeded ApiCallStatus = "succeeded"
ApiCallStatusFailed ApiCallStatus = "failed"

2
backend/internal/service/admin.go
View File

@@ -558,7 +558,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
switch value {
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
return true
default:
return false

20
backend/internal/service/admin_channel_test.go
View File

@@ -31,6 +31,26 @@ func TestChannelFromRequestStoresInterfaceType(t *testing.T) {
}
}
func TestChannelFromRequestStoresXAIVideoInterfaceType(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer server.Close()
channel, err := channelFromRequest(ChannelRequest{
Name: "xAI Video",
BaseURL: server.URL + "/v1",
APIKey: "secret",
InterfaceType: "xai-video",
Models: []string{"grok-imagine-video-1.5"},
}, model.ModelChannel{})
if err != nil {
t.Fatalf("channelFromRequest() error = %v", err)
}
if channel.InterfaceType != model.ChannelInterfaceXAIVideo {
t.Fatalf("InterfaceType = %q", channel.InterfaceType)
}
}
func TestMergeChannelRequestSupportsEnabledOnlyPatch(t *testing.T) {
enabled := false
req := mergeChannelRequest(ChannelRequest{Enabled: &enabled}, model.ModelChannel{

2
backend/internal/service/channel_models.go
View File

@@ -200,7 +200,7 @@ func capabilityForChannel(channel model.ModelChannel) string {
switch channel.InterfaceType {
case model.ChannelInterfaceOpenAIImage:
return "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2:
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
return "video"
default:
return "text"

24
backend/internal/service/provider.go
View File

@@ -574,12 +574,16 @@ func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]
}
id := resumedProviderRequestID(ctx)
var created map[string]interface{}
if id == "" && isGrokVideoConfig(input.Config) {
if id == "" && (input.Config.InterfaceType == "xai-video" || isGrokVideoConfig(input.Config)) {
requestBody, err := grokVideoBody(input)
if err != nil {
return nil, err
}
if err := postJSON(ctx, input.Config, "/videos", requestBody, &created); err != nil {
createPath := "/videos"
if input.Config.InterfaceType == "xai-video" {
createPath = "/videos/generations"
}
if err := postJSON(ctx, input.Config, createPath, requestBody, &created); err != nil {
return nil, err
}
} else if id == "" {
@@ -608,17 +612,11 @@ func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]
}
}
if id == "" {
id = stringField(created, "id")
if id == "" {
id = stringField(created, "task_id")
}
id = firstNonEmptyString(stringField(created, "id"), stringField(created, "request_id"), stringField(created, "task_id"))
}
if id == "" {
if data, ok := created["data"].(map[string]interface{}); ok {
id = stringField(data, "id")
if id == "" {
id = stringField(data, "task_id")
}
id = firstNonEmptyString(stringField(data, "id"), stringField(data, "request_id"), stringField(data, "task_id"))
}
}
if id == "" {
@@ -633,7 +631,7 @@ func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]
state = data
}
status := strings.ToLower(stringField(state, "status"))
if status == "completed" || status == "succeeded" || status == "success" {
if status == "completed" || status == "succeeded" || status == "success" || status == "done" {
if videoURL := newAPIVideoResultURL(state); videoURL != "" {
data, mimeType, err := getExternalBinary(withProviderRequestKind(ctx, "download"), videoURL)
if err != nil {
@@ -664,7 +662,7 @@ func newAPIVideoResultURL(state map[string]interface{}) string {
func nestedNewAPIVideoResultURL(payload map[string]interface{}, allowResultURL bool, depth int) string {
if depth < 2 {
for _, key := range []string{"data", "result"} {
for _, key := range []string{"data", "result", "video"} {
if nested, ok := payload[key].(map[string]interface{}); ok {
if videoURL := nestedNewAPIVideoResultURL(nested, true, depth+1); videoURL != "" {
return videoURL
@@ -986,7 +984,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
allowed := map[string]map[string]bool{
"text": {"chat-completion": true, "openai-response": true},
"image": {"openai-image": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true},
}
if allowed[mode] != nil && !allowed[mode][interfaceType] {
return fmt.Errorf("接口类型 %s 不支持%s生成", interfaceType, mode)

61
backend/internal/service/provider_test.go
View File

@@ -369,6 +369,64 @@ func TestRunVideoTaskUsesJSONForGrokVideo(t *testing.T) {
}
}
func TestRunVideoTaskUsesXAIVideoGenerationEndpoint(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
paths := make([]string, 0, 3)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths = append(paths, r.Method+" "+r.URL.Path)
switch r.Method + " " + r.URL.Path {
case "POST /v1/videos/generations":
if contentType := r.Header.Get("Content-Type"); !strings.HasPrefix(contentType, "application/json") {
t.Errorf("Content-Type = %q, want application/json", contentType)
}
var body map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode request: %v", err)
}
if body["model"] != "grok-imagine-video-1.5" || body["prompt"] != "make it move" {
t.Errorf("request body = %#v", body)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"request_id":"video-1"}`))
case "GET /v1/videos/video-1":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"status":"done","video":{"url":"` + server.URL + `/files/video.mp4"}}`))
case "GET /files/video.mp4":
if authorization := r.Header.Get("Authorization"); authorization != "" {
t.Errorf("file Authorization = %q, want empty", authorization)
}
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("video"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
result, err := runVideoTask(context.Background(), canvasGenerationInput{
Prompt: "make it move",
Config: providerConfig{
BaseURL: server.URL + "/v1",
APIKey: "test-key",
Model: "grok-imagine-video-1.5",
InterfaceType: "xai-video",
VideoSeconds: "10",
},
})
if err != nil {
t.Fatalf("runVideoTask() error = %v", err)
}
video, ok := result["video"].(map[string]interface{})
if !ok || video["dataUrl"] != "data:video/mp4;base64,dmlkZW8=" {
t.Fatalf("video = %#v", result["video"])
}
want := "POST /v1/videos/generations,GET /v1/videos/video-1,GET /files/video.mp4"
if got := strings.Join(paths, ","); got != want {
t.Fatalf("paths = %q, want %q", got, want)
}
}
func TestNewAPIVideoPromptKeepsTextOnlyPromptUnchanged(t *testing.T) {
input := canvasGenerationInput{
Prompt: "make it move",
@@ -622,6 +680,9 @@ func TestValidateGenerationInterfaceRejectsMismatchedType(t *testing.T) {
if err := validateGenerationInterface("video", "newapi-channel-2"); err != nil {
t.Fatalf("validateGenerationInterface() error = %v", err)
}
if err := validateGenerationInterface("video", "xai-video"); err != nil {
t.Fatalf("validateGenerationInterface() error = %v", err)
}
}
func TestProcessTaskValidatesInterfaceBeforeHydratingMedia(t *testing.T) {

3
web/src/components/layout/app-config-modal.tsx
View File

@@ -80,6 +80,7 @@ const channelProtocolOptions = [
{ label: "NewAPI 视频", value: "newapi" },
{ label: "NewAPI 渠道 1", value: "newapi-channel-1" },
{ label: "NewAPI 渠道 2", value: "newapi-channel-2" },
{ label: "xAI / Sub2API 视频", value: "xai-video" },
],
},
];
@@ -750,6 +751,8 @@ function channelProtocolLabel(channel: ModelChannel) {
return "NewAPI 渠道 1";
case "newapi-channel-2":
return "NewAPI 渠道 2";
case "xai-video":
return "xAI / Sub2API 视频";
default:
return "OpenAI 自动兼容";
}

8
web/src/pages/admin/channels/channels-page.tsx
View File

@@ -19,7 +19,7 @@ type ChannelFormValues = { name: string; baseUrl: string; apiKey?: string; inter
const interfaceTypeOptions = [
{ label: "文本", options: [{ label: "Chat Completions", value: "chat-completion" }, { label: "OpenAI Responses", value: "openai-response" }] },
{ label: "图片", options: [{ label: "OpenAI Images", value: "openai-image" }] },
{ label: "视频", options: [{ label: "NewAPI 视频", value: "newapi" }, { label: "NewAPI 渠道 1", value: "newapi-channel-1" }, { label: "NewAPI 渠道 2", value: "newapi-channel-2" }] },
{ label: "视频", options: [{ label: "NewAPI 视频", value: "newapi" }, { label: "NewAPI 渠道 1", value: "newapi-channel-1" }, { label: "NewAPI 渠道 2", value: "newapi-channel-2" }, { label: "xAI / Sub2API 视频", value: "xai-video" }] },
];
export default function ChannelsPage() {
@@ -134,7 +134,7 @@ export default function ChannelsPage() {
const columns: ColumnsType<ModelChannel> = [
{ title: "渠道", dataIndex: "name", render: (_, channel) => <div><div className="font-medium">{channel.name}</div><div className="max-w-lg truncate text-xs text-foreground/45">{channel.baseUrl}</div></div> },
{ title: "接口类型", dataIndex: "interfaceType", width: 160, render: (value: ChannelInterfaceType) => <Tag bordered={false} color={value === "newapi-channel-1" ? "green" : value === "newapi" ? "orange" : value === "newapi-channel-2" ? "purple" : "blue"}>{interfaceTypeLabel(value)}</Tag> },
{ title: "接口类型", dataIndex: "interfaceType", width: 160, render: (value: ChannelInterfaceType) => <Tag bordered={false} color={value === "newapi-channel-1" ? "green" : value === "newapi" ? "orange" : value === "newapi-channel-2" ? "purple" : value === "xai-video" ? "cyan" : "blue"}>{interfaceTypeLabel(value)}</Tag> },
{ title: "模型", dataIndex: "models", width: 100, render: (models: string[]) => `${models?.length || 0} 个` },
{ title: "密钥", dataIndex: "hasApiKey", width: 100, render: (configured) => <Tag bordered={false} color={configured ? "success" : "default"}>{configured ? "已配置" : "未配置"}</Tag> },
{ title: "状态", dataIndex: "enabled", width: 100, render: (enabled) => <Tag bordered={false} color={enabled !== false ? "success" : "default"}>{enabled !== false ? "已启用" : "已停用"}</Tag> },
@@ -171,5 +171,5 @@ export default function ChannelsPage() {
function positiveInt(value: string | null, fallback: number) { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; }
function normalizePageSize(value: string | null) { const parsed = positiveInt(value, 20); return [20, 50, 100].includes(parsed) ? parsed : 20; }
function normalizeStatus(value: string | null): "all" | "enabled" | "disabled" { return value === "enabled" || value === "disabled" ? value : "all"; }
function normalizeInterface(value: string | null): "all" | ChannelInterfaceType { return ["chat-completion", "openai-response", "openai-image", "newapi", "newapi-channel-1", "newapi-channel-2"].includes(value || "") ? value as ChannelInterfaceType : "all"; }
function interfaceTypeLabel(value?: ChannelInterfaceType) { return ({ "chat-completion": "Chat Completions", "openai-response": "OpenAI Responses", "openai-image": "OpenAI Images", newapi: "NewAPI 视频", "newapi-channel-1": "NewAPI 渠道 1", "newapi-channel-2": "NewAPI 渠道 2" } as Record<string, string>)[value || ""] || "未设置"; }
function normalizeInterface(value: string | null): "all" | ChannelInterfaceType { return ["chat-completion", "openai-response", "openai-image", "newapi", "newapi-channel-1", "newapi-channel-2", "xai-video"].includes(value || "") ? value as ChannelInterfaceType : "all"; }
function interfaceTypeLabel(value?: ChannelInterfaceType) { return ({ "chat-completion": "Chat Completions", "openai-response": "OpenAI Responses", "openai-image": "OpenAI Images", newapi: "NewAPI 视频", "newapi-channel-1": "NewAPI 渠道 1", "newapi-channel-2": "NewAPI 渠道 2", "xai-video": "xAI / Sub2API 视频" } as Record<string, string>)[value || ""] || "未设置"; }

2
web/src/pages/admin/components/channel-model-manager.tsx
View File

@@ -193,7 +193,7 @@ export function ChannelModelManager({ channel, onClose, onChanged }: { channel:
function capabilityFromInterface(value?: ModelChannel["interfaceType"]): ChannelModel["capability"] {
if (value === "openai-image") return "image";
if (value === "newapi" || value === "newapi-channel-1" || value === "newapi-channel-2") return "video";
if (value === "newapi" || value === "newapi-channel-1" || value === "newapi-channel-2" || value === "xai-video") return "video";
return "text";
}

23
web/src/services/api/video.ts
View File

@@ -8,8 +8,9 @@ import { buildApiUrl, isSystemProxyBaseUrl, modelOptionName, resolveModelRequest
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
type VideoResponse = { id?: string; request_id?: string; task_id?: string; status?: string; error?: { message?: string } };
type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string };
type ResolvedAiConfig = ReturnType<typeof resolveModelRequestConfig>;
type SeedanceTask = {
id: string;
task_id?: string;
@@ -77,9 +78,9 @@ export async function storeGeneratedVideo(result: VideoGenerationResult): Promis
throw new Error("视频接口没有返回可播放的视频");
}
async function createOpenAIVideoTask(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise<VideoGenerationTask> {
async function createOpenAIVideoTask(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise<VideoGenerationTask> {
const modelName = modelOptionName(model);
if (modelName.toLowerCase().includes("grok")) {
if (config.interfaceType === "xai-video" || modelName.toLowerCase().includes("grok")) {
const images = await Promise.all(references.slice(0, 7).map((image) => imageToDataUrl(image)));
const seconds = normalizeVideoSeconds(config.videoSeconds);
const payload = {
@@ -91,9 +92,11 @@ async function createOpenAIVideoTask(config: AiConfig, model: string, prompt: st
...(images.length ? { image: images[0], images } : {}),
};
try {
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, "/videos"), payload, { headers: aiHeaders(config, "application/json"), signal: options?.signal })).data);
if (!created.id) throw new Error("视频接口没有返回任务 ID");
return { id: created.id, provider: "openai", model };
const createPath = config.interfaceType === "xai-video" ? "/videos/generations" : "/videos";
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, createPath), payload, { headers: aiHeaders(config, "application/json"), signal: options?.signal })).data);
const id = videoTaskId(created);
if (!id) throw new Error("视频接口没有返回任务 ID");
return { id, provider: "openai", model };
} catch (error) {
throw new Error(readAxiosError(error, "视频任务创建失败"));
}
@@ -116,10 +119,10 @@ async function createOpenAIVideoTask(config: AiConfig, model: string, prompt: st
}
}
async function pollOpenAIVideoTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
async function pollOpenAIVideoTask(config: ResolvedAiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
try {
const video = unwrapVideoResponse((await axios.get<ApiVideoResponse>(aiApiUrl(config, `/videos/${task.id}`), { headers: aiHeaders(config), signal: options?.signal })).data);
if (video.status === "completed") {
if (video.status === "completed" || video.status === "succeeded" || video.status === "success" || video.status === "done") {
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${task.id}/content`), { headers: aiHeaders(config), responseType: "blob", signal: options?.signal });
await assertVideoBlob(content.data);
return { status: "completed", result: { blob: content.data } };
@@ -338,6 +341,10 @@ function unwrapVideoResponse(payload: ApiVideoResponse) {
return unwrapEnvelope(payload, "接口没有返回视频任务");
}
function videoTaskId(payload: VideoResponse) {
return payload.id || payload.request_id || payload.task_id || "";
}
function unwrapSeedanceTask(payload: ApiEnvelope<SeedanceTask>) {
return unwrapEnvelope(payload, "Seedance 接口没有返回任务");
}

8
web/src/stores/use-config-store.ts
View File

@@ -7,7 +7,7 @@ import { scopedLocalStorage } from "@/lib/user-scope";
import { normalizeVideoDuration, normalizeVideoResolution } from "@/lib/video-generation-options";
export type ApiCallFormat = "openai" | "gemini";
export type ChannelInterfaceType = "chat-completion" | "openai-response" | "openai-image" | "newapi" | "newapi-channel-1" | "newapi-channel-2";
export type ChannelInterfaceType = "chat-completion" | "openai-response" | "openai-image" | "newapi" | "newapi-channel-1" | "newapi-channel-2" | "xai-video";
export type ModelChannel = {
id: string;
@@ -441,14 +441,14 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) {
}
export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) {
if (interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2") return "";
if (interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2" || interfaceType === "xai-video") return "";
return OPENAI_BASE_URL;
}
function capabilityForChannelInterface(interfaceType?: ChannelInterfaceType): ModelCapability | undefined {
if (interfaceType === "chat-completion" || interfaceType === "openai-response") return "text";
if (interfaceType === "openai-image") return "image";
if (interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2") return "video";
if (interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2" || interfaceType === "xai-video") return "video";
return undefined;
}
@@ -457,7 +457,7 @@ function normalizeApiFormat(apiFormat: unknown): ApiCallFormat {
}
function normalizeChannelInterfaceType(value: unknown): ChannelInterfaceType | undefined {
return value === "chat-completion" || value === "openai-response" || value === "openai-image" || value === "newapi" || value === "newapi-channel-1" || value === "newapi-channel-2" ? value : undefined;
return value === "chat-completion" || value === "openai-response" || value === "openai-image" || value === "newapi" || value === "newapi-channel-1" || value === "newapi-channel-2" || value === "xai-video" ? value : undefined;
}
function uniqueRawModels(models: string[]) {