feat(media): 本地存储 - 支持自定义访问地址与短时签名链接

This commit is contained in:
ddCat
2026-08-04 22:13:38 +08:00
parent 660da45fdf
commit 8e0deaff48
6 changed files with 225 additions and 61 deletions

View File

@@ -261,6 +261,31 @@ func RegisterUserDataRoutes(r *gin.RouterGroup, svc *service.Service) {
}
c.DataFromReader(stream.StatusCode, stream.ContentLength, resource.MimeType, stream.Body, nil)
})
r.GET("/public/resources/:id/file", func(c *gin.Context) {
stream, err := svc.OpenPublicResourceRange(c.Param("id"), c.Query("expires"), c.Query("signature"), c.GetHeader("Range"))
if err != nil {
failService(c, err)
return
}
defer stream.Body.Close()
resource := stream.Resource
if resource.MimeType == "" {
resource.MimeType = "application/octet-stream"
}
c.Header("Cache-Control", "public, max-age=0, must-revalidate")
c.Header("Accept-Ranges", "bytes")
c.Header("Referrer-Policy", "no-referrer")
c.Header("X-Content-Type-Options", "nosniff")
if stream.ContentRange != "" {
c.Header("Content-Range", stream.ContentRange)
}
if seeker, ok := stream.Body.(io.ReadSeeker); ok {
c.Header("Content-Type", resource.MimeType)
http.ServeContent(c.Writer, c.Request, resource.ID, resource.UpdatedAt, seeker)
return
}
c.DataFromReader(stream.StatusCode, stream.ContentLength, resource.MimeType, stream.Body, nil)
})
r.GET("/assets", func(c *gin.Context) {
user, err := currentUser(c, svc)
if err != nil {

View File

@@ -15,7 +15,6 @@ import (
"net/http"
"net/textproto"
"net/url"
"path/filepath"
"strconv"
"strings"
"time"
@@ -251,50 +250,7 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ
if resource.Status != "ready" {
return errors.New("任务参考资源尚未上传完成")
}
// 本地素材自动迁移到 OSS
if resource.Provider == "local" {
setting, storageSettingID, useOSS, ossErr := s.activeResourceOSSSetting(userID)
if ossErr != nil || !useOSS || setting.Provider != "aliyun" {
return errors.New("当前 JSON 视频协议的参考素材需要公网可访问地址,请启用 OSS 后重新上传该素材")
}
// 读取本地文件并上传到 OSS
_, body, readErr := s.OpenResource(userID, resourceID)
if readErr != nil {
return fmt.Errorf("读取本地素材失败:%w", readErr)
}
ossKey := ossObjectKey(setting, userID, resource.Kind, filepath.Base(resource.ObjectKey), time.Now())
if _, uploadErr := putOSSObject(setting, ossKey, resource.MimeType, resource.Size, body); uploadErr != nil {
body.Close()
return fmt.Errorf("素材迁移到 OSS 失败:%w", uploadErr)
}
body.Close()
// 更新资源记录
resource.Provider = "aliyun"
resource.ObjectKey = ossKey
resource.Endpoint = setting.Endpoint
resource.Bucket = setting.Bucket
resource.StorageSettingID = storageSettingID
if saveErr := s.repo.SaveResource(resource); saveErr != nil {
return fmt.Errorf("更新资源记录失败:%w", saveErr)
}
// 生成签名 URL
signedURL, signErr := signedOSSObjectURL(setting, ossKey, time.Now().Add(providerResourceURLTTL))
if signErr != nil {
return fmt.Errorf("生成 OSS 签名地址失败:%w", signErr)
}
media.URL = signedURL
media.DataURL = ""
media.MimeType = firstNonEmpty(media.MimeType, resource.MimeType)
return nil
}
setting, err := s.ossSettingForResource(userID, resource)
if err != nil {
return err
}
if setting.Provider != "aliyun" {
return errors.New("当前 JSON 视频协议的私有参考素材暂时只支持阿里云 OSS 签名地址")
}
signedURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(providerResourceURLTTL))
signedURL, err := s.directResourceURL(resource, time.Now().Add(providerResourceURLTTL))
if err != nil {
return fmt.Errorf("生成 JSON 视频协议参考素材地址失败:%w", err)
}

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
@@ -57,26 +58,105 @@ func (s *Service) Resource(userID string, id string) (*model.Resource, error) {
return resource, err
}
// DirectResourceURL 先校验资源归属,再为私有 OSS 对象签发短时下载地址;本地资源继续由应用流式读取。
// DirectResourceURL 先校验资源归属,再按实际存储位置签发短时下载地址。
func (s *Service) DirectResourceURL(userID string, id string) (string, error) {
resource, err := s.repo.ResourceForUser(userID, id)
if err != nil {
return "", err
}
return s.directResourceURL(resource, time.Now().Add(directResourceURLTTL))
}
func (s *Service) directResourceURL(resource *model.Resource, expiresAt time.Time) (string, error) {
if resource == nil {
return "", errors.New("资源不存在")
}
if resource.Status != model.ResourceStatusReady {
return "", BadAuthRequest("资源尚未上传完成")
}
if resource.Provider == "local" {
return "", BadAuthRequest("当前资源未存储在 OSS")
return s.signedPublicResourceURL(resource.ID, expiresAt)
}
setting, err := s.ossSettingForResource(userID, resource)
setting, err := s.ossSettingForResource(resource.UserID, resource)
if err != nil {
return "", err
}
setting.Provider = firstNonEmpty(resource.Provider, setting.Provider)
setting.Endpoint = firstNonEmpty(resource.Endpoint, setting.Endpoint)
setting.Bucket = firstNonEmpty(resource.Bucket, setting.Bucket)
return signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(directResourceURLTTL))
return signedOSSObjectURL(setting, resource.ObjectKey, expiresAt)
}
func (s *Service) signedPublicResourceURL(resourceID string, expiresAt time.Time) (string, error) {
baseURL, err := s.publicResourceBaseURL()
if err != nil {
return "", err
}
expires := strconv.FormatInt(expiresAt.UTC().Unix(), 10)
signature, err := s.signPublicResource(resourceID, expires)
if err != nil {
return "", err
}
baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/api/public/resources/" + url.PathEscape(resourceID) + "/file"
query := baseURL.Query()
query.Set("expires", expires)
query.Set("signature", signature)
baseURL.RawQuery = query.Encode()
return baseURL.String(), nil
}
func (s *Service) verifyPublicResourceSignature(resourceID string, expires string, signature string) error {
if strings.TrimSpace(signature) == "" || !decimalDigits(expires) {
return Forbidden("匿名下载链接无效")
}
expiresAt, err := strconv.ParseInt(expires, 10, 64)
if err != nil || time.Now().UTC().Unix() > expiresAt {
return Forbidden("匿名下载链接已过期")
}
expected, err := s.signPublicResource(resourceID, expires)
if err != nil {
return err
}
if !hmac.Equal([]byte(expected), []byte(signature)) {
return Forbidden("匿名下载链接无效")
}
return nil
}
func (s *Service) signPublicResource(resourceID string, expires string) (string, error) {
key, err := s.settingsEncryptionKey()
if err != nil {
return "", err
}
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(resourceID + "\n" + expires))
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)), nil
}
func (s *Service) publicResourceBaseURL() (*url.URL, error) {
_, setting, err := s.readOSSSetting()
if err != nil {
return nil, err
}
raw := firstNonEmpty(setting.PublicBaseURL, os.Getenv("CANVAS_PUBLIC_BASE_URL"))
if raw == "" {
return nil, BadAuthRequest("服务器本地存储尚未配置服务器访问地址")
}
return validatePublicResourceBaseURL(raw)
}
func validatePublicResourceBaseURL(raw string) (*url.URL, error) {
parsed, err := ValidateOutboundURL(raw)
if err != nil {
return nil, err
}
if parsed.RawQuery != "" || parsed.Fragment != "" {
return nil, BadAuthRequest("服务器访问地址不能包含查询参数或片段")
}
if strings.HasSuffix(strings.TrimRight(parsed.Path, "/"), "/api") {
return nil, BadAuthRequest("服务器访问地址请填写根地址,不要包含 /api")
}
return parsed, nil
}
func (s *Service) UploadResource(userID string, header *multipart.FileHeader, kind string, width int, height int, durationMs int64) (*model.Resource, error) {
@@ -168,6 +248,20 @@ func (s *Service) OpenResourceRange(userID string, id string, rangeHeader string
return s.openResourceRange(userID, resource, rangeHeader)
}
func (s *Service) OpenPublicResourceRange(id string, expires string, signature string, rangeHeader string) (*ResourceStream, error) {
resource, err := s.repo.Resource(id)
if err != nil {
return nil, Forbidden("匿名下载链接无效")
}
if resource.Provider != "local" {
return nil, Forbidden("匿名下载链接无效")
}
if err := s.verifyPublicResourceSignature(resource.ID, expires, signature); err != nil {
return nil, err
}
return s.openResourceRange(resource.UserID, resource, rangeHeader)
}
func (s *Service) openResourceRange(userID string, resource *model.Resource, rangeHeader string) (*ResourceStream, error) {
if resource.Status != model.ResourceStatusReady {
return nil, BadAuthRequest("资源尚未上传完成")

View File

@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
@@ -109,16 +110,74 @@ func TestHydrateNewAPIChannel1ResourceUsesSignedOSSURL(t *testing.T) {
}
}
func TestHydrateNewAPIChannel1ResourceRejectsLocalStorage(t *testing.T) {
func TestHydrateNewAPIChannel1ResourceUsesSignedLocalURL(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer server.Close()
svc := newResourceTestService(t)
settingJSON, _ := json.Marshal(ossSettingValue{Provider: "aliyun", PublicBaseURL: server.URL})
if err := svc.repo.SaveSystemSetting(&model.SystemSetting{Key: ossSettingKey, ValueJSON: string(settingJSON)}); err != nil {
t.Fatal(err)
}
resource := model.Resource{ID: "resource-local", UserID: "user-1", Status: model.ResourceStatusReady, Provider: "local", ObjectKey: "local.png"}
if err := svc.repo.CreateResource(&resource); err != nil {
t.Fatal(err)
}
err := svc.hydrateProviderMedia("user-1", &providerMedia{StorageKey: "resource:resource-local"}, true)
if err == nil || !strings.Contains(err.Error(), "启用 OSS") {
media := providerMedia{StorageKey: "resource:resource-local"}
if err := svc.hydrateProviderMedia("user-1", &media, true); err != nil {
t.Fatalf("hydrateProviderMedia() error = %v", err)
}
if !strings.HasPrefix(media.URL, server.URL+"/api/public/resources/resource-local/file?") || !strings.Contains(media.URL, "signature=") || media.DataURL != "" {
t.Fatalf("media = %#v", media)
}
stored, err := svc.repo.Resource("resource-local")
if err != nil || stored.Provider != "local" {
t.Fatalf("resource provider changed: %#v, %v", stored, err)
}
}
func TestPublicResourceSignatureRejectsExpiredAndAlteredLinks(t *testing.T) {
svc := newResourceTestService(t)
expires := strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10)
signature, err := svc.signPublicResource("resource-local", expires)
if err != nil {
t.Fatal(err)
}
if err := svc.verifyPublicResourceSignature("resource-local", expires, signature); err != nil {
t.Fatalf("verifyPublicResourceSignature() error = %v", err)
}
if err := svc.verifyPublicResourceSignature("resource-other", expires, signature); err == nil {
t.Fatal("verifyPublicResourceSignature() accepted another resource ID")
}
expired := strconv.FormatInt(time.Now().Add(-time.Minute).Unix(), 10)
expiredSignature, err := svc.signPublicResource("resource-local", expired)
if err != nil {
t.Fatal(err)
}
if err := svc.verifyPublicResourceSignature("resource-local", expired, expiredSignature); err == nil {
t.Fatal("verifyPublicResourceSignature() accepted an expired link")
}
}
func TestUpdateOSSSettingRequiresLocalServerAddress(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer server.Close()
svc := newResourceTestService(t)
admin := &model.User{ID: "admin-1", Role: model.UserRoleAdmin}
if _, err := svc.UpdateOSSSetting(admin, OSSSettingRequest{Provider: "aliyun"}); err == nil || !strings.Contains(err.Error(), "服务器访问地址") {
t.Fatalf("UpdateOSSSetting() error = %v", err)
}
if _, err := svc.UpdateOSSSetting(admin, OSSSettingRequest{Provider: "aliyun", PublicBaseURL: server.URL + "/api"}); err == nil || !strings.Contains(err.Error(), "不要包含 /api") {
t.Fatalf("UpdateOSSSetting(/api) error = %v", err)
}
setting, err := svc.UpdateOSSSetting(admin, OSSSettingRequest{Provider: "aliyun", PublicBaseURL: server.URL})
if err != nil {
t.Fatal(err)
}
if setting.Enabled || setting.PublicBaseURL != server.URL {
t.Fatalf("setting = %#v", setting)
}
}
func TestActiveResourceOSSSettingPrefersUserVersion(t *testing.T) {

View File

@@ -85,6 +85,14 @@ func (s *Service) UpdateOSSSetting(actor *model.User, req OSSSettingRequest) (*P
if err != nil {
return nil, err
}
if !next.Enabled {
if next.PublicBaseURL == "" {
return nil, BadAuthRequest("服务器本地存储需要填写服务器访问地址")
}
if _, err := validatePublicResourceBaseURL(next.PublicBaseURL); err != nil {
return nil, fmt.Errorf("服务器访问地址无效:%w", err)
}
}
stored := next
stored.AccessKeySecret, err = s.encryptSettingSecret(next.AccessKeySecret)
if err != nil {

View File

@@ -1,5 +1,5 @@
import { App, Button, Form, Input, Segmented, Space, Tag } from "antd";
import { Cloud, HardDrive, Info, KeyRound, ShieldCheck } from "lucide-react";
import { Cloud, Globe, HardDrive, Info, KeyRound, LocateFixed, ShieldCheck } from "lucide-react";
import { useEffect, useMemo, useState, type ReactNode } from "react";
import { getAdminOSSSetting, updateAdminOSSSetting, type AdminOSSSetting } from "@/services/api/auth";
@@ -8,7 +8,7 @@ import { AdminPageFrame } from "../components/admin-shell";
import { configuredSecretText, SettingsSectionCard } from "../components/admin-ui";
type StorageMode = "local" | "oss";
type OSSFormValues = { mode: StorageMode; region?: string; endpoint?: string; bucket?: string; accessKeyId?: string; accessKeySecret?: string; pathPrefix?: string };
type OSSFormValues = { mode: StorageMode; publicBaseUrl?: string; region?: string; endpoint?: string; bucket?: string; accessKeyId?: string; accessKeySecret?: string; pathPrefix?: string };
export default function StorageSettingsPage() {
const { message } = App.useApp();
@@ -31,13 +31,14 @@ export default function StorageSettingsPage() {
await form.validateFields();
// 本地模式会卸载 OSS 字段,读取完整 store 才能保留已保存的 OSS 配置。
const values = form.getFieldsValue(true);
if (values.mode === "local" && !values.publicBaseUrl?.trim()) return message.error("请填写服务器访问地址");
if (values.mode === "oss" && !values.accessKeySecret?.trim() && !setting?.hasAccessKeySecret) return message.error("请填写 AccessKey Secret");
if (values.mode === "oss" && !values.endpoint?.trim()) return message.error("请填写 OSS Endpoint");
if (values.mode === "oss" && !values.bucket?.trim()) return message.error("请填写 OSS Bucket");
if (values.mode === "oss" && !values.accessKeyId?.trim()) return message.error("请填写 AccessKey ID");
setSaving(true);
try {
const result = await updateAdminOSSSetting({ enabled: values.mode === "oss", provider: "aliyun", region: values.region?.trim() || "", endpoint: values.endpoint?.trim() || "", bucket: values.bucket?.trim() || "", accessKeyId: values.accessKeyId?.trim() || "", accessKeySecret: values.accessKeySecret?.trim() || "", publicBaseUrl: "", pathPrefix: values.pathPrefix?.trim() || "" });
const result = await updateAdminOSSSetting({ enabled: values.mode === "oss", provider: "aliyun", region: values.region?.trim() || "", endpoint: values.endpoint?.trim() || "", bucket: values.bucket?.trim() || "", accessKeyId: values.accessKeyId?.trim() || "", accessKeySecret: values.accessKeySecret?.trim() || "", publicBaseUrl: values.publicBaseUrl?.trim() || "", pathPrefix: values.pathPrefix?.trim() || "" });
setSetting(result.setting);
form.setFieldsValue(formValues(result.setting));
message.success("存储配置已保存");
@@ -52,7 +53,7 @@ export default function StorageSettingsPage() {
<AdminPageFrame title="存储服务" description="配置新增资源的默认存储位置">
<div className="space-y-4 pt-4">
<div className="border-b border-border px-1 pb-4 text-foreground/75">
<div className="flex items-start gap-3"><span className="mt-0.5 grid size-8 shrink-0 place-items-center rounded-md bg-muted/60"><Info className="size-4" /></span><div><div className="text-sm font-semibold text-foreground">资源存储规则</div><p className="mt-1 text-xs leading-6 text-foreground/55">存储类型只影响之后新增的媒体资源,不迁移或改写历史文件。服务器本地与 OSS 资源均通过登录鉴权接口读取。</p></div></div>
<div className="flex items-start gap-3"><span className="mt-0.5 grid size-8 shrink-0 place-items-center rounded-md bg-muted/60"><Info className="size-4" /></span><div><div className="text-sm font-semibold text-foreground">资源存储规则</div><p className="mt-1 text-xs leading-6 text-foreground/55">存储类型只影响之后新增的媒体资源,不迁移或改写历史文件。日常读取继续校验登录态,仅在模型必须通过 URL 拉取本地素材时生成短时签名链接。</p></div></div>
</div>
<SettingsSectionCard
icon={<Cloud className="size-4" />}
@@ -82,19 +83,40 @@ export default function StorageSettingsPage() {
<Form.Item name="accessKeySecret" label={setting?.hasAccessKeySecret ? `AccessKey Secret(${configuredSecretText})` : "AccessKey Secret"}><Input.Password autoComplete="new-password" placeholder={setting?.hasAccessKeySecret ? "留空保留原密钥" : "阿里云 AccessKey Secret"} /></Form.Item>
</>
) : (
<div className="md:col-span-2 border-t border-border py-4 text-sm text-foreground/65">
新增资源将写入后端数据目录的 <code className="text-foreground">resources/</code>,部署时需持久化该数据卷。
</div>
<>
<Form.Item className="md:col-span-2" label="服务器访问地址" required tooltip="后端可从公网或模型服务所在网络访问的根地址,用于生成本地资源的短时签名链接。">
<Space.Compact className="w-full">
<Form.Item
name="publicBaseUrl"
noStyle
rules={[
{ required: true, message: "请填写服务器访问地址" },
{ type: "url", message: "请填写完整的 http/https 地址" },
]}
>
<Input className="min-w-0" autoComplete="off" placeholder="https://canvas.example.com 或 http://103.242.14.110:300" prefix={<Globe className="size-4 text-foreground/35" />} />
</Form.Item>
<Button icon={<LocateFixed className="size-4" />} onClick={() => form.setFieldValue("publicBaseUrl", window.location.origin)}>使用当前地址</Button>
</Space.Compact>
</Form.Item>
<div className="md:col-span-2 border-t border-border py-4 text-xs leading-6 text-foreground/60">
<div className="font-medium text-foreground/80">地址怎么设置</div>
<div className="mt-1">域名已反向代理到本服务时,可点击“使用当前地址”,例如 <code className="text-foreground">https://ddcat.pronhubcn.com</code>。</div>
<div>直接开放服务器端口时,填写完整的公网 IP 和端口,例如 <code className="text-foreground">http://103.242.14.110:300</code>。</div>
<div>地址末尾不要填写 <code className="text-foreground">/api</code>;保存前可在浏览器访问 <code className="text-foreground">&lt;服务器访问地址&gt;/api/health</code>,看到 <code className="text-foreground">status: ok</code> 即表示入口正确。</div>
<div className="mt-1 text-foreground/45">新增资源写入 <code>CANVAS_BACKEND_DATA_DIR/resources/</code>,部署时必须持久化该数据卷。供外部模型使用时,以上地址也必须能被模型服务访问,生产环境建议使用 HTTPS。</div>
</div>
</>
)}
</div>
</Form>
</SettingsSectionCard>
<div className="grid border-y border-border text-xs text-foreground/55 sm:grid-cols-3 sm:divide-x sm:divide-border"><Notice icon={mode === "oss" ? <Cloud className="size-3.5" /> : <HardDrive className="size-3.5" />} text={mode === "oss" ? "新资源写入阿里云 OSS" : "新资源写入服务器数据卷"} /><Notice icon={<ShieldCheck className="size-3.5" />} text="历史资源位置保持不变" /><Notice icon={<KeyRound className="size-3.5" />} text="资源读取继续校验登录态" /></div>
<div className="grid border-y border-border text-xs text-foreground/55 sm:grid-cols-3 sm:divide-x sm:divide-border"><Notice icon={mode === "oss" ? <Cloud className="size-3.5" /> : <HardDrive className="size-3.5" />} text={mode === "oss" ? "新资源写入阿里云 OSS" : "新资源写入服务器数据卷"} /><Notice icon={<ShieldCheck className="size-3.5" />} text="历史资源位置保持不变" /><Notice icon={<KeyRound className="size-3.5" />} text="外部链接仅在签名有效期内可用" /></div>
</div>
</AdminPageFrame>
);
}
function formValues(setting?: AdminOSSSetting | null): OSSFormValues { return { mode: setting?.enabled ? "oss" : "local", region: setting?.region || "", endpoint: setting?.endpoint || "", bucket: setting?.bucket || "", accessKeyId: setting?.accessKeyId || "", accessKeySecret: "", pathPrefix: setting?.pathPrefix || "" }; }
function formValues(setting?: AdminOSSSetting | null): OSSFormValues { return { mode: setting?.enabled ? "oss" : "local", publicBaseUrl: setting?.publicBaseUrl || "", region: setting?.region || "", endpoint: setting?.endpoint || "", bucket: setting?.bucket || "", accessKeyId: setting?.accessKeyId || "", accessKeySecret: "", pathPrefix: setting?.pathPrefix || "" }; }
function formatTime(value?: string) { return value ? new Date(value).toLocaleString("zh-CN", { hour12: false }) : "--"; }
function Notice({ icon, text }: { icon: ReactNode; text: string }) { return <div className="flex items-center gap-2 px-3 py-2.5"><span className="text-foreground/40">{icon}</span><span>{text}</span></div>; }