524 lines
18 KiB
Go
524 lines
18 KiB
Go
package skills
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"mime"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"gorm.io/gorm"
|
|
"yingce/backend/internal/kernel"
|
|
"yingce/backend/internal/model"
|
|
)
|
|
|
|
const (
|
|
maxSkillPackageBytes = 20 << 20
|
|
maxSkillFileBytes = 8 << 20
|
|
maxSkillPackageFiles = 512
|
|
maxSkillPreviewBytes = 512 << 10
|
|
SkillPackageUploadMaxBytes = maxSkillPackageBytes + (1 << 20)
|
|
)
|
|
|
|
type SkillInstallRequest struct {
|
|
Name string
|
|
Description string
|
|
Tag string
|
|
IsPrivate bool
|
|
}
|
|
|
|
type SkillGitHubInstallRequest struct {
|
|
URL string `json:"url"`
|
|
Ref string `json:"ref"`
|
|
Subdir string `json:"subdir"`
|
|
Tag string `json:"tag"`
|
|
IsPrivate bool `json:"isPrivate"`
|
|
AutoUpdate bool `json:"autoUpdate"`
|
|
}
|
|
|
|
type SkillPackageFileItem struct {
|
|
Path string `json:"path"`
|
|
Kind string `json:"kind"`
|
|
MimeType string `json:"mimeType"`
|
|
Size int64 `json:"size"`
|
|
SHA256 string `json:"sha256"`
|
|
}
|
|
|
|
type SkillPackageFileContent struct {
|
|
File SkillPackageFileItem `json:"file"`
|
|
Content string `json:"content"`
|
|
Binary bool `json:"binary"`
|
|
}
|
|
|
|
type SkillPackageBundleFile struct {
|
|
Path string `json:"path"`
|
|
MimeType string `json:"mimeType"`
|
|
ContentBase64 string `json:"contentBase64"`
|
|
}
|
|
|
|
type SkillPackageBundle struct {
|
|
SkillID string `json:"skillId"`
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
VersionID string `json:"versionId"`
|
|
Version string `json:"version"`
|
|
ContentHash string `json:"contentHash"`
|
|
Files []SkillPackageBundleFile `json:"files"`
|
|
}
|
|
|
|
type SkillFileSearchResult struct {
|
|
Path string `json:"path"`
|
|
Line int `json:"line"`
|
|
Snippet string `json:"snippet"`
|
|
}
|
|
|
|
type skillPackageMetadata struct {
|
|
Name string
|
|
Description string
|
|
Version string
|
|
}
|
|
|
|
type skillPackageArchive struct {
|
|
Files map[string][]byte
|
|
Metadata skillPackageMetadata
|
|
ContentHash string
|
|
TotalBytes int64
|
|
}
|
|
|
|
type skillPackageSnapshot struct {
|
|
skill *model.Skill
|
|
version *model.SkillVersion
|
|
files []model.SkillFile
|
|
legacy bool
|
|
contents map[string][]byte
|
|
}
|
|
|
|
type githubSkillSpec struct {
|
|
Owner string
|
|
Repo string
|
|
Ref string
|
|
Subdir string
|
|
}
|
|
|
|
func (s *Service) EnsureSkillPackages() error {
|
|
skills, err := s.repo.SkillsForPackageEnsure()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for index := range skills {
|
|
skill := &skills[index]
|
|
version, files, healthErr := s.skillPackageHealth(skill)
|
|
if healthErr == nil {
|
|
if err := s.syncSkillPackageMetadata(skill, version, files); err != nil {
|
|
return fmt.Errorf("同步技能 %s 文件包元数据失败: %w", skill.ID, err)
|
|
}
|
|
// User-installed ZIP/GitHub skills are authoritative in skill_files and
|
|
// must never be replaced by the legacy instruction column at startup.
|
|
if skill.Source == skillSourceUser || skill.SourceType == "builtin" {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(skill.Instruction) == "" {
|
|
continue
|
|
}
|
|
archive, archiveErr := archiveFromMarkdown([]byte(skill.Instruction), truncateSkillMetadata(skill.Name, 80), truncateSkillMetadata(skill.Description, 500))
|
|
if archiveErr != nil {
|
|
log.Printf("技能 %s 的内置正文无法刷新文件包：%v", skill.ID, archiveErr)
|
|
continue
|
|
}
|
|
if version.ContentHash == archive.ContentHash {
|
|
continue
|
|
}
|
|
if err := s.addSkillArchiveVersion(skill, archive, skillSourceTypeForRepair(skill), skill.SourceURL, skill.SourceRef, skill.SourceSubdir, skill.SourceCommit, skill.AutoUpdate); err != nil {
|
|
return fmt.Errorf("刷新技能 %s 文件包失败: %w", skill.ID, err)
|
|
}
|
|
continue
|
|
}
|
|
|
|
// A legacy row may contain only skills.instruction. Rebuild that single
|
|
// entry when package metadata, files, or the archive is missing. If no
|
|
// source body exists, keep the service available and leave a diagnostic
|
|
// instead of failing every user request during boot.
|
|
if strings.TrimSpace(skill.Instruction) == "" {
|
|
log.Printf("技能 %s 文件包不完整且没有 legacy instruction，跳过自动修复：%v", skill.ID, healthErr)
|
|
continue
|
|
}
|
|
archive, archiveErr := archiveFromMarkdown([]byte(skill.Instruction), truncateSkillMetadata(skill.Name, 80), truncateSkillMetadata(skill.Description, 500))
|
|
if archiveErr != nil {
|
|
log.Printf("技能 %s 文件包损坏且 legacy instruction 无法重建：%v", skill.ID, archiveErr)
|
|
continue
|
|
}
|
|
if err := s.addSkillArchiveVersion(skill, archive, skillSourceTypeForRepair(skill), skill.SourceURL, skill.SourceRef, skill.SourceSubdir, skill.SourceCommit, skill.AutoUpdate); err != nil {
|
|
return fmt.Errorf("修复技能 %s 文件包失败: %w", skill.ID, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) InstallSkillUpload(userID string, sourceType string, header *multipart.FileHeader, req SkillInstallRequest) (*SkillItem, error) {
|
|
sourceType = strings.ToLower(strings.TrimSpace(sourceType))
|
|
if sourceType == "" && header != nil {
|
|
switch strings.ToLower(path.Ext(header.Filename)) {
|
|
case ".md", ".markdown":
|
|
sourceType = "markdown"
|
|
case ".zip":
|
|
sourceType = "zip"
|
|
}
|
|
}
|
|
if sourceType != "markdown" && sourceType != "zip" {
|
|
return nil, kernel.BadAuthRequest("技能文件仅支持 Markdown 或 ZIP")
|
|
}
|
|
if header == nil || header.Size <= 0 || header.Size > maxSkillPackageBytes {
|
|
return nil, kernel.BadAuthRequest("技能文件大小必须在 1B-20MB 之间")
|
|
}
|
|
file, err := header.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
data, err := io.ReadAll(io.LimitReader(file, maxSkillPackageBytes+1))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(data) > maxSkillPackageBytes {
|
|
return nil, kernel.BadAuthRequest("技能文件不能超过 20MB")
|
|
}
|
|
var archive skillPackageArchive
|
|
if sourceType == "markdown" {
|
|
archive, err = archiveFromMarkdown(data, req.Name, req.Description)
|
|
} else {
|
|
archive, err = archiveFromZip(data, "")
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s.createSkillFromArchive(userID, archive, req, sourceType, "", "", "", "", false)
|
|
}
|
|
|
|
func (s *Service) InstallGitHubSkill(userID string, req SkillGitHubInstallRequest) (*SkillItem, error) {
|
|
spec, err := parseGitHubSkillURL(req.URL, req.Ref, req.Subdir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
archive, commit, canonicalURL, resolvedRef, err := fetchGitHubSkillArchive(context.Background(), spec)
|
|
if err != nil {
|
|
var appErr *kernel.AppError
|
|
if errors.As(err, &appErr) {
|
|
return nil, err
|
|
}
|
|
return nil, kernel.WrapAppError(http.StatusBadGateway, "GitHub 技能读取失败，请检查仓库地址和网络", err)
|
|
}
|
|
return s.createSkillFromArchive(userID, archive, SkillInstallRequest{Tag: req.Tag, IsPrivate: req.IsPrivate}, "github", canonicalURL, resolvedRef, spec.Subdir, commit, req.AutoUpdate)
|
|
}
|
|
|
|
func (s *Service) SyncGitHubSkill(userID string, skillID string) (*SkillItem, error) {
|
|
skill, err := s.ownedSkill(userID, skillID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if skill.SourceType != "github" {
|
|
return nil, kernel.BadAuthRequest("只有 GitHub 技能可以同步")
|
|
}
|
|
if err := s.syncGitHubSkill(skill); err != nil {
|
|
var appErr *kernel.AppError
|
|
if errors.As(err, &appErr) {
|
|
return nil, err
|
|
}
|
|
return nil, kernel.WrapAppError(http.StatusBadGateway, "GitHub 技能同步失败，请稍后重试", err)
|
|
}
|
|
return s.SkillDetail(userID, skill.ID)
|
|
}
|
|
|
|
func (s *Service) StartSyncWorker(ctx context.Context) {
|
|
s.runWorkerLoop(func(ctx context.Context) {
|
|
ticker := time.NewTicker(6 * time.Hour)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
skills, err := s.repo.AutoUpdatingGitHubSkills()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for index := range skills {
|
|
_ = s.syncGitHubSkillWithContext(ctx, &skills[index])
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func (s *Service) syncGitHubSkill(skill *model.Skill) error {
|
|
return s.syncGitHubSkillWithContext(context.Background(), skill)
|
|
}
|
|
|
|
func (s *Service) syncGitHubSkillWithContext(ctx context.Context, skill *model.Skill) error {
|
|
now := time.Now()
|
|
spec, err := parseGitHubSkillURL(skill.SourceURL, skill.SourceRef, skill.SourceSubdir)
|
|
if err != nil {
|
|
skill.SyncStatus = "failed"
|
|
skill.SyncError = err.Error()
|
|
skill.LastCheckedAt = &now
|
|
_ = s.repo.SaveSkill(skill)
|
|
return err
|
|
}
|
|
archive, commit, canonicalURL, resolvedRef, err := fetchGitHubSkillArchive(ctx, spec)
|
|
skill.LastCheckedAt = &now
|
|
if err != nil {
|
|
skill.SyncStatus = "failed"
|
|
skill.SyncError = err.Error()
|
|
_ = s.repo.SaveSkill(skill)
|
|
return err
|
|
}
|
|
if commit == skill.SourceCommit {
|
|
skill.SyncStatus = "synced"
|
|
skill.SyncError = ""
|
|
return s.repo.SaveSkill(skill)
|
|
}
|
|
skill.Name = archive.Metadata.Name
|
|
skill.Description = archive.Metadata.Description
|
|
skill.Instruction = string(archive.Files["SKILL.md"])
|
|
if err := s.addSkillArchiveVersion(skill, archive, "github", canonicalURL, resolvedRef, spec.Subdir, commit, skill.AutoUpdate); err != nil {
|
|
skill.SyncStatus = "failed"
|
|
skill.SyncError = err.Error()
|
|
_ = s.repo.SaveSkill(skill)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) createSingleMarkdownSkill(userID string, req SkillMutationRequest) (*SkillItem, error) {
|
|
archive, err := archiveFromMarkdown([]byte(req.Instruction), req.SkillName, req.Description)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s.createSkillFromArchive(userID, archive, SkillInstallRequest{Name: req.SkillName, Description: req.Description, Tag: req.Tag, IsPrivate: req.IsPrivate}, "markdown", req.MarkdownURL, "", "", "", false)
|
|
}
|
|
|
|
func (s *Service) updateSingleMarkdownSkill(skill *model.Skill, req SkillMutationRequest) error {
|
|
if strings.TrimSpace(req.Instruction) == "" {
|
|
return s.repo.SaveSkill(skill)
|
|
}
|
|
archive, err := archiveFromMarkdown([]byte(req.Instruction), req.SkillName, req.Description)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
skill.Instruction = req.Instruction
|
|
return s.addSkillArchiveVersion(skill, archive, "markdown", req.MarkdownURL, "", "", "", false)
|
|
}
|
|
|
|
func (s *Service) createSkillFromArchive(userID string, archive skillPackageArchive, req SkillInstallRequest, sourceType string, sourceURL string, sourceRef string, sourceSubdir string, sourceCommit string, autoUpdate bool) (*SkillItem, error) {
|
|
name := strings.TrimSpace(req.Name)
|
|
if name == "" {
|
|
name = archive.Metadata.Name
|
|
}
|
|
description := strings.TrimSpace(req.Description)
|
|
if description == "" {
|
|
description = archive.Metadata.Description
|
|
}
|
|
if utf8.RuneCountInString(name) == 0 || utf8.RuneCountInString(name) > 80 {
|
|
return nil, kernel.BadAuthRequest("技能名称必须为 1-80 个字符")
|
|
}
|
|
if utf8.RuneCountInString(description) == 0 || utf8.RuneCountInString(description) > 500 {
|
|
return nil, kernel.BadAuthRequest("技能简介必须为 1-500 个字符")
|
|
}
|
|
tag := strings.TrimSpace(req.Tag)
|
|
if _, ok := skillCategoryLabels[tag]; !ok {
|
|
tag = "others"
|
|
}
|
|
skillID := kernel.NewID()
|
|
versionID := kernel.NewID()
|
|
packageKey, version, files, err := s.persistSkillArchive(skillID, versionID, archive, sourceCommit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
now := time.Now()
|
|
versionLabel := archive.Metadata.Version
|
|
if versionLabel == "" {
|
|
versionLabel = "1"
|
|
if sourceCommit != "" {
|
|
versionLabel = shortCommit(sourceCommit)
|
|
}
|
|
}
|
|
version.VersionLabel = versionLabel
|
|
skill := &model.Skill{
|
|
ID: skillID, OwnerID: userID, Name: name, Description: description, Instruction: string(archive.Files["SKILL.md"]),
|
|
CurrentVersionID: versionID, VersionLabel: versionLabel, ContentHash: archive.ContentHash, FileCount: len(archive.Files), TotalBytes: archive.TotalBytes,
|
|
SourceType: sourceType, SourceURL: sourceURL, SourceRef: sourceRef, SourceSubdir: sourceSubdir, SourceCommit: sourceCommit,
|
|
SyncStatus: "synced", AutoUpdate: autoUpdate, LastCheckedAt: &now, LastSyncedAt: &now,
|
|
Status: skillStatusEnabled, Source: skillSourceUser, Tag: tag, IsPrivate: req.IsPrivate, MarkdownURL: sourceURL, ShowcaseMediaJSON: "[]",
|
|
}
|
|
state := &model.UserSkillState{ID: kernel.NewID(), UserID: userID, SkillID: skillID, Added: true, InstalledVersionID: versionID, AutoUpdate: autoUpdate}
|
|
if err := s.repo.CreateSkillWithPackage(skill, version, files, state); err != nil {
|
|
_ = os.Remove(filepath.Join(s.dataDir, "skill-packages", filepath.FromSlash(packageKey)))
|
|
return nil, err
|
|
}
|
|
return s.SkillDetail(userID, skillID)
|
|
}
|
|
|
|
func (s *Service) addSkillArchiveVersion(skill *model.Skill, archive skillPackageArchive, sourceType string, sourceURL string, sourceRef string, sourceSubdir string, sourceCommit string, autoUpdate bool) error {
|
|
versionID := kernel.NewID()
|
|
packageKey, version, files, err := s.persistSkillArchive(skill.ID, versionID, archive, sourceCommit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
versionLabel := archive.Metadata.Version
|
|
if versionLabel == "" {
|
|
versionLabel = "1"
|
|
if sourceCommit != "" {
|
|
versionLabel = shortCommit(sourceCommit)
|
|
} else if skill.CurrentVersionID != "" {
|
|
versionLabel = time.Now().UTC().Format("20060102-150405")
|
|
}
|
|
}
|
|
version.VersionLabel = versionLabel
|
|
now := time.Now()
|
|
skill.CurrentVersionID = versionID
|
|
skill.VersionLabel = versionLabel
|
|
skill.ContentHash = archive.ContentHash
|
|
skill.FileCount = len(archive.Files)
|
|
skill.TotalBytes = archive.TotalBytes
|
|
skill.SourceType = sourceType
|
|
skill.SourceURL = sourceURL
|
|
skill.SourceRef = sourceRef
|
|
skill.SourceSubdir = sourceSubdir
|
|
skill.SourceCommit = sourceCommit
|
|
skill.SyncStatus = "synced"
|
|
skill.SyncError = ""
|
|
skill.AutoUpdate = autoUpdate
|
|
skill.LastCheckedAt = &now
|
|
skill.LastSyncedAt = &now
|
|
if err := s.repo.AddSkillVersion(skill, version, files); err != nil {
|
|
_ = os.Remove(filepath.Join(s.dataDir, "skill-packages", filepath.FromSlash(packageKey)))
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) persistSkillArchive(skillID string, versionID string, archive skillPackageArchive, sourceCommit string) (string, *model.SkillVersion, []model.SkillFile, error) {
|
|
packageKey := path.Join(skillID, versionID+".zip")
|
|
absolute := filepath.Join(s.dataDir, "skill-packages", filepath.FromSlash(packageKey))
|
|
if err := os.MkdirAll(filepath.Dir(absolute), 0o750); err != nil {
|
|
return "", nil, nil, err
|
|
}
|
|
data, err := encodeSkillArchive(archive.Files)
|
|
if err != nil {
|
|
return "", nil, nil, err
|
|
}
|
|
temporary, err := os.CreateTemp(filepath.Dir(absolute), ".skill-*.zip")
|
|
if err != nil {
|
|
return "", nil, nil, err
|
|
}
|
|
temporaryPath := temporary.Name()
|
|
defer os.Remove(temporaryPath)
|
|
if err := temporary.Chmod(0o600); err != nil {
|
|
temporary.Close()
|
|
return "", nil, nil, err
|
|
}
|
|
if _, err := temporary.Write(data); err != nil {
|
|
temporary.Close()
|
|
return "", nil, nil, err
|
|
}
|
|
if err := temporary.Close(); err != nil {
|
|
return "", nil, nil, err
|
|
}
|
|
if err := os.Rename(temporaryPath, absolute); err != nil {
|
|
return "", nil, nil, err
|
|
}
|
|
version := &model.SkillVersion{ID: versionID, SkillID: skillID, ContentHash: archive.ContentHash, EntryPath: "SKILL.md", PackageKey: packageKey, FileCount: len(archive.Files), TotalBytes: archive.TotalBytes, SourceCommit: sourceCommit}
|
|
paths := sortedSkillPaths(archive.Files)
|
|
files := make([]model.SkillFile, 0, len(paths))
|
|
for _, filePath := range paths {
|
|
content := archive.Files[filePath]
|
|
digest := sha256.Sum256(content)
|
|
files = append(files, model.SkillFile{ID: kernel.NewID(), SkillVersionID: versionID, Path: filePath, Kind: skillFileKind(filePath), MimeType: skillFileMime(filePath, content), Size: int64(len(content)), SHA256: hex.EncodeToString(digest[:])})
|
|
}
|
|
return packageKey, version, files, nil
|
|
}
|
|
|
|
func normalizeSkillPath(value string) (string, error) {
|
|
value = strings.ReplaceAll(strings.TrimSpace(value), "\\", "/")
|
|
if value == "" || strings.ContainsRune(value, 0) || strings.HasPrefix(value, "/") {
|
|
return "", kernel.BadAuthRequest("技能文件路径无效")
|
|
}
|
|
clean := path.Clean(value)
|
|
if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || strings.Contains(clean, "/../") || strings.HasPrefix(clean, ".git/") || clean == ".git" {
|
|
return "", kernel.BadAuthRequest("技能文件路径越界或包含禁止目录")
|
|
}
|
|
if utf8.RuneCountInString(clean) > 1000 {
|
|
return "", kernel.BadAuthRequest("技能文件路径过长")
|
|
}
|
|
return clean, nil
|
|
}
|
|
|
|
func skillFileMime(filePath string, content []byte) string {
|
|
if filePath == "SKILL.md" || strings.HasSuffix(strings.ToLower(filePath), ".md") {
|
|
return "text/markdown; charset=utf-8"
|
|
}
|
|
if detected := mime.TypeByExtension(strings.ToLower(path.Ext(filePath))); detected != "" {
|
|
return detected
|
|
}
|
|
return http.DetectContentType(content)
|
|
}
|
|
|
|
func skillFileKind(filePath string) string {
|
|
extension := strings.ToLower(path.Ext(filePath))
|
|
switch extension {
|
|
case ".md", ".mdx":
|
|
return "markdown"
|
|
case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg":
|
|
return "image"
|
|
case ".mp4", ".webm", ".mov":
|
|
return "video"
|
|
case ".mp3", ".wav", ".m4a", ".ogg":
|
|
return "audio"
|
|
case ".py", ".js", ".ts", ".tsx", ".jsx", ".go", ".sh", ".json", ".yaml", ".yml", ".toml", ".html", ".css":
|
|
return "code"
|
|
case ".txt", ".csv":
|
|
return "text"
|
|
default:
|
|
return "binary"
|
|
}
|
|
}
|
|
|
|
func isPreviewText(mimeType string, filePath string) bool {
|
|
return strings.HasPrefix(mimeType, "text/") || strings.Contains(mimeType, "json") || strings.Contains(mimeType, "yaml") || strings.Contains(mimeType, "toml") || skillFileKind(filePath) == "code"
|
|
}
|
|
|
|
func skillFileItem(file model.SkillFile) SkillPackageFileItem {
|
|
return SkillPackageFileItem{Path: file.Path, Kind: file.Kind, MimeType: file.MimeType, Size: file.Size, SHA256: file.SHA256}
|
|
}
|
|
|
|
func sortedSkillPaths(files map[string][]byte) []string {
|
|
paths := make([]string, 0, len(files))
|
|
for filePath := range files {
|
|
paths = append(paths, filePath)
|
|
}
|
|
sort.Strings(paths)
|
|
return paths
|
|
}
|
|
|
|
func shortCommit(value string) string {
|
|
if len(value) <= 12 {
|
|
return value
|
|
}
|
|
return value[:12]
|
|
}
|
|
|
|
func isSkillPackageNotFound(err error) bool {
|
|
return errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, os.ErrNotExist)
|
|
}
|