fix(agent): skill_search tokenizer keeps single CJK runes

cloudAgentSkillSearchTokens dropped any token shorter than 2 runes.
That filter was copied from English stop-word logic (a / I), but a single
Han rune such as the ones meaning hook, gag or twist is already a complete
semantic unit in Chinese. When the whole keyword consisted of such runes,
tokens came back empty and cloudAgentSearchSkills degraded to listing the
full index of enabled skills - the user thinks a search happened but gets
an unsorted dump.

Fix: keep a token when it is exactly one Han rune (unicode.Han); single
ASCII/digit runes and empty strings are still dropped as before.

Test: TestCloudAgentSkillSearchTokensKeepsSingleCJKRune fails on the old
logic (got [反转 钩子], want [反转 钩子 梗 钩]) and passes after the fix.
recall_lessons has an isomorphic copy of the same tokenizer, fixed
separately in #593.
This commit is contained in:
Wyatt_K
2026-09-22 16:30:07 +08:00
parent e81b5bbbe3
commit 77830b0dfc
2 changed files with 33 additions and 1 deletions

View File

@@ -262,3 +262,30 @@ func TestCloudAgentSkillSearchNeverInlinesSkillBody(t *testing.T) {
t.Fatal("skill_search leaked instruction body")
}
}
func TestCloudAgentSkillSearchTokensKeepsSingleCJKRune(t *testing.T) {
// 单字停用词过滤照拿英文逻辑(a / I);汉字单字「梗」「钩」「戏」本身就是完整语义的
// 最小单位。被 < 2 一刀切后 tokens 为空,cloudAgentSearchSkills 会把检索退化成
// 「列全部已启用技能索引」——用户以为搜过了,拿到的却是未排序的全量清单。
got := cloudAgentSkillSearchTokens("反转 钩子 梗 钩")
want := []string{"反转", "钩子", "梗", "钩"}
if len(got) != len(want) {
t.Fatalf("切词结果 %v,期望 %v", got, want)
}
for index := range want {
if got[index] != want[index] {
t.Fatalf("切词结果 %v,期望 %v", got, want)
}
}
// 只放行汉字单字:英文/数字单字与空串仍按停用词处理。
for _, keyword := range []string{"a", "I", "7", "", ",、。;"} {
if tokens := cloudAgentSkillSearchTokens(keyword); len(tokens) != 0 {
t.Fatalf("%q 应无候选词,得到 %v", keyword, tokens)
}
}
// 端到端:单个汉字 token 必须能对描述命中打分,否则放行也没有意义。
skill := cloudAgentSkill{ID: "x1", Name: "comedy-beat-lab", Description: "专门讲「梗」的密度、预期违背与排布节奏"}
if score, _ := cloudAgentSkillMatch(skill, []string{"梗"}); score == 0 {
t.Fatal("单个汉字 token 应该能命中描述")
}
}

View File

@@ -68,7 +68,12 @@ func cloudAgentSkillSearchTokens(keyword string) []string {
seen := make(map[string]bool, cloudAgentSkillSearchTokenMax)
for _, raw := range strings.FieldsFunc(keyword, cloudAgentSkillTokenSeparator) {
token := strings.ToLower(strings.TrimSpace(raw))
if utf8.RuneCountInString(token) < 2 || seen[token] {
// 单字停用词过滤照拿的是英文逻辑(a / I);汉字单字「梗」「钩」「戏」本身是完整
// 语义的最小单位。一刀切丢后 tokens 为空,检索会退化成「列全部已启用技能索引」。
// 这里只放行单个汉字的 token,英文/数字单字与空串仍按停用词丢掉。
first, size := utf8.DecodeRuneInString(token)
singleHan := size == len(token) && unicode.Is(unicode.Han, first)
if (!singleHan && utf8.RuneCountInString(token) < 2) || seen[token] {
continue
}
seen[token] = true