fix(agent): skill_search tokenizer keeps single CJK runes
cloudAgentSkillSearchTokens dropped any token shorter than 2 runes. That filter was copied from English stop-word logic (a / I), but a single Han rune such as the ones meaning hook, gag or twist is already a complete semantic unit in Chinese. When the whole keyword consisted of such runes, tokens came back empty and cloudAgentSearchSkills degraded to listing the full index of enabled skills - the user thinks a search happened but gets an unsorted dump. Fix: keep a token when it is exactly one Han rune (unicode.Han); single ASCII/digit runes and empty strings are still dropped as before. Test: TestCloudAgentSkillSearchTokensKeepsSingleCJKRune fails on the old logic (got [反转 钩子], want [反转 钩子 梗 钩]) and passes after the fix. recall_lessons has an isomorphic copy of the same tokenizer, fixed separately in #593.
This commit is contained in:
1 parent
e81b5bbbe3
commit
77830b0dfc
2 files changed
+33
-1
No files matched your search
@@ -68,7 +68,12 @@ func cloudAgentSkillSearchTokens(keyword string) []string {
|
||||
seen := make(map[string]bool, cloudAgentSkillSearchTokenMax)
|
||||
for _, raw := range strings.FieldsFunc(keyword, cloudAgentSkillTokenSeparator) {
|
||||
token := strings.ToLower(strings.TrimSpace(raw))
|
||||
if utf8.RuneCountInString(token) < 2 || seen[token] {
|
||||
// 单字停用词过滤照拿的是英文逻辑(a / I);汉字单字「梗」「钩」「戏」本身是完整
|
||||
// 语义的最小单位。一刀切丢后 tokens 为空,检索会退化成「列全部已启用技能索引」。
|
||||
// 这里只放行单个汉字的 token,英文/数字单字与空串仍按停用词丢掉。
|
||||
first, size := utf8.DecodeRuneInString(token)
|
||||
singleHan := size == len(token) && unicode.Is(unicode.Han, first)
|
||||
if (!singleHan && utf8.RuneCountInString(token) < 2) || seen[token] {
|
||||
continue
|
||||
}
|
||||
seen[token] = true
|
||||
|
||||
Reference in new issue
Block a user