fix(agent): recall_lessons tokenizer keeps single CJK runes
cloudAgentLessonSearchTokens dropped any token shorter than 2 runes. That filter was copied from English stop-word logic (a / I), but a single Han rune such as the ones meaning hook, gag or twist is already a complete semantic unit in Chinese. When the whole keyword consisted of such runes, tokens came back empty and cloudAgentSearchLessons silently fell back to "list the first N lessons" - the user thinks a search happened but gets an arbitrary list. Fix: keep a token when it is exactly one Han rune (unicode.Han); single ASCII/digit runes and empty strings are still dropped as before. Test: TestLessonSearchTokensKeepsSingleCJKRune fails on the old logic (got [反转 钩子], want [反转 钩子 梗 钩]) and passes after the fix. The isomorphic copy in skill_search (cloud_agent_tools.go) is fixed separately on the #580 branch and deliberately not included here. Gates: go build ./... and go vet EXIT=0; gofmt clean on LF content; the 3 go-sqlite3 cgo-stub failures in internal/app are pre-existing and unrelated.
This commit is contained in:
@@ -545,7 +545,12 @@ func cloudAgentLessonSearchTokens(keyword string) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.FieldsFunc(keyword, cloudAgentLessonTokenSeparator) {
|
||||
token := strings.ToLower(strings.TrimSpace(raw))
|
||||
if utf8.RuneCountInString(token) < 2 || seen[token] {
|
||||
// 单字停用词过滤照搬的是英文逻辑(a / I);汉字单字「梗」「钩」「戏」本身是完整
|
||||
// 语义的最小单位。一刀切丢掉后 tokens 为空,检索会静默回落为「列前 N 条」。
|
||||
// 这里只放行单个汉字的 token,英文/数字单字与空串仍按停用词丢掉。
|
||||
first, size := utf8.DecodeRuneInString(token)
|
||||
singleHan := size == len(token) && unicode.Is(unicode.Han, first)
|
||||
if (!singleHan && utf8.RuneCountInString(token) < 2) || seen[token] {
|
||||
continue
|
||||
}
|
||||
seen[token] = true
|
||||
|
||||
@@ -414,6 +414,28 @@ func TestLessonSearchTokensAreCapped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLessonSearchTokensKeepsSingleCJKRune(t *testing.T) {
|
||||
// 单字停用词过滤照搬的是英文逻辑(a / I);中文单字「梗」「钩」「戏」本身就是完整语义的最小单位,
|
||||
// 被 < 2 一刀切后 tokens 为空,cloudAgentSearchLessons 会静默回落为「列前 N 条」——
|
||||
// 用户以为搜过了,拿到的却是任意清单。
|
||||
got := cloudAgentLessonSearchTokens("反转 钩子 梗 钩")
|
||||
want := []string{"反转", "钩子", "梗", "钩"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("切词结果 %v,期望 %v", got, want)
|
||||
}
|
||||
for index := range want {
|
||||
if got[index] != want[index] {
|
||||
t.Fatalf("切词结果 %v,期望 %v", got, want)
|
||||
}
|
||||
}
|
||||
// 只放行汉字单字:英文/数字单字与空串仍按停用词处理。
|
||||
for _, keyword := range []string{"a", "I", "7", "", ",、。;"} {
|
||||
if tokens := cloudAgentLessonSearchTokens(keyword); len(tokens) != 0 {
|
||||
t.Fatalf("%q 应无候选词,得到 %v", keyword, tokens)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLessonMatchScoreRanksByHit(t *testing.T) {
|
||||
tokens := cloudAgentLessonSearchTokens("视频 合并 拼接")
|
||||
strong := model.AgentLesson{Topic: "video.merge-concat-not-a-feature", Situation: "用户要把多条镜头视频合并成一条成片"}
|
||||
|
||||
Reference in New Issue
Block a user