fix(platform): API 调用日志截断对齐 UTF-8 字符边界 - 防止截断报文写库失败导致订单误标 uncertain (#695)

truncateAPICallPayload 原先按字节硬切超长报文,切点多字节字符时会产生
非法 UTF-8,写 PostgreSQL 报 invalid byte sequence,调用日志入库失败并
连锁导致订单被标为 uncertain。

- 截断点回退到最近的 UTF-8 字符边界
- 对源报文本身的非法字节做 strings.ToValidUTF8 兜底
- 补充回归测试:中文/emoji/中日韩混排/非法字节等场景

Co-authored-by: xiaomuclaw <282836183+xiaomuclaw@users.noreply.github.com>
This commit is contained in:
xiaomuclaw
2026-10-06 09:28:17 +08:00
committed by GitHub
parent 25b1aac165
commit fb101b76d0
2 changed files with 72 additions and 1 deletions

View File

@@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"strings"
"unicode/utf8"
)
const maxAPICallPayloadBytes = 128 << 10
@@ -151,5 +152,14 @@ func truncateAPICallPayload(value string) string {
if len(value) <= maxAPICallPayloadBytes {
return value
}
return value[:maxAPICallPayloadBytes] + fmt.Sprintf("\n[报文已截断,原始长度 %d 字节]", len(value))
cut := maxAPICallPayloadBytes
for cut > 0 && !utf8.RuneStart(value[cut]) {
cut--
}
// 上游报文不保证是合法 UTF-8,直接入库会被 PostgreSQL 以 invalid byte sequence 拒收。
truncated := value[:cut]
if !utf8.ValidString(truncated) {
truncated = strings.ToValidUTF8(truncated, string(utf8.RuneError))
}
return truncated + fmt.Sprintf("\n[报文已截断,原始长度 %d 字节]", len(value))
}

View File

@@ -0,0 +1,61 @@
package platform
import (
"fmt"
"strings"
"testing"
"unicode/utf8"
)
// 回归测试:截断必须落在 UTF-8 字符边界上,且产出永远是合法 UTF-8,
// 否则调用日志写 PostgreSQL 会报 invalid byte sequence,订单被误标 uncertain。
func TestTruncateAPICallPayloadAlwaysValidUTF8(t *testing.T) {
cases := []struct {
name string
value string
}{
{"短文本原样返回", "你好,世界"},
{"纯 ASCII 长文", strings.Repeat("a", maxAPICallPayloadBytes+500)},
{"中文长文(3 字节字符)", strings.Repeat("测", (maxAPICallPayloadBytes+500)/3)},
{"emoji 长文(4 字节字符)", strings.Repeat("🎬", (maxAPICallPayloadBytes+500)/4)},
{"中日韩混排", strings.Repeat("测试テスト", (maxAPICallPayloadBytes+500)/18)},
{"源报文本身含非法 UTF-8", strings.Repeat("ok", 70000) + "\xff\xfe" + strings.Repeat("尾", 100)},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
got := truncateAPICallPayload(tc.value)
if !utf8.ValidString(got) {
t.Fatalf("截断结果含非法 UTF-8")
}
if len(tc.value) <= maxAPICallPayloadBytes {
if got != tc.value {
t.Fatalf("短文本应原样返回")
}
return
}
if !strings.HasSuffix(got, fmt.Sprintf("[报文已截断,原始长度 %d 字节]", len(tc.value))) {
t.Fatalf("缺少截断标记: %q", got[len(got)-60:])
}
if len(got) > maxAPICallPayloadBytes+64 {
t.Fatalf("截断后长度越界: got %d", len(got))
}
})
}
}
func TestSanitizeAPICallPayloadTruncatedJSONIsValidUTF8(t *testing.T) {
// 中文 prompt + 大报文,走 json.MarshalIndent 格式化后仍超限的路径。
big := make([]string, 0, 20000)
for i := 0; i < 20000; i++ {
big = append(big, `{"prompt":"用参考图画一段夜晚赛博朋克城市的天际线航拍镜头","index":`+fmt.Sprint(i)+`}`)
}
data := []byte("[" + strings.Join(big, ",") + "]")
got := SanitizeAPICallPayload(data, "application/json")
if !utf8.ValidString(got) {
t.Fatalf("JSON 载体截断结果含非法 UTF-8")
}
if len(data) <= maxAPICallPayloadBytes {
t.Fatalf("用例应超过截断阈值")
}
}