From c59f62ba84c6d9ce50051bb8ce756493f7b9f145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Mon, 20 Apr 2026 10:59:12 -0400 Subject: [PATCH] =?UTF-8?q?=E6=94=B6=E7=B4=A7=20token=20=E4=BC=B0=E7=AE=97?= =?UTF-8?q?=E5=88=B0=202=20chars/token=20(C3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原先 4 chars/token 对中文 BPE 严重低估（汉字实际约 1.5-2 tokens/char）， 120k 阈值只有在真实上下文已经爆窗之后才触发。2 chars/token 对拉丁文保守 一些，换来中文 agent 不再漏触发压缩。pi-ai 0.67.68 没暴露真 tokenizer， 换 tiktoken/anthropic-tokenizer 的跟进方案见 docs/proposals/C3-real-tokenizer-followup.md。 --- src/main/agent-runtime/context-compactor.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/agent-runtime/context-compactor.ts b/src/main/agent-runtime/context-compactor.ts index 0b89bd9e..d754e6b4 100644 --- a/src/main/agent-runtime/context-compactor.ts +++ b/src/main/agent-runtime/context-compactor.ts @@ -28,8 +28,17 @@ const COMPACTION_TRIGGER_TOKENS = 120_000; * losing mid-stream state (tool-call pairs, user intent). */ const RECENT_MESSAGE_TAIL = 20; -/** Rough tokens-per-char estimator; good enough for gating. */ -const CHARS_PER_TOKEN = 4; +/** + * Conservative tokens-per-char estimator. The former 4 chars/token value + * under-counts CJK traffic by roughly 2x — Chinese BPE tokenizers land + * closer to 1.5-2 tokens per character, so a 120k-token gate would + * silently trip only after the real context already blew past the + * window. 2 chars/token is a deliberate over-estimate for latin text in + * exchange for not under-triggering on Chinese agent runs. The proper + * fix is a real tokenizer — pi-ai 0.67.68 does not export one yet, so + * see docs/proposals/C3-real-tokenizer-followup.md for the upgrade path. + */ +const CHARS_PER_TOKEN = 2; /** Per-assistant-message summary cap to keep the compacted turn readable. */ const MAX_SUMMARY_CHARS_PER_MESSAGE = 160;