fix(sampler): a max_tokens stop after complete tool calls runs them
The Messages stream failed every max_tokens stop with MaxTokensTruncation, so a response that had already emitted complete tool calls before the output budget ran out lost those calls and failed the turn. The stream now completes with StopReason::ToolCalls when every collected call's arguments parse (or are empty, the zero-argument convention); a cut call or a text-only truncation still fails so compact-on-error keeps working. The ChatCompletions and Responses streams already behaved this way. Ported from upstream 1.0.1x LengthPolicy::CompleteToolCalls, reshaped into the stream layer because Kigi has no caller that chooses a policy. Claude-Session: https://claude.ai/code/session_01RUqV8HVGXcAETyYbZ2tPQ4
This commit is contained in:
1 parent
63d26d4a3e
commit
017a570d7d
2 files changed
+108
-5
No files matched your search
@@ -55,6 +55,15 @@ enum BlockType {
|
||||
Thinking,
|
||||
}
|
||||
|
||||
/// Executable after a `max_tokens` stop: every call's arguments parse or are empty (zero-arg convention).
|
||||
fn tool_calls_complete(tool_calls: &[ToolCall]) -> bool {
|
||||
!tool_calls.is_empty()
|
||||
&& tool_calls.iter().all(|tc| {
|
||||
tc.arguments.trim().is_empty()
|
||||
|| serde_json::from_str::<serde::de::IgnoredAny>(&tc.arguments).is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
/// Transform a raw Anthropic Messages API stream into a stream of
|
||||
/// [`SamplingEvent`]s.
|
||||
///
|
||||
@@ -454,11 +463,18 @@ pub fn stream_messages<'a>(
|
||||
}
|
||||
|
||||
if final_stop_reason == Some(StopReason::Length) {
|
||||
yield SamplingEvent::Failed {
|
||||
request_id: request_id.clone(),
|
||||
error: SamplingErrorInfo::from(&SamplingError::MaxTokensTruncation),
|
||||
};
|
||||
return;
|
||||
if !tool_calls_complete(&assistant_tool_calls) {
|
||||
yield SamplingEvent::Failed {
|
||||
request_id: request_id.clone(),
|
||||
error: SamplingErrorInfo::from(&SamplingError::MaxTokensTruncation),
|
||||
};
|
||||
return;
|
||||
}
|
||||
tracing::info!(
|
||||
request_id = %request_id,
|
||||
tool_calls = assistant_tool_calls.len(),
|
||||
"max_tokens stop after complete tool calls; completing the turn so they run"
|
||||
);
|
||||
}
|
||||
|
||||
// Build the final response
|
||||
|
||||
Reference in new issue
Block a user