mirror of
https://github.com/tiennm99/ccs.git
synced 2026-10-11 03:13:12 +00:00
fix(transformers): preserve tool ordering across proxy streams
- reject pending tool_result layouts that cannot be translated without reordering user content - keep interleaved GLMT tool_use blocks open until finalization instead of stopping early - cover leading/interleaved tool_result regressions and interleaved streaming tool fragments
This commit is contained in:
1 parent
889d3d89e9
commit
399f403322
5 files changed
+202
-34
No files matched your search
@@ -284,7 +284,10 @@ export class DeltaAccumulator {
|
|||||||
|
|
||||||
getToolCallBlockIndex(toolCallIndex: number): number {
|
getToolCallBlockIndex(toolCallIndex: number): number {
|
||||||
const toolCall = this.toolCallsIndex[toolCallIndex];
|
const toolCall = this.toolCallsIndex[toolCallIndex];
|
||||||
return toolCall?.blockIndex ?? 0;
|
if (!toolCall || toolCall.blockIndex < 0) {
|
||||||
|
throw new Error(`Tool call ${toolCallIndex} does not have an assigned content block`);
|
||||||
|
}
|
||||||
|
return toolCall.blockIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
getUnstoppedBlocks(): ContentBlock[] {
|
getUnstoppedBlocks(): ContentBlock[] {
|
||||||
|
|||||||
@@ -5,20 +5,12 @@
|
|||||||
* - Process tool call deltas from OpenAI
|
* - Process tool call deltas from OpenAI
|
||||||
* - Generate tool_use content blocks for Anthropic format
|
* - Generate tool_use content blocks for Anthropic format
|
||||||
* - Handle input_json_delta events
|
* - Handle input_json_delta events
|
||||||
* - Close previous tool_use blocks before starting new ones (Anthropic sequential block requirement)
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { DeltaAccumulator } from '../delta-accumulator';
|
import type { DeltaAccumulator } from '../delta-accumulator';
|
||||||
import type { OpenAIToolCallDelta, OpenAIToolCall, ContentBlock, AnthropicSSEEvent } from './types';
|
import type { OpenAIToolCallDelta, OpenAIToolCall, ContentBlock, AnthropicSSEEvent } from './types';
|
||||||
import { ResponseBuilder } from './response-builder';
|
|
||||||
|
|
||||||
export class ToolCallHandler {
|
export class ToolCallHandler {
|
||||||
private responseBuilder: ResponseBuilder;
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
this.responseBuilder = new ResponseBuilder(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
processToolCalls(toolCalls: OpenAIToolCall[]): ContentBlock[] {
|
processToolCalls(toolCalls: OpenAIToolCall[]): ContentBlock[] {
|
||||||
const content: ContentBlock[] = [];
|
const content: ContentBlock[] = [];
|
||||||
|
|
||||||
@@ -54,12 +46,8 @@ export class ToolCallHandler {
|
|||||||
accumulator.addToolCallDelta(toolCallDelta);
|
accumulator.addToolCallDelta(toolCallDelta);
|
||||||
|
|
||||||
if (isNewToolCall) {
|
if (isNewToolCall) {
|
||||||
const previousBlock = accumulator.getCurrentBlock();
|
// OpenAI may interleave tool_call fragments across chunks, so blocks must stay open
|
||||||
if (previousBlock && previousBlock.type === 'tool_use' && !previousBlock.stopped) {
|
// until the stream finalizes. Closing on a later index truncates earlier tool input.
|
||||||
events.push(this.responseBuilder.createContentBlockStopEvent(previousBlock));
|
|
||||||
previousBlock.stopped = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const block = accumulator.startBlock('tool_use');
|
const block = accumulator.startBlock('tool_use');
|
||||||
const toolCall = accumulator.getToolCall(toolCallDelta.index);
|
const toolCall = accumulator.getToolCall(toolCallDelta.index);
|
||||||
accumulator.setToolCallBlockIndex(toolCallDelta.index, block.index);
|
accumulator.setToolCallBlockIndex(toolCallDelta.index, block.index);
|
||||||
|
|||||||
@@ -457,9 +457,35 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (role === 'user') {
|
if (role === 'user') {
|
||||||
const preToolResultParts: OpenAIContentPart[] = [];
|
const userParts: OpenAIContentPart[] = [];
|
||||||
|
const followUpParts: OpenAIContentPart[] = [];
|
||||||
const resolvedToolUseIds = new Set<string>();
|
const resolvedToolUseIds = new Set<string>();
|
||||||
|
|
||||||
|
const handleUserPart = (
|
||||||
|
part: OpenAIContentPart,
|
||||||
|
blockIndex: number,
|
||||||
|
kind: 'text' | 'image'
|
||||||
|
) => {
|
||||||
|
if (!pendingToolUseIds || pendingToolUseIds.size === 0) {
|
||||||
|
userParts.push(part);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resolvedToolUseIds.size === 0) {
|
||||||
|
throw new Error(
|
||||||
|
`messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed before tool_result blocks for pending tool_use ids`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resolvedToolUseIds.size !== pendingToolUseIds.size) {
|
||||||
|
throw new Error(
|
||||||
|
`messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed between tool_result blocks for pending tool_use ids`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
followUpParts.push(part);
|
||||||
|
};
|
||||||
|
|
||||||
content.forEach((block, blockIndex) => {
|
content.forEach((block, blockIndex) => {
|
||||||
const parsed = assertObject(
|
const parsed = assertObject(
|
||||||
block,
|
block,
|
||||||
@@ -472,25 +498,16 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
|
|||||||
|
|
||||||
if (parsed.type === 'text') {
|
if (parsed.type === 'text') {
|
||||||
const text = typeof parsed.text === 'string' ? parsed.text : '';
|
const text = typeof parsed.text === 'string' ? parsed.text : '';
|
||||||
if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) {
|
handleUserPart({ type: 'text', text }, blockIndex, 'text');
|
||||||
preToolResultParts.push({ type: 'text', text });
|
|
||||||
} else {
|
|
||||||
translatedMessages.push({ role: 'user', content: text });
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isImageBlock(parsed)) {
|
if (isImageBlock(parsed)) {
|
||||||
if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) {
|
handleUserPart(
|
||||||
preToolResultParts.push(
|
toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`),
|
||||||
toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)
|
blockIndex,
|
||||||
);
|
'image'
|
||||||
} else {
|
);
|
||||||
translatedMessages.push({
|
|
||||||
role: 'user',
|
|
||||||
content: [toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -555,8 +572,12 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (preToolResultParts.length > 0) {
|
if (userParts.length > 0) {
|
||||||
flushUserContent(translatedMessages, preToolResultParts);
|
flushUserContent(translatedMessages, userParts);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (followUpParts.length > 0) {
|
||||||
|
flushUserContent(translatedMessages, followUpParts);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,10 +23,34 @@ function startMockUpstream(): Promise<void> {
|
|||||||
body += chunk.toString();
|
body += chunk.toString();
|
||||||
}
|
}
|
||||||
upstreamBody = JSON.parse(body);
|
upstreamBody = JSON.parse(body);
|
||||||
const parsed = upstreamBody as { stream?: boolean };
|
const parsed = upstreamBody as {
|
||||||
|
stream?: boolean;
|
||||||
|
messages?: Array<{ role?: string; content?: string | Array<{ type?: string; text?: string }> }>;
|
||||||
|
};
|
||||||
|
|
||||||
if (parsed.stream) {
|
if (parsed.stream) {
|
||||||
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
||||||
|
|
||||||
|
if (parsed.messages?.[0]?.content === 'interleaved tool fragments') {
|
||||||
|
res.write(
|
||||||
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search"}}]}}]}\n\n'
|
||||||
|
);
|
||||||
|
res.write(
|
||||||
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_2","type":"function","function":{"name":"open"}}]}}]}\n\n'
|
||||||
|
);
|
||||||
|
res.write(
|
||||||
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"path\\":\\"a.ts\\"}"}}]}}]}\n\n'
|
||||||
|
);
|
||||||
|
res.write(
|
||||||
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\\"path\\":\\"b.ts\\"}"}}]}}]}\n\n'
|
||||||
|
);
|
||||||
|
res.write(
|
||||||
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":9,"completion_tokens":4}}\n\n'
|
||||||
|
);
|
||||||
|
res.end('data: [DONE]\n\n');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
res.write(
|
res.write(
|
||||||
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n'
|
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n'
|
||||||
);
|
);
|
||||||
@@ -356,6 +380,66 @@ describe('openai proxy messages endpoint', () => {
|
|||||||
expect(userAfterTool?.[1]?.content).toBe('What should I do next?');
|
expect(userAfterTool?.[1]?.content).toBe('What should I do next?');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects text before tool_result blocks when tool results are pending', async () => {
|
||||||
|
const response = await requestProxy({
|
||||||
|
model: 'hf-model',
|
||||||
|
stream: false,
|
||||||
|
messages: [
|
||||||
|
{ role: 'user', content: 'search docs' },
|
||||||
|
{
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{ type: 'tool_use', id: 'toolu_01', name: 'search', input: { q: 'docs' } }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'text', text: 'before' },
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_01', content: 'found 3 docs' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const body = (await response.json()) as { error?: { type?: string; message?: string } };
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(body.error?.type).toBe('invalid_request_error');
|
||||||
|
expect(body.error?.message).toContain(
|
||||||
|
'text is not allowed before tool_result blocks for pending tool_use ids'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects follow-up text between pending tool_result blocks', async () => {
|
||||||
|
const response = await requestProxy({
|
||||||
|
model: 'hf-model',
|
||||||
|
stream: false,
|
||||||
|
messages: [
|
||||||
|
{ role: 'user', content: 'read both files' },
|
||||||
|
{
|
||||||
|
role: 'assistant',
|
||||||
|
content: [
|
||||||
|
{ type: 'tool_use', id: 'toolu_01', name: 'Read', input: { file_path: 'a.ts' } },
|
||||||
|
{ type: 'tool_use', id: 'toolu_02', name: 'Read', input: { file_path: 'b.ts' } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_01', content: 'content of a' },
|
||||||
|
{ type: 'text', text: 'Now compare them' },
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_02', content: 'content of b' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const body = (await response.json()) as { error?: { type?: string; message?: string } };
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
expect(body.error?.type).toBe('invalid_request_error');
|
||||||
|
expect(body.error?.message).toContain(
|
||||||
|
'text is not allowed between tool_result blocks for pending tool_use ids'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('translates parallel tool calls with streaming', async () => {
|
it('translates parallel tool calls with streaming', async () => {
|
||||||
const response = await requestProxy({
|
const response = await requestProxy({
|
||||||
model: 'hf-model',
|
model: 'hf-model',
|
||||||
@@ -409,4 +493,36 @@ describe('openai proxy messages endpoint', () => {
|
|||||||
expect(toolMsgs?.[0]?.tool_call_id).toBe('toolu_01');
|
expect(toolMsgs?.[0]?.tool_call_id).toBe('toolu_01');
|
||||||
expect(toolMsgs?.[1]?.tool_call_id).toBe('toolu_02');
|
expect(toolMsgs?.[1]?.tool_call_id).toBe('toolu_02');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('streams interleaved tool call fragments without premature block stops', async () => {
|
||||||
|
const response = await requestProxy({
|
||||||
|
model: 'hf-model',
|
||||||
|
stream: true,
|
||||||
|
messages: [{ role: 'user', content: [{ type: 'text', text: 'interleaved tool fragments' }] }],
|
||||||
|
tools: [
|
||||||
|
{
|
||||||
|
name: 'search',
|
||||||
|
description: 'Search docs',
|
||||||
|
input_schema: { type: 'object', properties: { path: { type: 'string' } } },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'open',
|
||||||
|
description: 'Open a file',
|
||||||
|
input_schema: { type: 'object', properties: { path: { type: 'string' } } },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const body = await response.text();
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(body.match(/event: content_block_start/g)?.length).toBe(2);
|
||||||
|
expect(body.match(/event: content_block_stop/g)?.length).toBe(2);
|
||||||
|
|
||||||
|
const stopIndex = body.indexOf('event: content_block_stop');
|
||||||
|
const deltaAIndex = body.indexOf('"partial_json":"{\\"path\\":\\"a.ts\\"}"');
|
||||||
|
const deltaBIndex = body.indexOf('"partial_json":"{\\"path\\":\\"b.ts\\"}"');
|
||||||
|
expect(deltaAIndex).toBeGreaterThan(-1);
|
||||||
|
expect(deltaBIndex).toBeGreaterThan(-1);
|
||||||
|
expect(stopIndex).toBeGreaterThan(deltaBIndex);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -120,8 +120,48 @@ describe('ProxyRequestTransformer regressions', () => {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
).toThrow('text is not allowed before tool_result blocks for pending tool_use ids');
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
new ProxyRequestTransformer().transform({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{ type: 'tool_use', id: 'toolu_1', name: 'vision', input: { detail: 'high' } }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'result' },
|
||||||
|
{ type: 'text', text: 'follow-up' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
).not.toThrow();
|
).not.toThrow();
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
new ProxyRequestTransformer().transform({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: 'assistant',
|
||||||
|
content: [
|
||||||
|
{ type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'docs' } },
|
||||||
|
{ type: 'tool_use', id: 'toolu_2', name: 'open', input: { url: 'https://example.com' } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'partial' },
|
||||||
|
{ type: 'text', text: 'follow-up' },
|
||||||
|
{ type: 'tool_result', tool_use_id: 'toolu_2', content: 'done' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
).toThrow('text is not allowed between tool_result blocks for pending tool_use ids');
|
||||||
|
|
||||||
expect(() =>
|
expect(() =>
|
||||||
new ProxyRequestTransformer().transform({
|
new ProxyRequestTransformer().transform({
|
||||||
messages: [
|
messages: [
|
||||||
|
|||||||
Reference in new issue
Block a user