From 074e90055789bd146d96fd3d1ad917f91e602186 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 14 Apr 2026 19:26:59 -0400 Subject: [PATCH 1/8] feat(proxy): add openai-compatible local proxy runtime --- src/ccs.ts | 52 +++ src/commands/command-catalog.ts | 7 + src/commands/completion-backend.ts | 8 + src/commands/help-command.ts | 3 + src/commands/index.ts | 1 + src/commands/proxy-command.ts | 165 ++++++++ src/commands/root-command-router.ts | 7 + src/config/reserved-names.ts | 1 + src/glmt/sse-parser.ts | 102 ++--- src/proxy/index.ts | 7 + src/proxy/profile-router.ts | 70 ++++ src/proxy/proxy-daemon-entry.ts | 80 ++++ src/proxy/proxy-daemon-paths.ts | 17 + src/proxy/proxy-daemon-state.ts | 82 ++++ src/proxy/proxy-daemon.ts | 380 ++++++++++++++++++ src/proxy/proxy-env.ts | 23 ++ src/proxy/server/http-helpers.ts | 81 ++++ src/proxy/server/messages-route.ts | 174 ++++++++ src/proxy/server/proxy-server.ts | 75 ++++ src/proxy/transformers/request-transformer.ts | 89 ++++ .../transformers/sse-stream-transformer.ts | 14 + src/proxy/upstream-url.ts | 36 ++ 22 files changed, 1428 insertions(+), 46 deletions(-) create mode 100644 src/commands/proxy-command.ts create mode 100644 src/proxy/index.ts create mode 100644 src/proxy/profile-router.ts create mode 100644 src/proxy/proxy-daemon-entry.ts create mode 100644 src/proxy/proxy-daemon-paths.ts create mode 100644 src/proxy/proxy-daemon-state.ts create mode 100644 src/proxy/proxy-daemon.ts create mode 100644 src/proxy/proxy-env.ts create mode 100644 src/proxy/server/http-helpers.ts create mode 100644 src/proxy/server/messages-route.ts create mode 100644 src/proxy/server/proxy-server.ts create mode 100644 src/proxy/transformers/request-transformer.ts create mode 100644 src/proxy/transformers/sse-stream-transformer.ts create mode 100644 src/proxy/upstream-url.ts diff --git a/src/ccs.ts b/src/ccs.ts index 5c907085..c1bb2ec3 100644 --- a/src/ccs.ts +++ b/src/ccs.ts @@ -97,6 +97,11 @@ import { } from './targets/droid-reasoning-runtime'; import { DroidCommandRouterError, routeDroidCommandArgs } from './targets/droid-command-router'; import { resolveCliproxyBridgeMetadata } from './api/services/cliproxy-profile-bridge'; +import { + buildOpenAICompatProxyEnv, + resolveOpenAICompatProfileConfig, + startOpenAICompatProxy, +} from './proxy'; // Version and Update check utilities import { getVersion } from './utils/version'; @@ -1304,6 +1309,53 @@ async function main(): Promise { const browserArgs = browserRuntimeEnv ? appendBrowserToolArgs(imageAnalysisArgs) : imageAnalysisArgs; + const openAICompatProfile = resolveOpenAICompatProfileConfig( + profileInfo.name, + expandedSettingsPath, + settingsEnv + ); + if (openAICompatProfile) { + const proxyStart = await startOpenAICompatProxy(openAICompatProfile, { + insecure: openAICompatProfile.insecure, + }); + if (!proxyStart.success) { + console.error(fail(proxyStart.error || 'Failed to start local OpenAI-compatible proxy')); + process.exit(1); + } + + console.error( + info( + `Using local OpenAI-compatible proxy for "${profileInfo.name}" on port ${proxyStart.port}` + ) + ); + + const proxyEnv = { + ...envVars, + ...buildOpenAICompatProxyEnv( + openAICompatProfile, + proxyStart.port, + proxyStart.authToken || '', + inheritedClaudeConfigDir + ), + }; + delete proxyEnv.ANTHROPIC_API_KEY; + + const launchArgs = [ + '--settings', + expandedSettingsPath, + ...appendThirdPartyWebSearchToolArgs(browserArgs), + ]; + const traceEnv = createWebSearchTraceContext({ + launcher: 'ccs.settings-profile.proxy', + args: launchArgs, + profile: profileInfo.name, + profileType: profileInfo.type, + settingsPath: expandedSettingsPath, + }); + + execClaude(claudeCli, launchArgs, { ...proxyEnv, ...traceEnv }); + return; + } const launchArgs = [ '--settings', expandedSettingsPath, diff --git a/src/commands/command-catalog.ts b/src/commands/command-catalog.ts index 3b14c3b7..099923d7 100644 --- a/src/commands/command-catalog.ts +++ b/src/commands/command-catalog.ts @@ -109,6 +109,12 @@ export const ROOT_COMMAND_CATALOG: readonly RootCommandEntry[] = [ group: 'runtime', visibility: 'public', }, + { + name: 'proxy', + summary: 'Start or inspect the OpenAI-compatible local proxy', + group: 'runtime', + visibility: 'public', + }, { name: 'copilot', summary: 'Run or manage the GitHub Copilot bridge', @@ -252,6 +258,7 @@ export const CLIPROXY_SUBCOMMANDS = [ ] as const; export const CONFIG_SUBCOMMANDS = ['auth', 'channels', 'image-analysis', 'thinking'] as const; export const DOCKER_SUBCOMMANDS = ['up', 'down', 'status', 'update', 'logs', 'config'] as const; +export const PROXY_SUBCOMMANDS = ['start', 'stop', 'status', 'activate'] as const; export const TOKENS_FLAGS = [ '--show', '--api-key', diff --git a/src/commands/completion-backend.ts b/src/commands/completion-backend.ts index 523c08ad..9082687f 100644 --- a/src/commands/completion-backend.ts +++ b/src/commands/completion-backend.ts @@ -11,6 +11,7 @@ import { ROOT_COMMAND_FLAGS, ROOT_HELP_TOPICS, TOKENS_FLAGS, + PROXY_SUBCOMMANDS, PROVIDER_FLAGS, uniqueStrings, getPublicRootCommandTokens, @@ -189,6 +190,13 @@ function getSuggestionsForCommand(tokensBeforeCurrent: string[]): CompletionSugg return completeSubcommands([], COMMAND_FLAG_SUGGESTIONS.docker); case 'cursor': return completeSubcommands(CURSOR_COMPLETION_SUBCOMMANDS); + case 'proxy': + if (lastToken === '--shell') + return completeSubcommands(['auto', 'bash', 'zsh', 'fish', 'powershell']); + return completeSubcommands( + [...PROXY_SUBCOMMANDS], + ['--port', '--shell', '--insecure', '--help', '-h'] + ); case 'copilot': return completeSubcommands(COPILOT_COMPLETION_SUBCOMMANDS); case 'env': diff --git a/src/commands/help-command.ts b/src/commands/help-command.ts index e3912a11..d1e4ab9f 100644 --- a/src/commands/help-command.ts +++ b/src/commands/help-command.ts @@ -215,6 +215,7 @@ export async function handleHelpCommand(writeLine: HelpWriter = console.log): Pr name: 'ccs cliproxy --help', summary: 'Deep help for variants, routing, quota, and lifecycle', }, + { name: 'ccs proxy --help', summary: 'Deep help for the OpenAI-compatible local proxy' }, { name: 'ccs docker --help', summary: 'Deep help for Docker deployment commands' }, { name: 'ccs cursor --help', summary: 'Deep help for Cursor runtime/admin commands' }, { name: 'ccs copilot --help', summary: 'Deep help for GitHub Copilot commands' }, @@ -275,6 +276,8 @@ export async function handleHelpRoute( process.exit(await (await import('./copilot-command')).handleCopilotCommand(['--help'])), cursor: async () => process.exit(await (await import('./cursor-command')).handleCursorCommand(['--help'])), + proxy: async () => + process.exit(await (await import('./proxy-command')).handleProxyCommand(['--help'])), docker: async () => (await import('./docker/help-subcommand')).showHelp(), migrate: async () => (await import('./migrate-command')).printMigrateHelp(), setup: async () => (await import('./setup-command')).handleSetupCommand(['--help']), diff --git a/src/commands/index.ts b/src/commands/index.ts index 22d62035..ef9de709 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -13,6 +13,7 @@ export { handleDockerCommand } from './docker-command'; export { handleHelpCommand } from './help-command'; export { handleInstallCommand } from './install-command'; export { handleMigrateCommand } from './migrate-command'; +export { handleProxyCommand } from './proxy-command'; export { handleShellCompletionCommand } from './shell-completion-command'; export { handleSyncCommand } from './sync-command'; export { handleUpdateCommand } from './update-command'; diff --git a/src/commands/proxy-command.ts b/src/commands/proxy-command.ts new file mode 100644 index 00000000..9110f3f1 --- /dev/null +++ b/src/commands/proxy-command.ts @@ -0,0 +1,165 @@ +import { detectShell, formatExportLine } from './env-command'; +import { getSettingsPath, loadSettings } from '../utils/config-manager'; +import { expandPath } from '../utils/helpers'; +import { fail, info, ok } from '../utils/ui'; +import { + buildOpenAICompatProxyEnv, + getOpenAICompatProxyStatus, + resolveOpenAICompatProfileConfig, + startOpenAICompatProxy, + stopOpenAICompatProxy, +} from '../proxy'; + +function parseOptionValue(args: string[], key: string): string | undefined { + const exactIndex = args.findIndex((arg) => arg === key); + if (exactIndex !== -1 && args[exactIndex + 1]) { + return args[exactIndex + 1]; + } + + const prefix = `${key}=`; + const withEquals = args.find((arg) => arg.startsWith(prefix)); + return withEquals ? withEquals.slice(prefix.length) : undefined; +} + +function showHelp(): number { + console.log('OpenAI-Compatible Proxy'); + console.log(''); + console.log('Usage: ccs proxy [profile] [options]'); + console.log(''); + console.log('Commands:'); + console.log( + ' start Start the local proxy for an OpenAI-compatible settings profile' + ); + console.log(' stop Stop the running proxy'); + console.log(' status Show daemon status and active profile'); + console.log(' activate Print shell exports for the running proxy'); + console.log(''); + console.log('Options:'); + console.log(' --port Override the local proxy port (default: 3456)'); + console.log(' --shell activate only: auto|bash|zsh|fish|powershell'); + console.log(' --insecure Disable upstream TLS verification'); + console.log(''); + console.log('Examples:'); + console.log(' ccs proxy start hf'); + console.log(' eval "$(ccs proxy activate)"'); + console.log(' ccs proxy status'); + console.log(' ccs proxy stop'); + console.log(''); + return 0; +} + +function resolveProfile(profileName: string) { + const settingsPath = expandPath(getSettingsPath(profileName)); + const settings = loadSettings(settingsPath); + const profile = resolveOpenAICompatProfileConfig(profileName, settingsPath, settings.env || {}); + if (!profile) { + throw new Error(`Profile "${profileName}" is not configured for an OpenAI-compatible endpoint`); + } + return profile; +} + +async function handleStart(args: string[]): Promise { + const profileName = args.find((arg) => !arg.startsWith('-')); + if (!profileName) { + console.error(fail('Usage: ccs proxy start [--port ] [--insecure]')); + return 1; + } + + const portValue = parseOptionValue(args, '--port'); + const port = portValue ? Number.parseInt(portValue, 10) || 3456 : undefined; + let profile; + try { + profile = resolveProfile(profileName); + } catch (error) { + console.error(fail((error as Error).message)); + return 1; + } + + const result = await startOpenAICompatProxy(profile, { + ...(port ? { port } : {}), + insecure: args.includes('--insecure'), + }); + + if (!result.success) { + console.error(fail(result.error || 'Failed to start proxy')); + return 1; + } + + console.log( + result.alreadyRunning + ? info(`Proxy already running on port ${result.port}`) + : ok(`Proxy started on port ${result.port}`) + ); + return 0; +} + +async function handleStatus(): Promise { + const status = await getOpenAICompatProxyStatus(); + if (!status.running) { + console.log(info('Proxy is not running')); + return 0; + } + + console.log(ok(`Proxy running on port ${status.port}`)); + console.log(` Profile: ${status.profileName}`); + console.log(` Base URL: ${status.baseUrl}`); + if (status.model) { + console.log(` Model: ${status.model}`); + } + if (status.pid) { + console.log(` PID: ${status.pid}`); + } + return 0; +} + +async function handleActivate(args: string[]): Promise { + const status = await getOpenAICompatProxyStatus(); + if (!status.running || !status.profileName || !status.port || !status.authToken) { + console.error(fail('Proxy is not running. Start it with: ccs proxy start ')); + return 1; + } + + const shell = detectShell(parseOptionValue(args, '--shell')); + let profile; + try { + profile = resolveProfile(status.profileName); + } catch (error) { + console.error(fail((error as Error).message)); + return 1; + } + + const env = buildOpenAICompatProxyEnv(profile, status.port, status.authToken); + Object.entries(env).forEach(([key, value]) => { + console.log(formatExportLine(shell, key, value)); + }); + return 0; +} + +export async function handleProxyCommand(args: string[]): Promise { + const subcommand = args[0]; + switch (subcommand) { + case undefined: + case 'help': + case '--help': + case '-h': + return showHelp(); + case 'start': + return handleStart(args.slice(1)); + case 'stop': { + const result = await stopOpenAICompatProxy(); + if (!result.success) { + console.error(fail(result.error || 'Failed to stop proxy')); + return 1; + } + console.log(ok('Proxy stopped')); + return 0; + } + case 'status': + return handleStatus(); + case 'activate': + return handleActivate(args.slice(1)); + default: + console.error(fail(`Unknown proxy subcommand: ${subcommand}`)); + return 1; + } +} diff --git a/src/commands/root-command-router.ts b/src/commands/root-command-router.ts index be45acc7..250c5910 100644 --- a/src/commands/root-command-router.ts +++ b/src/commands/root-command-router.ts @@ -136,6 +136,13 @@ export const ROOT_COMMAND_ROUTES: readonly NamedCommandRoute[] = [ await handleCliproxyCommand(args); }, }, + { + name: 'proxy', + handle: async (args) => { + const { handleProxyCommand } = await import('./proxy-command'); + process.exit(await handleProxyCommand(args)); + }, + }, { name: 'docker', handle: async (args) => { diff --git a/src/config/reserved-names.ts b/src/config/reserved-names.ts index 8705d63a..552621b5 100644 --- a/src/config/reserved-names.ts +++ b/src/config/reserved-names.ts @@ -17,6 +17,7 @@ export const RESERVED_PROFILE_NAMES = [ 'default', 'config', 'cliproxy', + 'proxy', ] as const; export type ReservedProfileName = (typeof RESERVED_PROFILE_NAMES)[number]; diff --git a/src/glmt/sse-parser.ts b/src/glmt/sse-parser.ts index 9a731654..c161a7df 100644 --- a/src/glmt/sse-parser.ts +++ b/src/glmt/sse-parser.ts @@ -49,63 +49,73 @@ export class SSEParser { * @returns Array of parsed events */ parse(chunk: Buffer | string): SSEEvent[] { - this.buffer += chunk.toString(); + this.buffer += chunk.toString().replace(/\r\n/g, '\n'); // C-01 Fix: Prevent unbounded buffer growth (DoS protection) if (this.buffer.length > this.maxBufferSize) { throw new Error(`SSE buffer exceeded ${this.maxBufferSize} bytes (DoS protection)`); } - const lines = this.buffer.split('\n'); - - // Keep incomplete line in buffer - this.buffer = lines.pop() || ''; - const events: SSEEvent[] = []; - let currentEvent: SSEEvent = { event: 'message', data: '' }; + const segments = this.buffer.split('\n\n'); + this.buffer = segments.pop() || ''; - for (const line of lines) { - if (line.startsWith('event: ')) { - currentEvent.event = line.substring(7).trim(); - } else if (line.startsWith('data: ')) { - const data = line.substring(6); + for (const segment of segments) { + const lines = segment.split('\n'); + const currentEvent: SSEEvent = { event: 'message', data: '' }; + const dataLines: string[] = []; - if (data === '[DONE]') { - this.eventCount++; - events.push({ - event: 'done', - data: null, - index: this.eventCount, - }); - currentEvent = { event: 'message', data: '' }; - } else { - try { - currentEvent.data = JSON.parse(data); - this.eventCount++; - currentEvent.index = this.eventCount; - events.push({ ...currentEvent }); - currentEvent = { event: 'message', data: '' }; - } catch (e) { - // H-01 Fix: Log parse errors for debugging - if (typeof console !== 'undefined' && console.error) { - console.error( - '[SSEParser] Malformed JSON event:', - (e as Error).message, - 'Data:', - data.substring(0, 100) - ); - } - if (this.throwOnMalformedJson) { - throw new Error(`Malformed SSE JSON event: ${(e as Error).message}`); - } - } + for (const rawLine of lines) { + const line = rawLine.trimEnd(); + if (!line || line.startsWith(':')) { + continue; + } + if (line.startsWith('event: ')) { + currentEvent.event = line.substring(7).trim(); + continue; + } + if (line.startsWith('data:')) { + dataLines.push(line.substring(5).trimStart()); + continue; + } + if (line.startsWith('id: ')) { + currentEvent.id = line.substring(4).trim(); + continue; + } + if (line.startsWith('retry: ')) { + currentEvent.retry = parseInt(line.substring(7), 10); + } + } + + const data = dataLines.join('\n'); + if (!data) { + continue; + } + + if (data === '[DONE]') { + this.eventCount++; + events.push({ event: 'done', data: null, index: this.eventCount }); + continue; + } + + try { + currentEvent.data = JSON.parse(data); + this.eventCount++; + currentEvent.index = this.eventCount; + events.push({ ...currentEvent }); + } catch (e) { + if (typeof console !== 'undefined' && console.error) { + console.error( + '[SSEParser] Malformed JSON event:', + (e as Error).message, + 'Data:', + data.substring(0, 100) + ); + } + if (this.throwOnMalformedJson) { + throw new Error(`Malformed SSE JSON event: ${(e as Error).message}`); } - } else if (line.startsWith('id: ')) { - currentEvent.id = line.substring(4).trim(); - } else if (line.startsWith('retry: ')) { - currentEvent.retry = parseInt(line.substring(7), 10); } - // Empty lines separate events (already handled by JSON parsing) } return events; diff --git a/src/proxy/index.ts b/src/proxy/index.ts new file mode 100644 index 00000000..24a0a516 --- /dev/null +++ b/src/proxy/index.ts @@ -0,0 +1,7 @@ +export * from './profile-router'; +export * from './proxy-daemon'; +export * from './proxy-daemon-paths'; +export * from './proxy-env'; +export * from './upstream-url'; +export * from './transformers/request-transformer'; +export * from './transformers/sse-stream-transformer'; diff --git a/src/proxy/profile-router.ts b/src/proxy/profile-router.ts new file mode 100644 index 00000000..da856f6f --- /dev/null +++ b/src/proxy/profile-router.ts @@ -0,0 +1,70 @@ +import { resolveDroidProvider, type DroidProvider } from '../targets/droid-provider'; + +export interface OpenAICompatProfileConfig { + profileName: string; + settingsPath: string; + baseUrl: string; + apiKey: string; + provider: DroidProvider; + insecure?: boolean; + model?: string; + opusModel?: string; + sonnetModel?: string; + haikuModel?: string; +} + +export interface OpenAICompatProfileEnv { + ANTHROPIC_BASE_URL?: string; + ANTHROPIC_AUTH_TOKEN?: string; + ANTHROPIC_API_KEY?: string; + ANTHROPIC_MODEL?: string; + ANTHROPIC_DEFAULT_OPUS_MODEL?: string; + ANTHROPIC_DEFAULT_SONNET_MODEL?: string; + ANTHROPIC_DEFAULT_HAIKU_MODEL?: string; + ANTHROPIC_SMALL_FAST_MODEL?: string; + CCS_DROID_PROVIDER?: string; + CCS_OPENAI_PROXY_INSECURE?: string; +} + +export function isOpenAICompatProvider(provider: DroidProvider | null): provider is DroidProvider { + return provider === 'openai' || provider === 'generic-chat-completion-api'; +} + +export function resolveOpenAICompatProfileConfig( + profileName: string, + settingsPath: string, + env: OpenAICompatProfileEnv +): OpenAICompatProfileConfig | null { + const baseUrl = env.ANTHROPIC_BASE_URL?.trim() || ''; + const apiKey = env.ANTHROPIC_AUTH_TOKEN?.trim() || env.ANTHROPIC_API_KEY?.trim() || ''; + if (!baseUrl || !apiKey) { + return null; + } + + const provider = resolveDroidProvider({ + provider: env.CCS_DROID_PROVIDER, + baseUrl, + model: env.ANTHROPIC_MODEL, + }); + if (!isOpenAICompatProvider(provider)) { + return null; + } + + return { + profileName, + settingsPath, + baseUrl, + apiKey, + provider, + insecure: + env.CCS_OPENAI_PROXY_INSECURE === '1' || + env.CCS_OPENAI_PROXY_INSECURE?.toLowerCase() === 'true', + model: env.ANTHROPIC_MODEL?.trim() || undefined, + opusModel: env.ANTHROPIC_DEFAULT_OPUS_MODEL?.trim() || undefined, + sonnetModel: env.ANTHROPIC_DEFAULT_SONNET_MODEL?.trim() || undefined, + haikuModel: + env.ANTHROPIC_DEFAULT_HAIKU_MODEL?.trim() || + env.ANTHROPIC_SMALL_FAST_MODEL?.trim() || + undefined, + }; +} diff --git a/src/proxy/proxy-daemon-entry.ts b/src/proxy/proxy-daemon-entry.ts new file mode 100644 index 00000000..226433bb --- /dev/null +++ b/src/proxy/proxy-daemon-entry.ts @@ -0,0 +1,80 @@ +import { loadSettings } from '../utils/config-manager'; +import { resolveOpenAICompatProfileConfig } from './profile-router'; +import { startOpenAICompatProxyServer } from './server/proxy-server'; + +interface RuntimeOptions { + port: number; + profileName: string; + settingsPath: string; + authToken: string; + insecure: boolean; +} + +function parseArgs(argv: string[]): RuntimeOptions { + let port = 3456; + let profileName = ''; + let settingsPath = ''; + let authToken = ''; + let insecure = false; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--port' && argv[i + 1]) { + port = Number.parseInt(argv[++i] || '', 10) || port; + continue; + } + if (arg === '--profile' && argv[i + 1]) { + profileName = argv[++i] || ''; + continue; + } + if (arg === '--settings-path' && argv[i + 1]) { + settingsPath = argv[++i] || ''; + continue; + } + if (arg === '--auth-token' && argv[i + 1]) { + authToken = argv[++i] || ''; + continue; + } + if (arg === '--insecure') { + insecure = true; + } + } + + return { port, profileName, settingsPath, authToken, insecure }; +} + +function startRuntime(options: RuntimeOptions): void { + if (!options.authToken.trim()) { + throw new Error('Missing local proxy auth token'); + } + + const settings = loadSettings(options.settingsPath); + const profile = resolveOpenAICompatProfileConfig( + options.profileName, + options.settingsPath, + settings.env || {} + ); + if (!profile) { + throw new Error( + `Profile "${options.profileName}" is not an OpenAI-compatible settings profile` + ); + } + + const server = startOpenAICompatProxyServer({ + profile, + port: options.port, + authToken: options.authToken, + insecure: options.insecure, + }); + server.once('error', (error) => { + console.error((error as Error).message); + process.exit(1); + }); + const shutdown = () => server.close(); + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); +} + +if (require.main === module) { + startRuntime(parseArgs(process.argv.slice(2))); +} diff --git a/src/proxy/proxy-daemon-paths.ts b/src/proxy/proxy-daemon-paths.ts new file mode 100644 index 00000000..92b6b95b --- /dev/null +++ b/src/proxy/proxy-daemon-paths.ts @@ -0,0 +1,17 @@ +import * as path from 'path'; +import { getCcsDir } from '../utils/config-manager'; + +export const OPENAI_COMPAT_PROXY_DEFAULT_PORT = 3456; +export const OPENAI_COMPAT_PROXY_SERVICE_NAME = 'ccs-openai-compat-proxy'; + +export function getOpenAICompatProxyDir(): string { + return path.join(getCcsDir(), 'proxy'); +} + +export function getOpenAICompatProxyPidPath(): string { + return path.join(getOpenAICompatProxyDir(), 'daemon.pid'); +} + +export function getOpenAICompatProxySessionPath(): string { + return path.join(getOpenAICompatProxyDir(), 'session.json'); +} diff --git a/src/proxy/proxy-daemon-state.ts b/src/proxy/proxy-daemon-state.ts new file mode 100644 index 00000000..c1921948 --- /dev/null +++ b/src/proxy/proxy-daemon-state.ts @@ -0,0 +1,82 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { + getOpenAICompatProxyDir, + getOpenAICompatProxyPidPath, + getOpenAICompatProxySessionPath, +} from './proxy-daemon-paths'; + +export interface OpenAICompatProxySession { + profileName: string; + settingsPath: string; + port: number; + baseUrl: string; + authToken: string; + model?: string; + insecure?: boolean; +} + +function ensureProxyDir(): void { + fs.mkdirSync(getOpenAICompatProxyDir(), { recursive: true }); +} + +export function getOpenAICompatProxyPid(): number | null { + try { + const raw = fs.readFileSync(getOpenAICompatProxyPidPath(), 'utf8').trim(); + const pid = Number.parseInt(raw, 10); + return Number.isInteger(pid) ? pid : null; + } catch { + return null; + } +} + +export function writeOpenAICompatProxyPid(pid: number): void { + ensureProxyDir(); + fs.writeFileSync(getOpenAICompatProxyPidPath(), String(pid), 'utf8'); +} + +export function removeOpenAICompatProxyPid(): void { + try { + fs.unlinkSync(getOpenAICompatProxyPidPath()); + } catch { + // Best-effort cleanup. + } +} + +export function readOpenAICompatProxySession(): OpenAICompatProxySession | null { + try { + return JSON.parse( + fs.readFileSync(getOpenAICompatProxySessionPath(), 'utf8') + ) as OpenAICompatProxySession; + } catch { + return null; + } +} + +export function writeOpenAICompatProxySession(session: OpenAICompatProxySession): void { + ensureProxyDir(); + fs.writeFileSync( + getOpenAICompatProxySessionPath(), + JSON.stringify(session, null, 2) + '\n', + 'utf8' + ); +} + +export function removeOpenAICompatProxySession(): void { + try { + fs.unlinkSync(getOpenAICompatProxySessionPath()); + } catch { + // Best-effort cleanup. + } +} + +export function resolveOpenAICompatProxyEntrypointCandidates(): string[] { + const jsEntry = path.join(__dirname, 'proxy-daemon-entry.js'); + const tsEntry = path.join(__dirname, 'proxy-daemon-entry.ts'); + const isBunRuntime = process.execPath.toLowerCase().includes('bun'); + const runningFromDist = __filename.endsWith('.js'); + if (runningFromDist) { + return [jsEntry]; + } + return isBunRuntime ? [tsEntry, jsEntry] : [jsEntry]; +} diff --git a/src/proxy/proxy-daemon.ts b/src/proxy/proxy-daemon.ts new file mode 100644 index 00000000..603c3097 --- /dev/null +++ b/src/proxy/proxy-daemon.ts @@ -0,0 +1,380 @@ +import { spawn, type ChildProcess } from 'child_process'; +import * as crypto from 'crypto'; +import * as fs from 'fs'; +import * as http from 'http'; +import * as net from 'net'; +import * as lockfile from 'proper-lockfile'; +import { verifyProcessOwnership } from '../cursor/daemon-process-ownership'; +import type { OpenAICompatProfileConfig } from './profile-router'; +import { + OPENAI_COMPAT_PROXY_DEFAULT_PORT, + OPENAI_COMPAT_PROXY_SERVICE_NAME, + getOpenAICompatProxyDir, +} from './proxy-daemon-paths'; +import { + getOpenAICompatProxyPid, + readOpenAICompatProxySession, + removeOpenAICompatProxyPid, + removeOpenAICompatProxySession, + resolveOpenAICompatProxyEntrypointCandidates, + type OpenAICompatProxySession, + writeOpenAICompatProxyPid, + writeOpenAICompatProxySession, +} from './proxy-daemon-state'; + +export interface OpenAICompatProxyStatus extends Partial { + running: boolean; + pid?: number; +} + +export interface StartOpenAICompatProxyResult { + success: boolean; + alreadyRunning?: boolean; + authToken?: string; + pid?: number; + port: number; + error?: string; +} + +function generateProxyAuthToken(): string { + return crypto.randomBytes(24).toString('hex'); +} + +async function withOpenAICompatProxyLock(operation: () => Promise): Promise { + const proxyDir = getOpenAICompatProxyDir(); + await fs.promises.mkdir(proxyDir, { recursive: true }); + + let release: (() => Promise) | undefined; + try { + release = await lockfile.lock(proxyDir, { + stale: 10000, + retries: { retries: 20, minTimeout: 50, maxTimeout: 250 }, + realpath: false, + }); + } catch (error) { + throw new Error( + `Failed to lock OpenAI-compatible proxy directory (${proxyDir}): ${(error as Error).message}` + ); + } + + try { + return await operation(); + } finally { + if (release) { + try { + await release(); + } catch { + // Best-effort release. + } + } + } +} + +async function isPortOccupied(port: number): Promise { + return new Promise((resolve) => { + const socket = net.createConnection({ host: '127.0.0.1', port }); + const finish = (occupied: boolean) => { + socket.removeAllListeners(); + socket.destroy(); + resolve(occupied); + }; + + socket.once('connect', () => finish(true)); + socket.once('error', () => finish(false)); + socket.setTimeout(500, () => { + finish(false); + }); + }); +} + +async function findOpenAICompatProxyPort(): Promise { + for ( + let candidate = OPENAI_COMPAT_PROXY_DEFAULT_PORT; + candidate <= OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10; + candidate += 1 + ) { + if (!(await isPortOccupied(candidate))) { + return candidate; + } + } + + return 0; +} + +async function resolveDaemonEntrypoint(): Promise { + for (const candidate of resolveOpenAICompatProxyEntrypointCandidates()) { + try { + await fs.promises.access(candidate, fs.constants.R_OK); + return candidate; + } catch { + // Try next candidate. + } + } + return null; +} + +export async function isOpenAICompatProxyRunning(port: number): Promise { + return new Promise((resolve) => { + const req = http.request( + { hostname: '127.0.0.1', port, path: '/health', method: 'GET', timeout: 3000 }, + (res) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => (body += chunk)); + res.on('end', () => { + if (res.statusCode !== 200) { + resolve(false); + return; + } + try { + const payload = JSON.parse(body) as { service?: string }; + resolve(payload.service === OPENAI_COMPAT_PROXY_SERVICE_NAME); + } catch { + resolve(false); + } + }); + } + ); + req.on('error', () => resolve(false)); + req.on('timeout', () => { + req.destroy(); + resolve(false); + }); + req.end(); + }); +} + +export async function getOpenAICompatProxyStatus(): Promise { + const session = readOpenAICompatProxySession(); + const port = session?.port ?? OPENAI_COMPAT_PROXY_DEFAULT_PORT; + const running = await isOpenAICompatProxyRunning(port); + return { + running, + pid: running ? getOpenAICompatProxyPid() || undefined : undefined, + ...session, + }; +} + +async function stopOpenAICompatProxyUnlocked(): Promise<{ success: boolean; error?: string }> { + const pid = getOpenAICompatProxyPid(); + if (!pid) { + removeOpenAICompatProxySession(); + return { success: true }; + } + + const ownership = verifyProcessOwnership( + pid, + (commandLine) => + commandLine.includes('--ccs-openai-proxy-daemon') && + commandLine.includes('proxy-daemon-entry') + ); + + if (ownership === 'not-owned') { + removeOpenAICompatProxyPid(); + return { success: true }; + } + + if (ownership === 'unknown') { + return { + success: false, + error: `Refusing to stop PID ${pid}: unable to verify daemon ownership`, + }; + } + + if (ownership === 'not-running') { + removeOpenAICompatProxyPid(); + removeOpenAICompatProxySession(); + return { success: true }; + } + + try { + process.kill(pid, 'SIGTERM'); + let attempts = 0; + while (attempts < 10) { + await new Promise((resolve) => setTimeout(resolve, 500)); + try { + process.kill(pid, 0); + attempts += 1; + } catch { + break; + } + } + + if (attempts >= 10) { + try { + process.kill(pid, 'SIGKILL'); + } catch { + // Already exited. + } + } + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== 'ESRCH') { + return { success: false, error: `Failed to stop daemon: ${err.message}` }; + } + } + + removeOpenAICompatProxyPid(); + removeOpenAICompatProxySession(); + return { success: true }; +} + +export async function stopOpenAICompatProxy(): Promise<{ success: boolean; error?: string }> { + return withOpenAICompatProxyLock(() => stopOpenAICompatProxyUnlocked()); +} + +export async function startOpenAICompatProxy( + profile: OpenAICompatProfileConfig, + options: { port?: number; insecure?: boolean } = {} +): Promise { + return withOpenAICompatProxyLock(async () => { + const status = await getOpenAICompatProxyStatus(); + const port = + typeof options.port === 'number' + ? options.port + : status.running && status.profileName === profile.profileName && status.port + ? status.port + : await findOpenAICompatProxyPort(); + if (port === 0) { + return { + success: false, + port: OPENAI_COMPAT_PROXY_DEFAULT_PORT, + error: `No free proxy port found in range ${OPENAI_COMPAT_PROXY_DEFAULT_PORT}-${OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10}`, + }; + } + if (!Number.isInteger(port) || port < 1 || port > 65535) { + return { success: false, port, error: `Invalid port: ${port}` }; + } + if (status.running && status.profileName === profile.profileName && status.port === port) { + return { + success: true, + alreadyRunning: true, + pid: status.pid, + port, + authToken: status.authToken, + }; + } + if (status.running) { + if (status.profileName !== profile.profileName) { + return { + success: false, + port, + error: `Proxy already running for profile "${status.profileName}" on port ${status.port}. Stop it before starting a different profile.`, + }; + } + + const stopped = await stopOpenAICompatProxyUnlocked(); + if (!stopped.success) { + return { + success: false, + port, + error: stopped.error || 'Failed to restart the running proxy', + }; + } + } + + const daemonEntry = await resolveDaemonEntrypoint(); + if (!daemonEntry) { + return { + success: false, + port, + error: 'OpenAI proxy daemon entrypoint not found. Run `bun run build` and retry.', + }; + } + + return new Promise((resolve) => { + let resolved = false; + let timeout: NodeJS.Timeout | null = null; + const authToken = generateProxyAuthToken(); + + const finish = (result: StartOpenAICompatProxyResult) => { + if (resolved) return; + resolved = true; + if (timeout) clearTimeout(timeout); + if (!result.success) { + removeOpenAICompatProxyPid(); + removeOpenAICompatProxySession(); + } + resolve(result); + }; + + const proc: ChildProcess = spawn( + process.execPath, + [ + daemonEntry, + '--port', + String(port), + '--profile', + profile.profileName, + '--settings-path', + profile.settingsPath, + '--auth-token', + authToken, + ...(options.insecure ? ['--insecure'] : []), + '--ccs-openai-proxy-daemon', + ], + { stdio: 'ignore', detached: true } + ); + + proc.unref(); + if (proc.pid) { + writeOpenAICompatProxyPid(proc.pid); + } + writeOpenAICompatProxySession({ + profileName: profile.profileName, + settingsPath: profile.settingsPath, + port, + baseUrl: profile.baseUrl, + authToken, + model: profile.model, + insecure: options.insecure, + }); + + let attempts = 0; + const poll = async () => { + attempts += 1; + if (await isOpenAICompatProxyRunning(port)) { + finish({ success: true, pid: proc.pid, port, authToken }); + return; + } + if (attempts >= 30) { + finish({ + success: false, + port, + error: `Proxy daemon did not start within 30 seconds on port ${port}`, + }); + return; + } + timeout = setTimeout(poll, 1000); + }; + + timeout = setTimeout(poll, 1000); + proc.on('error', (error) => { + finish({ success: false, port, error: error.message }); + }); + proc.on('exit', (code, signal) => { + if (code === 0) { + finish({ + success: false, + port, + error: 'Proxy daemon exited before becoming healthy', + }); + return; + } + if (code !== null) { + finish({ + success: false, + port, + error: `Proxy daemon exited with code ${code}`, + }); + return; + } + finish({ + success: false, + port, + error: `Proxy daemon was killed by signal ${signal}`, + }); + }); + }); + }); +} diff --git a/src/proxy/proxy-env.ts b/src/proxy/proxy-env.ts new file mode 100644 index 00000000..ca11813a --- /dev/null +++ b/src/proxy/proxy-env.ts @@ -0,0 +1,23 @@ +import type { OpenAICompatProfileConfig } from './profile-router'; + +export function buildOpenAICompatProxyEnv( + profile: OpenAICompatProfileConfig, + port: number, + authToken: string, + claudeConfigDir?: string +): Record { + return { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, + ANTHROPIC_AUTH_TOKEN: authToken, + ...(profile.model ? { ANTHROPIC_MODEL: profile.model } : {}), + ...(profile.opusModel ? { ANTHROPIC_DEFAULT_OPUS_MODEL: profile.opusModel } : {}), + ...(profile.sonnetModel ? { ANTHROPIC_DEFAULT_SONNET_MODEL: profile.sonnetModel } : {}), + ...(profile.haikuModel + ? { + ANTHROPIC_DEFAULT_HAIKU_MODEL: profile.haikuModel, + ANTHROPIC_SMALL_FAST_MODEL: profile.haikuModel, + } + : {}), + ...(claudeConfigDir ? { CLAUDE_CONFIG_DIR: claudeConfigDir } : {}), + }; +} diff --git a/src/proxy/server/http-helpers.ts b/src/proxy/server/http-helpers.ts new file mode 100644 index 00000000..77a4adbc --- /dev/null +++ b/src/proxy/server/http-helpers.ts @@ -0,0 +1,81 @@ +import * as http from 'http'; +import { Readable } from 'stream'; + +const MAX_BODY_SIZE = 10 * 1024 * 1024; + +export function writeJson(res: http.ServerResponse, statusCode: number, payload: unknown): void { + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(payload)); +} + +export function readJsonBody(req: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + + const resolveOnce = (payload: unknown) => { + if (!settled) { + settled = true; + resolve(payload); + } + }; + + const rejectOnce = (error: Error) => { + if (!settled) { + settled = true; + reject(error); + } + }; + + req.on('data', (chunk: Buffer) => { + total += chunk.length; + if (total > MAX_BODY_SIZE) { + req.pause(); + rejectOnce(new Error('Request body too large (max 10MB)')); + return; + } + chunks.push(chunk); + }); + + req.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8').trim(); + if (!raw) { + resolveOnce({}); + return; + } + + try { + resolveOnce(JSON.parse(raw)); + } catch { + rejectOnce(new Error('Invalid JSON in request body')); + } + }); + + req.on('error', (error) => { + rejectOnce(error instanceof Error ? error : new Error(String(error))); + }); + }); +} + +export async function pipeWebResponseToNode( + response: Response, + res: http.ServerResponse +): Promise { + res.statusCode = response.status; + response.headers.forEach((value, key) => { + res.setHeader(key, value); + }); + + if (!response.body) { + res.end(); + return; + } + + const nodeStream = Readable.fromWeb(response.body as unknown as ReadableStream); + await new Promise((resolve, reject) => { + nodeStream.on('error', reject); + nodeStream.on('end', resolve); + nodeStream.pipe(res); + }); +} diff --git a/src/proxy/server/messages-route.ts b/src/proxy/server/messages-route.ts new file mode 100644 index 00000000..49a66585 --- /dev/null +++ b/src/proxy/server/messages-route.ts @@ -0,0 +1,174 @@ +import * as http from 'http'; +import type { Dispatcher } from 'undici'; +import type { OpenAICompatProfileConfig } from '../profile-router'; +import { ProxyRequestTransformer } from '../transformers/request-transformer'; +import { ProxySseStreamTransformer } from '../transformers/sse-stream-transformer'; +import { resolveOpenAIChatCompletionsUrl } from '../upstream-url'; +import { pipeWebResponseToNode, readJsonBody, writeJson } from './http-helpers'; + +const REQUEST_TIMEOUT_MS = 30_000; + +class ProxyInputError extends Error { + constructor(message: string) { + super(message); + this.name = 'ProxyInputError'; + } +} + +function buildUpstreamHeaders(profile: OpenAICompatProfileConfig): Record { + return { + 'Content-Type': 'application/json', + Authorization: `Bearer ${profile.apiKey}`, + 'User-Agent': 'CCS-OpenAI-Compat-Proxy/1.0', + }; +} + +function buildUpstreamBody(profile: OpenAICompatProfileConfig, rawBody: unknown): string { + let transformed; + try { + const transformer = new ProxyRequestTransformer(); + transformed = transformer.transform(rawBody); + } catch (error) { + const message = error instanceof Error ? error.message : 'Invalid Anthropic request'; + throw new ProxyInputError(message); + } + const body = { + ...transformed, + model: transformed.model || profile.model, + stream: transformed.stream === true, + }; + return JSON.stringify(body); +} + +export function extractIncomingProxyToken(headers: http.IncomingHttpHeaders): string | null { + const xApiKey = headers['x-api-key']; + if (typeof xApiKey === 'string' && xApiKey.trim().length > 0) { + return xApiKey.trim(); + } + + const anthropicApiKey = headers['anthropic-api-key']; + if (typeof anthropicApiKey === 'string' && anthropicApiKey.trim().length > 0) { + return anthropicApiKey.trim(); + } + + const authHeader = headers.authorization; + if (typeof authHeader === 'string' && authHeader.trim().length > 0) { + const trimmed = authHeader.trim(); + const bearerPrefix = 'Bearer '; + return trimmed.startsWith(bearerPrefix) ? trimmed.slice(bearerPrefix.length).trim() : trimmed; + } + + return null; +} + +export function validateIncomingProxyAuth( + headers: http.IncomingHttpHeaders, + expectedToken: string +): boolean { + return extractIncomingProxyToken(headers) === expectedToken; +} + +function buildFetchInit( + profile: OpenAICompatProfileConfig, + body: string, + signal: AbortSignal, + insecureDispatcher?: Dispatcher +): RequestInit { + const init: RequestInit = { + method: 'POST', + headers: buildUpstreamHeaders(profile), + body, + signal, + }; + + if (insecureDispatcher) { + (init as Record).dispatcher = insecureDispatcher; + } + + return init; +} + +export async function handleProxyMessagesRequest( + req: http.IncomingMessage, + res: http.ServerResponse, + profile: OpenAICompatProfileConfig, + expectedAuthToken: string, + insecureDispatcher?: Dispatcher +): Promise { + const transformer = new ProxySseStreamTransformer(); + + if (!validateIncomingProxyAuth(req.headers, expectedAuthToken)) { + await pipeWebResponseToNode( + transformer.error(401, 'authentication_error', 'Missing or invalid local proxy token'), + res + ); + return; + } + + try { + const rawBody = await readJsonBody(req); + const upstreamBody = buildUpstreamBody(profile, rawBody); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + const abortOnDisconnect = () => { + if (!controller.signal.aborted && !res.writableEnded) { + controller.abort(); + } + }; + + req.on('aborted', abortOnDisconnect); + req.on('close', abortOnDisconnect); + res.on('close', abortOnDisconnect); + + try { + const upstreamResponse = await fetch( + resolveOpenAIChatCompletionsUrl(profile.baseUrl), + buildFetchInit(profile, upstreamBody, controller.signal, insecureDispatcher) + ); + clearTimeout(timeout); + const response = await transformer.transform(upstreamResponse); + await pipeWebResponseToNode(response, res); + } finally { + clearTimeout(timeout); + } + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown proxy error'; + const status = + error instanceof Error && error.name === 'AbortError' + ? 502 + : error instanceof ProxyInputError + ? 400 + : message.includes('Request body too large') + ? 413 + : message.includes('Invalid JSON') + ? 400 + : 502; + const type = status >= 500 ? 'api_error' : 'invalid_request_error'; + await pipeWebResponseToNode( + transformer.error( + status, + type, + error instanceof Error && error.name === 'AbortError' + ? 'The upstream provider did not respond within 30 seconds' + : message + ), + res + ); + } +} + +export function handleProxyModelsRequest( + res: http.ServerResponse, + profile: OpenAICompatProfileConfig +): void { + const data = [profile.model, profile.opusModel, profile.sonnetModel, profile.haikuModel] + .filter((value): value is string => typeof value === 'string' && value.length > 0) + .map((id) => ({ + id, + object: 'model', + created: 0, + owned_by: profile.provider, + })); + + writeJson(res, 200, { object: 'list', data }); +} diff --git a/src/proxy/server/proxy-server.ts b/src/proxy/server/proxy-server.ts new file mode 100644 index 00000000..b808d9ee --- /dev/null +++ b/src/proxy/server/proxy-server.ts @@ -0,0 +1,75 @@ +import * as http from 'http'; +import { Agent } from 'undici'; +import type { OpenAICompatProfileConfig } from '../profile-router'; +import { OPENAI_COMPAT_PROXY_SERVICE_NAME } from '../proxy-daemon-paths'; +import { + handleProxyMessagesRequest, + handleProxyModelsRequest, + validateIncomingProxyAuth, +} from './messages-route'; +import { writeJson } from './http-helpers'; + +export interface OpenAICompatProxyServerOptions { + profile: OpenAICompatProfileConfig; + port: number; + authToken: string; + insecure?: boolean; +} + +export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOptions): http.Server { + const insecureDispatcher = options.insecure + ? new Agent({ connect: { rejectUnauthorized: false } }) + : undefined; + const server = http.createServer(async (req, res) => { + const method = req.method || 'GET'; + const requestUrl = req.url || '/'; + const parsedUrl = new URL(requestUrl, 'http://127.0.0.1'); + const pathname = + parsedUrl.pathname.length > 1 ? parsedUrl.pathname.replace(/\/+$/, '') : parsedUrl.pathname; + + if (method === 'GET' && pathname === '/health') { + writeJson(res, 200, { + ok: true, + service: OPENAI_COMPAT_PROXY_SERVICE_NAME, + profile: options.profile.profileName, + port: options.port, + }); + return; + } + + if (method === 'GET' && pathname === '/v1/models') { + if (!validateIncomingProxyAuth(req.headers, options.authToken)) { + writeJson(res, 401, { + type: 'error', + error: { + type: 'authentication_error', + message: 'Missing or invalid local proxy token', + }, + }); + return; + } + handleProxyModelsRequest(res, options.profile); + return; + } + + if (method === 'POST' && pathname === '/v1/messages') { + await handleProxyMessagesRequest( + req, + res, + options.profile, + options.authToken, + insecureDispatcher + ); + return; + } + + writeJson(res, 404, { error: 'Not found' }); + }); + + server.on('close', () => { + void insecureDispatcher?.close(); + }); + + server.listen(options.port, '127.0.0.1'); + return server; +} diff --git a/src/proxy/transformers/request-transformer.ts b/src/proxy/transformers/request-transformer.ts new file mode 100644 index 00000000..1804d426 --- /dev/null +++ b/src/proxy/transformers/request-transformer.ts @@ -0,0 +1,89 @@ +import { translateAnthropicRequest } from '../../cursor/cursor-anthropic-translator'; + +interface AnthropicProxyRequestShape { + max_tokens?: unknown; + temperature?: unknown; + top_p?: unknown; + stop_sequences?: unknown; + metadata?: unknown; + tools?: unknown; +} + +export interface ProxyOpenAIRequest extends ReturnType { + max_tokens?: number; + temperature?: number; + top_p?: number; + stop?: string[]; + metadata?: Record; + tools?: Array<{ + type: 'function'; + function: { + name: string; + description?: string; + parameters: Record; + }; + }>; +} + +function asNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function asStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + + const result = value.filter( + (entry): entry is string => typeof entry === 'string' && entry.length > 0 + ); + return result.length > 0 ? result : undefined; +} + +function asMetadata(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function transformTools(value: unknown): ProxyOpenAIRequest['tools'] { + if (!Array.isArray(value)) { + return undefined; + } + + const tools = value + .filter( + (entry): entry is { name?: unknown; description?: unknown; input_schema?: unknown } => + typeof entry === 'object' && entry !== null + ) + .map((entry) => ({ + type: 'function' as const, + function: { + name: typeof entry.name === 'string' ? entry.name : 'tool', + ...(typeof entry.description === 'string' ? { description: entry.description } : {}), + parameters: + typeof entry.input_schema === 'object' && entry.input_schema !== null + ? (entry.input_schema as Record) + : { type: 'object', properties: {} }, + }, + })); + + return tools.length > 0 ? tools : undefined; +} + +export class ProxyRequestTransformer { + transform(raw: unknown): ProxyOpenAIRequest { + const translated = translateAnthropicRequest(raw); + const source = (raw || {}) as AnthropicProxyRequestShape; + + return { + ...translated, + max_tokens: asNumber(source.max_tokens), + temperature: asNumber(source.temperature), + top_p: asNumber(source.top_p), + stop: asStringArray(source.stop_sequences), + metadata: asMetadata(source.metadata), + tools: transformTools(source.tools), + }; + } +} diff --git a/src/proxy/transformers/sse-stream-transformer.ts b/src/proxy/transformers/sse-stream-transformer.ts new file mode 100644 index 00000000..53774c21 --- /dev/null +++ b/src/proxy/transformers/sse-stream-transformer.ts @@ -0,0 +1,14 @@ +import { + createAnthropicErrorResponse, + createAnthropicProxyResponse, +} from '../../cursor/cursor-anthropic-response'; + +export class ProxySseStreamTransformer { + async transform(response: Response): Promise { + return createAnthropicProxyResponse(response); + } + + error(status: number, type: string, message: string): Response { + return createAnthropicErrorResponse(status, type, message); + } +} diff --git a/src/proxy/upstream-url.ts b/src/proxy/upstream-url.ts new file mode 100644 index 00000000..d41d89b6 --- /dev/null +++ b/src/proxy/upstream-url.ts @@ -0,0 +1,36 @@ +function normalizePathname(pathname: string): string { + const trimmed = pathname.replace(/\/+$/, ''); + return trimmed || ''; +} + +function ensureSupportedProtocol(parsed: URL): void { + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`Unsupported upstream protocol: ${parsed.protocol}`); + } +} + +function buildResolvedUrl(baseUrl: string, suffix: string): string { + const parsed = new URL(baseUrl); + ensureSupportedProtocol(parsed); + + const pathname = normalizePathname(parsed.pathname); + if (pathname.endsWith(suffix)) { + return parsed.toString(); + } + + if (pathname.endsWith('/v1') || pathname.endsWith('/api')) { + parsed.pathname = `${pathname}${suffix.startsWith('/') ? suffix : `/${suffix}`}`; + return parsed.toString(); + } + + parsed.pathname = pathname ? `${pathname}/v1${suffix}` : `/v1${suffix}`; + return parsed.toString(); +} + +export function resolveOpenAIChatCompletionsUrl(baseUrl: string): string { + return buildResolvedUrl(baseUrl, '/chat/completions'); +} + +export function resolveOpenAIModelsUrl(baseUrl: string): string { + return buildResolvedUrl(baseUrl, '/models'); +} From 5c66c1f12162306d1039ec2e255ad9894199d1f4 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 14 Apr 2026 19:27:19 -0400 Subject: [PATCH 2/8] test(proxy): cover proxy lifecycle and routing --- tests/e2e/openai-provider-routing.e2e.test.ts | 188 ++++++++++++++++ tests/e2e/proxy-command.e2e.test.ts | 88 ++++++++ .../proxy/daemon-lifecycle.test.ts | 127 +++++++++++ .../proxy/messages-endpoint.test.ts | 204 ++++++++++++++++++ tests/unit/glmt/sse-parser.test.ts | 35 +++ tests/unit/proxy/profile-router.test.ts | 40 ++++ .../transformers/request-transformer.test.ts | 63 ++++++ 7 files changed, 745 insertions(+) create mode 100644 tests/e2e/openai-provider-routing.e2e.test.ts create mode 100644 tests/e2e/proxy-command.e2e.test.ts create mode 100644 tests/integration/proxy/daemon-lifecycle.test.ts create mode 100644 tests/integration/proxy/messages-endpoint.test.ts create mode 100644 tests/unit/glmt/sse-parser.test.ts create mode 100644 tests/unit/proxy/profile-router.test.ts create mode 100644 tests/unit/proxy/transformers/request-transformer.test.ts diff --git a/tests/e2e/openai-provider-routing.e2e.test.ts b/tests/e2e/openai-provider-routing.e2e.test.ts new file mode 100644 index 00000000..cc2d823e --- /dev/null +++ b/tests/e2e/openai-provider-routing.e2e.test.ts @@ -0,0 +1,188 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as http from 'http'; +import * as os from 'os'; +import * as path from 'path'; +import { spawn, spawnSync } from 'child_process'; +import getPort from 'get-port'; + +const DIST_ENTRY = path.join(process.cwd(), 'dist', 'ccs.js'); + +let originalCcsHome: string | undefined; +let tempDir: string; +let upstreamServer: http.Server; +let upstreamBody: unknown; + +beforeEach(() => { + originalCcsHome = process.env.CCS_HOME; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-routing-e2e-')); + upstreamBody = undefined; +}); + +afterEach(() => { + try { + upstreamServer?.close(); + } catch { + // Best-effort cleanup. + } + spawnSync(process.execPath, [DIST_ENTRY, 'proxy', 'stop'], { + env: { ...process.env, CCS_HOME: tempDir }, + }); + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +function startMockUpstream(port: number): Promise { + return new Promise((resolve) => { + upstreamServer = http.createServer(async (req, res) => { + let body = ''; + for await (const chunk of req) { + body += chunk.toString(); + } + upstreamBody = JSON.parse(body); + + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search","arguments":"{\\"q\\":\\"docs\\"}"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":9,"completion_tokens":4}}\n\n' + ); + res.end('data: [DONE]\n\n'); + }); + upstreamServer.listen(port, '127.0.0.1', () => resolve()); + }); +} + +function runCli(args: string[], env: Record): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(process.execPath, [DIST_ENTRY, ...args], { + env: { + ...process.env, + ...env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('close', (code) => { + resolve({ code, stdout, stderr }); + }); + }); +} + +beforeAll(() => { + const result = spawnSync(process.execPath, ['run', 'build'], { + encoding: 'utf8', + env: process.env, + }); + expect(result.status).toBe(0); +}); + +describe('openai provider routing e2e', () => { + it('routes a settings profile through the local proxy into an OpenAI-compatible upstream', async () => { + const upstreamPort = await getPort(); + await startMockUpstream(upstreamPort); + + const ccsDir = path.join(tempDir, '.ccs'); + const binDir = path.join(tempDir, 'bin'); + const outputPath = path.join(tempDir, 'claude-output.json'); + fs.mkdirSync(ccsDir, { recursive: true }); + fs.mkdirSync(binDir, { recursive: true }); + + const settingsPath = path.join(ccsDir, 'hf.settings.json'); + fs.writeFileSync( + path.join(ccsDir, 'config.json'), + JSON.stringify({ profiles: { hf: settingsPath } }, null, 2), + 'utf8' + ); + fs.writeFileSync( + settingsPath, + JSON.stringify({ + env: { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${upstreamPort}`, + ANTHROPIC_AUTH_TOKEN: 'hf_token', + ANTHROPIC_MODEL: 'hf-model', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }, + }), + 'utf8' + ); + fs.writeFileSync( + path.join(binDir, 'claude'), + `#!/usr/bin/env node +const fs = require('fs'); +(async () => { + const response = await fetch(\`\${process.env.ANTHROPIC_BASE_URL}/v1/messages\`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + 'x-api-key': process.env.ANTHROPIC_AUTH_TOKEN, + }, + body: JSON.stringify({ + model: process.env.ANTHROPIC_MODEL, + stream: true, + tools: [{ name: 'search', description: 'Search docs', input_schema: { type: 'object' } }], + messages: [{ role: 'user', content: 'Find docs' }], + }), + }); + const text = await response.text(); + fs.writeFileSync(process.env.CCS_E2E_OUTPUT, JSON.stringify({ + status: response.status, + baseUrl: process.env.ANTHROPIC_BASE_URL, + authToken: process.env.ANTHROPIC_AUTH_TOKEN, + text + }, null, 2)); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`, + { mode: 0o755 } + ); + + const result = await runCli(['hf'], { + ...process.env, + CCS_HOME: tempDir, + CCS_E2E_OUTPUT: outputPath, + PATH: `${binDir}:${process.env.PATH || ''}`, + }); + + expect(result.code).toBe(0); + const payload = JSON.parse(fs.readFileSync(outputPath, 'utf8')) as { + status: number; + baseUrl: string; + authToken: string; + text: string; + }; + + expect(payload.status).toBe(200); + expect(payload.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(payload.authToken).toMatch(/^[a-f0-9]{48}$/); + expect(payload.text).toContain('event: message_start'); + expect(payload.text).toContain('tool_use'); + expect(payload.text).toContain('message_stop'); + + const parsedUpstream = upstreamBody as { + messages?: Array<{ role: string; content: string }>; + tools?: Array<{ type: string }>; + }; + expect(parsedUpstream.messages?.[0]).toEqual({ role: 'user', content: 'Find docs' }); + expect(parsedUpstream.tools?.[0]?.type).toBe('function'); + }, 35000); +}); diff --git a/tests/e2e/proxy-command.e2e.test.ts b/tests/e2e/proxy-command.e2e.test.ts new file mode 100644 index 00000000..550ee8a2 --- /dev/null +++ b/tests/e2e/proxy-command.e2e.test.ts @@ -0,0 +1,88 @@ +import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import getPort from 'get-port'; + +const DIST_ENTRY = path.join(process.cwd(), 'dist', 'ccs.js'); + +let originalCcsHome: string | undefined; +let tempDir: string; + +function runCli(args: string[], extraEnv: Record = {}) { + return spawnSync(process.execPath, [DIST_ENTRY, ...args], { + encoding: 'utf8', + env: { + ...process.env, + CCS_HOME: tempDir, + ...extraEnv, + }, + }); +} + +beforeAll(() => { + const result = spawnSync(process.execPath, ['run', 'build'], { + encoding: 'utf8', + env: process.env, + }); + expect(result.status).toBe(0); +}); + +beforeEach(() => { + originalCcsHome = process.env.CCS_HOME; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-proxy-e2e-')); +}); + +afterEach(() => { + runCli(['proxy', 'stop']); + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('proxy command e2e', () => { + it('starts, reports status, activates, and stops via the built CLI', async () => { + const port = await getPort(); + const ccsDir = path.join(tempDir, '.ccs'); + fs.mkdirSync(ccsDir, { recursive: true }); + const settingsPath = path.join(ccsDir, 'hf.settings.json'); + fs.writeFileSync( + path.join(ccsDir, 'config.json'), + JSON.stringify({ profiles: { hf: settingsPath } }, null, 2), + 'utf8' + ); + fs.writeFileSync( + settingsPath, + JSON.stringify({ + env: { + ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', + ANTHROPIC_AUTH_TOKEN: 'ollama', + ANTHROPIC_MODEL: 'qwen3-coder', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }, + }), + 'utf8' + ); + + const started = runCli(['proxy', 'start', 'hf', '--port', String(port)]); + expect(started.status).toBe(0); + + const status = runCli(['proxy', 'status']); + expect(status.stdout).toContain(`Proxy running on port ${port}`); + expect(status.stdout).toContain('Profile: hf'); + + const activate = runCli(['proxy', 'activate', '--shell', 'bash']); + expect(activate.stdout).toContain(`export ANTHROPIC_BASE_URL='http://127.0.0.1:${port}'`); + expect(activate.stdout).toMatch(/export ANTHROPIC_AUTH_TOKEN='[a-f0-9]{48}'/); + + const health = await fetch(`http://127.0.0.1:${port}/health`); + expect(health.status).toBe(200); + + const stopped = runCli(['proxy', 'stop']); + expect(stopped.status).toBe(0); + }, 35000); +}); diff --git a/tests/integration/proxy/daemon-lifecycle.test.ts b/tests/integration/proxy/daemon-lifecycle.test.ts new file mode 100644 index 00000000..5a5aa86d --- /dev/null +++ b/tests/integration/proxy/daemon-lifecycle.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import getPort from 'get-port'; +import { + getOpenAICompatProxyStatus, + startOpenAICompatProxy, + stopOpenAICompatProxy, +} from '../../../src/proxy/proxy-daemon'; +import { resolveOpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; + +let originalCcsHome: string | undefined; +let tempDir: string; + +beforeEach(() => { + originalCcsHome = process.env.CCS_HOME; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-openai-proxy-')); + process.env.CCS_HOME = tempDir; +}); + +afterEach(async () => { + await stopOpenAICompatProxy(); + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('openai proxy daemon lifecycle', () => { + it('starts, reports status, serves health/models, and stops', async () => { + const port = await getPort(); + const settingsPath = path.join(tempDir, 'hf.settings.json'); + fs.writeFileSync( + settingsPath, + JSON.stringify({ + env: { + ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', + ANTHROPIC_AUTH_TOKEN: 'ollama', + ANTHROPIC_MODEL: 'qwen3-coder', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }, + }), + 'utf8' + ); + + const profile = resolveOpenAICompatProfileConfig('hf', settingsPath, { + ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', + ANTHROPIC_AUTH_TOKEN: 'ollama', + ANTHROPIC_MODEL: 'qwen3-coder', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + if (!profile) { + throw new Error('Expected an OpenAI-compatible profile'); + } + + const started = await startOpenAICompatProxy(profile, { port }); + expect(started.success).toBe(true); + expect(started.authToken).toBeTruthy(); + + const status = await getOpenAICompatProxyStatus(); + expect(status.running).toBe(true); + expect(status.profileName).toBe('hf'); + expect(status.authToken).toBe(started.authToken); + + const health = await fetch(`http://127.0.0.1:${port}/health`); + expect(health.status).toBe(200); + + const models = (await ( + await fetch(`http://127.0.0.1:${port}/v1/models`, { + headers: { 'x-api-key': started.authToken! }, + }) + ).json()) as { data?: Array<{ id: string }> }; + expect(models.data?.map((entry) => entry.id)).toEqual(['qwen3-coder']); + + const stopped = await stopOpenAICompatProxy(); + expect(stopped.success).toBe(true); + expect((await getOpenAICompatProxyStatus()).running).toBe(false); + }, 35000); + + it('refuses to replace a running proxy for a different profile', async () => { + const firstPort = await getPort(); + const firstSettingsPath = path.join(tempDir, 'hf.settings.json'); + fs.writeFileSync( + firstSettingsPath, + JSON.stringify({ + env: { + ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', + ANTHROPIC_AUTH_TOKEN: 'ollama', + ANTHROPIC_MODEL: 'qwen3-coder', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }, + }), + 'utf8' + ); + const firstProfile = resolveOpenAICompatProfileConfig('hf', firstSettingsPath, { + ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434', + ANTHROPIC_AUTH_TOKEN: 'ollama', + ANTHROPIC_MODEL: 'qwen3-coder', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + if (!firstProfile) { + throw new Error('Expected first OpenAI-compatible profile'); + } + + const firstStart = await startOpenAICompatProxy(firstProfile, { port: firstPort }); + expect(firstStart.success).toBe(true); + + const secondProfile = resolveOpenAICompatProfileConfig('openai', path.join(tempDir, 'openai.settings.json'), { + ANTHROPIC_BASE_URL: 'https://api.openai.com/v1', + ANTHROPIC_AUTH_TOKEN: 'sk-openai', + ANTHROPIC_MODEL: 'gpt-4.1', + }); + if (!secondProfile) { + throw new Error('Expected second OpenAI-compatible profile'); + } + + const secondStart = await startOpenAICompatProxy(secondProfile, { port: await getPort() }); + expect(secondStart.success).toBe(false); + expect(secondStart.error).toContain('Proxy already running for profile "hf"'); + + const health = await fetch(`http://127.0.0.1:${firstPort}/health`); + expect(health.status).toBe(200); + }); +}); diff --git a/tests/integration/proxy/messages-endpoint.test.ts b/tests/integration/proxy/messages-endpoint.test.ts new file mode 100644 index 00000000..ee96ce4c --- /dev/null +++ b/tests/integration/proxy/messages-endpoint.test.ts @@ -0,0 +1,204 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import getPort from 'get-port'; +import * as http from 'http'; +import { startOpenAICompatProxyServer } from '../../../src/proxy/server/proxy-server'; +import type { OpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; + +let upstreamServer: http.Server; +let proxyServer: http.Server; +let upstreamBody: unknown; +let upstreamPort: number; +let proxyPort: number; + +function startMockUpstream(): Promise { + return new Promise((resolve) => { + upstreamServer = http.createServer(async (req, res) => { + if (req.method !== 'POST' || req.url !== '/v1/chat/completions') { + res.writeHead(404).end(); + return; + } + + let body = ''; + for await (const chunk of req) { + body += chunk.toString(); + } + upstreamBody = JSON.parse(body); + const parsed = upstreamBody as { stream?: boolean }; + + if (parsed.stream) { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search","arguments":"{\\"q\\":\\"docs\\"}"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":9,"completion_tokens":4}}\n\n' + ); + res.end('data: [DONE]\n\n'); + return; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + id: 'chatcmpl_1', + model: 'hf-model', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'Plain answer' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 2, completion_tokens: 3 }, + }) + ); + }); + + upstreamServer.listen(upstreamPort, '127.0.0.1', () => resolve()); + }); +} + +async function requestProxy(payload: unknown): Promise { + return fetch(`http://127.0.0.1:${proxyPort}/v1/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + 'x-api-key': 'test-proxy-token', + }, + body: JSON.stringify(payload), + }); +} + +beforeEach(async () => { + upstreamPort = await getPort(); + proxyPort = await getPort(); + upstreamBody = undefined; + await startMockUpstream(); + const profile: OpenAICompatProfileConfig = { + profileName: 'hf', + settingsPath: '/tmp/hf.settings.json', + baseUrl: `http://127.0.0.1:${upstreamPort}`, + apiKey: 'hf_token', + provider: 'generic-chat-completion-api', + model: 'hf-model', + }; + proxyServer = startOpenAICompatProxyServer({ + profile, + port: proxyPort, + authToken: 'test-proxy-token', + }); +}); + +afterEach(async () => { + await new Promise((resolve) => proxyServer.close(() => resolve())); + await new Promise((resolve) => upstreamServer.close(() => resolve())); +}); + +describe('openai proxy messages endpoint', () => { + it('translates Anthropic requests to OpenAI upstream and streams Anthropic SSE back', async () => { + const response = await requestProxy({ + model: 'hf-model', + stream: true, + messages: [{ role: 'user', content: [{ type: 'text', text: 'Find docs' }] }], + tools: [{ name: 'search', description: 'Search docs', input_schema: { type: 'object' } }], + }); + + const body = await response.text(); + expect(response.status).toBe(200); + expect(body).toContain('event: message_start'); + expect(body).toContain('content_block_delta'); + expect(body).toContain('tool_use'); + expect(body).toContain('message_stop'); + + const parsedUpstream = upstreamBody as { + messages?: Array<{ role: string; content: string }>; + tools?: Array<{ type: string; function: { name: string } }>; + }; + expect(parsedUpstream.messages?.[0]).toEqual({ role: 'user', content: 'Find docs' }); + expect(parsedUpstream.tools?.[0]?.type).toBe('function'); + expect(parsedUpstream.tools?.[0]?.function.name).toBe('search'); + }); + + it('falls back to Anthropic JSON for non-streaming requests', async () => { + const response = await requestProxy({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }); + const body = (await response.json()) as { + type?: string; + content?: Array<{ type?: string; text?: string }>; + }; + + expect(response.status).toBe(200); + expect(body.type).toBe('message'); + expect(body.content?.[0]).toEqual({ type: 'text', text: 'Plain answer' }); + }); + + it('returns invalid_request_error for malformed JSON', async () => { + const response = await fetch(`http://127.0.0.1:${proxyPort}/v1/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + 'x-api-key': 'test-proxy-token', + }, + body: '{bad-json', + }); + const body = (await response.json()) as { error?: { type?: string; message?: string } }; + + expect(response.status).toBe(400); + expect(body.error?.type).toBe('invalid_request_error'); + expect(body.error?.message).toContain('Invalid JSON'); + }); + + it('rejects requests without the local proxy auth token', async () => { + const response = await fetch(`http://127.0.0.1:${proxyPort}/v1/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }), + }); + const body = (await response.json()) as { error?: { type?: string } }; + + expect(response.status).toBe(401); + expect(body.error?.type).toBe('authentication_error'); + }); + + it('returns invalid_request_error for unsupported Anthropic content blocks', async () => { + const response = await requestProxy({ + model: 'hf-model', + messages: [{ role: 'user', content: [{ type: 'image' }] }], + }); + const body = (await response.json()) as { error?: { type?: string; message?: string } }; + + expect(response.status).toBe(400); + expect(body.error?.type).toBe('invalid_request_error'); + expect(body.error?.message).toContain('is not supported'); + }); + + it('accepts query strings and trailing slashes on the messages route', async () => { + const response = await fetch(`http://127.0.0.1:${proxyPort}/v1/messages/?beta=test`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + authorization: 'Bearer test-proxy-token', + }, + body: JSON.stringify({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }), + }); + + expect(response.status).toBe(200); + }); +}); diff --git a/tests/unit/glmt/sse-parser.test.ts b/tests/unit/glmt/sse-parser.test.ts new file mode 100644 index 00000000..10f14177 --- /dev/null +++ b/tests/unit/glmt/sse-parser.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'bun:test'; +import { SSEParser } from '../../../src/glmt/sse-parser'; + +describe('SSEParser', () => { + it('merges multi-line data fields into one JSON payload', () => { + const parser = new SSEParser({ throwOnMalformedJson: true }); + const events = parser.parse( + [ + 'event: message', + 'data: {"choices":[', + 'data: {"delta":{"content":"Hello"}}', + 'data: ]}', + '', + '', + ].join('\n') + ); + + expect(events).toHaveLength(1); + expect( + (events[0]?.data as { choices?: Array<{ delta?: { content?: string } }> })?.choices?.[0] + ?.delta?.content + ).toBe('Hello'); + }); + + it('keeps incomplete events buffered until the separator arrives', () => { + const parser = new SSEParser({ throwOnMalformedJson: true }); + expect(parser.parse('data: {"choices":[{"delta":{"content":"Hi"}}]}')).toEqual([]); + const events = parser.parse('\n\n'); + + expect(events).toHaveLength(1); + expect((events[0]?.data as { choices?: Array<{ delta?: { content?: string } }> })?.choices?.[0]?.delta?.content).toBe( + 'Hi' + ); + }); +}); diff --git a/tests/unit/proxy/profile-router.test.ts b/tests/unit/proxy/profile-router.test.ts new file mode 100644 index 00000000..76bfe7be --- /dev/null +++ b/tests/unit/proxy/profile-router.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'bun:test'; +import { resolveOpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; + +describe('resolveOpenAICompatProfileConfig', () => { + it('detects generic OpenAI-compatible profiles from base URL and provider hint', () => { + const result = resolveOpenAICompatProfileConfig('hf', '/tmp/hf.settings.json', { + ANTHROPIC_BASE_URL: 'https://router.huggingface.co/v1', + ANTHROPIC_AUTH_TOKEN: 'hf_token', + ANTHROPIC_MODEL: 'openai/gpt-oss-120b:fastest', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + + expect(result).not.toBeNull(); + expect(result?.provider).toBe('generic-chat-completion-api'); + expect(result?.profileName).toBe('hf'); + }); + + it('detects official OpenAI-style endpoints', () => { + const result = resolveOpenAICompatProfileConfig('openai', '/tmp/openai.settings.json', { + ANTHROPIC_BASE_URL: 'https://api.openai.com/v1', + ANTHROPIC_AUTH_TOKEN: 'sk-openai', + ANTHROPIC_MODEL: 'gpt-4.1', + CCS_OPENAI_PROXY_INSECURE: 'true', + }); + + expect(result?.provider).toBe('openai'); + expect(result?.model).toBe('gpt-4.1'); + expect(result?.insecure).toBe(true); + }); + + it('ignores Anthropic-compatible profiles', () => { + const result = resolveOpenAICompatProfileConfig('glm', '/tmp/glm.settings.json', { + ANTHROPIC_BASE_URL: 'https://api.z.ai/api/anthropic', + ANTHROPIC_AUTH_TOKEN: 'glm-key', + ANTHROPIC_MODEL: 'glm-5', + }); + + expect(result).toBeNull(); + }); +}); diff --git a/tests/unit/proxy/transformers/request-transformer.test.ts b/tests/unit/proxy/transformers/request-transformer.test.ts new file mode 100644 index 00000000..e3d6afad --- /dev/null +++ b/tests/unit/proxy/transformers/request-transformer.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'bun:test'; +import { ProxyRequestTransformer } from '../../../../src/proxy/transformers/request-transformer'; + +describe('ProxyRequestTransformer', () => { + it('translates Anthropic messages into OpenAI-compatible chat payloads', () => { + const transformer = new ProxyRequestTransformer(); + const result = transformer.transform({ + model: 'claude-sonnet-4.5', + stream: true, + messages: [ + { role: 'user', content: [{ type: 'text', text: 'Find release notes' }] }, + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'release' } }], + }, + { + role: 'user', + content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'v7.69.1' }], + }, + ], + thinking: { type: 'enabled', budget_tokens: 9000 }, + max_tokens: 1024, + temperature: 0.2, + top_p: 0.9, + stop_sequences: ['STOP'], + metadata: { trace: 'abc' }, + }); + + expect(result.stream).toBe(true); + expect(result.reasoning_effort).toBe('high'); + expect(result.max_tokens).toBe(1024); + expect(result.temperature).toBe(0.2); + expect(result.top_p).toBe(0.9); + expect(result.stop).toEqual(['STOP']); + expect(result.metadata).toEqual({ trace: 'abc' }); + expect(result.messages[0]).toEqual({ role: 'user', content: 'Find release notes' }); + expect(result.messages[1]?.tool_calls?.[0]?.function.name).toBe('search'); + expect(result.messages[2]).toEqual({ + role: 'tool', + tool_call_id: 'toolu_1', + content: 'v7.69.1', + }); + }); + + it('drops malformed optional fields but preserves the translated core request', () => { + const transformer = new ProxyRequestTransformer(); + const result = transformer.transform({ + messages: [{ role: 'user', content: 'hello' }], + max_tokens: 'bad', + temperature: 'bad', + top_p: 'bad', + stop_sequences: ['A', 1], + metadata: 'bad', + }); + + expect(result.messages).toEqual([{ role: 'user', content: 'hello' }]); + expect(result.max_tokens).toBeUndefined(); + expect(result.temperature).toBeUndefined(); + expect(result.top_p).toBeUndefined(); + expect(result.stop).toEqual(['A']); + expect(result.metadata).toBeUndefined(); + }); +}); From 7b6485249530c890d9bf3549221dcbd0f3f6d18f Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 14 Apr 2026 19:27:38 -0400 Subject: [PATCH 3/8] docs(proxy): document openai-compatible routing --- README.md | 19 ++++ docs/openai-compatible-providers.md | 135 ++++++++++++++++++++++++++++ docs/project-roadmap.md | 3 +- 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 docs/openai-compatible-providers.md diff --git a/README.md b/README.md index cb981b5b..dbc10eb3 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,25 @@ ccs glm ccs ollama ``` +## OpenAI-Compatible Routing + +CCS can now bridge Claude Code into OpenAI-compatible providers through a local +Anthropic-compatible proxy instead of requiring a native Anthropic upstream. + +```bash +ccs api create --preset hf +ccs hf +``` + +Need to manage the proxy manually? + +```bash +ccs proxy start hf +eval "$(ccs proxy activate)" +``` + +Guide: [OpenAI-Compatible Provider Routing](./docs/openai-compatible-providers.md) + Need the full setup path instead of the short version? | Need | Start here | diff --git a/docs/openai-compatible-providers.md b/docs/openai-compatible-providers.md new file mode 100644 index 00000000..9a8a0466 --- /dev/null +++ b/docs/openai-compatible-providers.md @@ -0,0 +1,135 @@ +# OpenAI-Compatible Provider Routing + +CCS can route Claude Code traffic through a local Anthropic-compatible proxy when +your API profile points at an OpenAI-compatible chat completions endpoint. + +This is useful for providers such as: + +- Hugging Face Inference Providers +- OpenRouter +- Ollama +- llama.cpp servers +- OpenAI-compatible self-hosted gateways + +## What CCS Does + +When you launch a compatible settings profile with the Claude target, CCS now: + +1. Starts a local proxy on `127.0.0.1` +2. Accepts Anthropic `/v1/messages` traffic from Claude Code +3. Translates requests into OpenAI chat-completions format +4. Forwards them to your configured upstream provider +5. Translates streaming responses back into Anthropic SSE + +You do not need to rewrite your profile by hand each time. + +## Quick Start + +Create or reuse an API profile that points at an OpenAI-compatible endpoint: + +```bash +ccs api create --preset hf +``` + +Then you can use the profile directly: + +```bash +ccs hf +``` + +CCS detects that the profile is OpenAI-compatible and auto-routes Claude Code +through the local proxy. + +## Manual Proxy Lifecycle + +If you want to manage the proxy explicitly: + +```bash +ccs proxy start hf +eval "$(ccs proxy activate)" +ccs proxy status +ccs proxy stop +``` + +`ccs proxy activate` prints the `ANTHROPIC_*` exports needed for a local +Anthropic-compatible session against the running proxy. + +## One Active Proxy Profile + +The current runtime is a single local proxy daemon. + +- Reusing the same OpenAI-compatible profile is supported +- Starting a different OpenAI-compatible profile while one proxy is already + running is rejected instead of silently replacing the active upstream + +This is intentional to avoid breaking an in-flight Claude session by swapping +its upstream provider out from under it. + +## How Profile Detection Works + +CCS keeps these profiles in the normal API/settings-profile flow. + +Anthropic-compatible endpoints such as: + +- `https://api.anthropic.com` +- `https://api.z.ai/api/anthropic` +- `https://api.deepseek.com/anthropic` + +continue to launch directly. + +OpenAI-compatible endpoints such as: + +- `https://router.huggingface.co/v1` +- `https://api.openai.com/v1` +- `http://localhost:11434` + +are routed through the local proxy for Claude-target launches. + +## Self-Signed TLS + +If your upstream gateway uses a self-signed or privately issued certificate, +set this in the profile settings JSON: + +```json +{ + "env": { + "CCS_OPENAI_PROXY_INSECURE": "1" + } +} +``` + +That flag is respected by both: + +- `ccs ` auto-routing +- `ccs proxy start ` + +## Supported Runtime Paths + +- `ccs ` with Claude target: auto-starts the local proxy when needed +- `ccs proxy start `: starts the proxy explicitly +- `GET /health`: proxy liveness check +- `GET /v1/models`: local view of the configured model mapping +- `POST /v1/messages`: Anthropic-compatible request entrypoint + +## Validation + +The shipped coverage includes: + +- unit tests for OpenAI-compatible profile detection +- unit tests for Anthropic -> OpenAI request translation +- unit tests for multi-line SSE parsing +- integration tests for `/v1/messages` request/response translation +- integration tests for daemon lifecycle and `/health` / `/v1/models` +- e2e tests for `ccs proxy` lifecycle +- e2e tests for `ccs ` auto-routing through a mock upstream + +## Current Scope + +The current implementation focuses on the core routing path: + +- local proxy lifecycle +- Anthropic/OpenAI request-response translation +- Claude-target settings profile auto-routing + +Scenario-based routing and token-count-driven model switching remain follow-up +work if they are needed beyond the base provider-routing flow. diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index 4f977fdd..3b82689d 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -1,6 +1,6 @@ # CCS Project Roadmap -Last Updated: 2026-04-10 +Last Updated: 2026-04-14 Forward-looking roadmap documenting current priorities, GitHub issues, and future feature plans. @@ -41,6 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic ### Recent Fixes +- **2026-04-14**: **#991** CCS now auto-routes Claude-target settings profiles that use OpenAI-compatible endpoints through a local Anthropic-compatible proxy instead of sending raw Anthropic `/v1/messages` traffic directly to chat-completions backends. The new `ccs proxy` command supports `start`, `status`, `activate`, and `stop`, the runtime exposes `/health` and `/v1/models`, the shared SSE parser now handles multi-line `data:` payloads, and the feature ships with dedicated unit, integration, and e2e coverage. - **2026-04-10**: **#765** `/providers` now includes a first-class Hugging Face preset for API Profiles. CCS exposes Hugging Face Inference Providers through the existing OpenAI-compatible profile flow with the official router endpoint `https://router.huggingface.co/v1`, a short `hf` default profile name, and `hf` preset alias support for both the dashboard chooser and `ccs api create --preset hf`. - **2026-04-10**: **#944** Image Analysis auth readiness no longer collapses to native Read when merged runtime-status dependency overrides include a missing initializer value. CCS now preserves default dependency functions when override entries are `undefined`, still reads token-backed auth status directly in the local readiness path, and includes regression coverage for the missing-initializer case that previously surfaced as `deps.initializeAccounts is not a function`. - **2026-04-10**: **#945** CCS now normalizes Gemini CLI and Antigravity tier signals around an explicit `free / pro / ultra / unknown` model, preserves raw tier ids such as `g1-pro-tier`, enriches Gemini quota responses with provider entitlement evidence, classifies `MODEL_CAPACITY_EXHAUSTED` separately from auth/entitlement failures, fixes the Antigravity CLI quota table so live quota-derived tiers no longer collapse back to stale `unknown`, adds Gemini tier ids to CLI quota output, extends Gemini Flash Lite grouping to cover `gemini-3.1-flash-lite-preview`, and allows Gemini account surfaces to render the same tier badge semantics as Antigravity. From a6aa576d5a6e11d53cdd1d85f22b57a63fbe9b47 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Tue, 14 Apr 2026 23:49:34 -0400 Subject: [PATCH 4/8] fix(proxy): detect dashscope compatible-mode profiles --- src/proxy/profile-router.ts | 19 +++++++++++++------ src/targets/droid-provider.ts | 2 ++ tests/unit/proxy/profile-router.test.ts | 13 +++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/proxy/profile-router.ts b/src/proxy/profile-router.ts index da856f6f..9fe6b092 100644 --- a/src/proxy/profile-router.ts +++ b/src/proxy/profile-router.ts @@ -1,4 +1,8 @@ -import { resolveDroidProvider, type DroidProvider } from '../targets/droid-provider'; +import { + inferDroidProviderFromBaseUrl, + resolveDroidProvider, + type DroidProvider, +} from '../targets/droid-provider'; export interface OpenAICompatProfileConfig { profileName: string; @@ -41,11 +45,14 @@ export function resolveOpenAICompatProfileConfig( return null; } - const provider = resolveDroidProvider({ - provider: env.CCS_DROID_PROVIDER, - baseUrl, - model: env.ANTHROPIC_MODEL, - }); + const providerFromUrl = inferDroidProviderFromBaseUrl(baseUrl); + const provider = isOpenAICompatProvider(providerFromUrl) + ? providerFromUrl + : resolveDroidProvider({ + provider: env.CCS_DROID_PROVIDER, + baseUrl, + model: env.ANTHROPIC_MODEL, + }); if (!isOpenAICompatProvider(provider)) { return null; } diff --git a/src/targets/droid-provider.ts b/src/targets/droid-provider.ts index 22a8d8c2..1c5f9850 100644 --- a/src/targets/droid-provider.ts +++ b/src/targets/droid-provider.ts @@ -86,8 +86,10 @@ export function inferDroidProviderFromBaseUrl( host.includes('api.deepinfra.com') || host.includes('api.fireworks.ai') || host.includes('inference.baseten.co') || + host.includes('dashscope') || host.includes('huggingface.co') || host.includes('ollama.com') || + pathname.includes('/compatible-mode') || pathname.includes('/openai') || pathname.includes('/chat/completions') ) { diff --git a/tests/unit/proxy/profile-router.test.ts b/tests/unit/proxy/profile-router.test.ts index 76bfe7be..ba523ed9 100644 --- a/tests/unit/proxy/profile-router.test.ts +++ b/tests/unit/proxy/profile-router.test.ts @@ -37,4 +37,17 @@ describe('resolveOpenAICompatProfileConfig', () => { expect(result).toBeNull(); }); + + it('prefers OpenAI-compatible URL inference over stale anthropic provider hints', () => { + const result = resolveOpenAICompatProfileConfig('q', '/tmp/q.settings.json', { + ANTHROPIC_BASE_URL: 'https://dashscope-us.aliyuncs.com/compatible-mode/v1', + ANTHROPIC_AUTH_TOKEN: 'q-token', + ANTHROPIC_MODEL: 'qwen3.6-plus', + CCS_DROID_PROVIDER: 'anthropic', + }); + + expect(result).not.toBeNull(); + expect(result?.provider).toBe('generic-chat-completion-api'); + expect(result?.model).toBe('qwen3.6-plus'); + }); }); From b346693c3a70205a3f6cce46e556cd6e1ceaad8b Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Wed, 15 Apr 2026 00:01:38 -0400 Subject: [PATCH 5/8] docs(proxy): credit claude-code-router --- README.md | 11 +++++++++++ docs/openai-compatible-providers.md | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/README.md b/README.md index dbc10eb3..300e2190 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,17 @@ eval "$(ccs proxy activate)" Guide: [OpenAI-Compatible Provider Routing](./docs/openai-compatible-providers.md) +### Related Project: claude-code-router + +[claude-code-router](https://github.com/musistudio/claude-code-router) is an +excellent standalone tool for routing Claude Code requests to OpenAI-compatible +providers. CCS's local proxy and SSE transformation work was directly informed +by CCR's transformer architecture. + +Use CCR when you want a standalone router without CCS profile management. +Use CCS when you want the routing flow integrated with CCS profiles, runtime +bridges, and the existing `ccs` command surface. + Need the full setup path instead of the short version? | Need | Start here | diff --git a/docs/openai-compatible-providers.md b/docs/openai-compatible-providers.md index 9a8a0466..ddae7656 100644 --- a/docs/openai-compatible-providers.md +++ b/docs/openai-compatible-providers.md @@ -11,6 +11,23 @@ This is useful for providers such as: - llama.cpp servers - OpenAI-compatible self-hosted gateways +## Related Project: claude-code-router + +[claude-code-router](https://github.com/musistudio/claude-code-router) is the +main external reference that informed this CCS work. Their Anthropic/OpenAI +transformer design helped shape the routing approach here. + +When to use CCR: + +- you want a standalone router without CCS profile integration +- you do not need CCS account/runtime management around the request flow + +When to use CCS: + +- you already use CCS API profiles or runtime bridges +- you want the proxy flow available through `ccs ` and `ccs proxy ...` +- you want the routing behavior documented and tested inside the CCS workflow + ## What CCS Does When you launch a compatible settings profile with the Claude target, CCS now: From 17187c4abd5d62552e137cd783a92e7dbd356e88 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Wed, 15 Apr 2026 00:55:11 -0400 Subject: [PATCH 6/8] feat(proxy): complete openai routing scope --- README.md | 4 + docs/openai-compatible-providers.md | 184 ++++++++- docs/project-roadmap.md | 2 +- src/commands/proxy-command.ts | 23 +- src/config/unified-config-loader.ts | 23 ++ src/config/unified-config-types.ts | 26 ++ src/proxy/proxy-daemon-entry.ts | 9 +- src/proxy/proxy-daemon-state.ts | 1 + src/proxy/proxy-daemon.ts | 13 +- src/proxy/proxy-env.ts | 10 +- src/proxy/request-router.ts | 222 +++++++++++ src/proxy/routing-config.ts | 74 ++++ src/proxy/server/messages-route.ts | 93 ++++- src/proxy/server/proxy-server.ts | 37 +- src/proxy/transformers/request-transformer.ts | 358 +++++++++++++++++- src/types/config.ts | 15 + src/types/index.ts | 2 + src/web-server/routes/config-routes.ts | 4 + tests/e2e/proxy-command.e2e.test.ts | 24 +- .../proxy/messages-edge-cases.test.ts | 259 +++++++++++++ .../proxy/messages-endpoint.test.ts | 40 +- .../integration/proxy/request-routing.test.ts | 217 +++++++++++ tests/unit/proxy/request-router.test.ts | 194 ++++++++++ .../transformers/request-transformer.test.ts | 38 ++ 24 files changed, 1819 insertions(+), 53 deletions(-) create mode 100644 src/proxy/request-router.ts create mode 100644 src/proxy/routing-config.ts create mode 100644 tests/integration/proxy/messages-edge-cases.test.ts create mode 100644 tests/integration/proxy/request-routing.test.ts create mode 100644 tests/unit/proxy/request-router.test.ts diff --git a/README.md b/README.md index 300e2190..ec27693d 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ ccs proxy start hf eval "$(ccs proxy activate)" ``` +The proxy also supports request-time `profile:model` selectors, scenario-based +model routing through `proxy.routing`, and explicit activation helpers such as +`ccs proxy activate --fish`. + Guide: [OpenAI-Compatible Provider Routing](./docs/openai-compatible-providers.md) ### Related Project: claude-code-router diff --git a/docs/openai-compatible-providers.md b/docs/openai-compatible-providers.md index ddae7656..1cf86650 100644 --- a/docs/openai-compatible-providers.md +++ b/docs/openai-compatible-providers.md @@ -68,8 +68,22 @@ ccs proxy status ccs proxy stop ``` -`ccs proxy activate` prints the `ANTHROPIC_*` exports needed for a local -Anthropic-compatible session against the running proxy. +Useful variants: + +```bash +ccs proxy start hf --host 127.0.0.1 +ccs proxy activate --fish +``` + +`ccs proxy activate` now prints the full local runtime contract: + +- `ANTHROPIC_BASE_URL` +- `ANTHROPIC_AUTH_TOKEN` +- `ANTHROPIC_MODEL` plus tier defaults when present +- `DISABLE_TELEMETRY` +- `DISABLE_COST_WARNINGS` +- `API_TIMEOUT_MS` +- `NO_PROXY` ## One Active Proxy Profile @@ -82,6 +96,56 @@ The current runtime is a single local proxy daemon. This is intentional to avoid breaking an in-flight Claude session by swapping its upstream provider out from under it. +## Request-Time Routing + +The proxy is no longer limited to the startup profile's default model. + +Supported request-time selectors: + +- `profile:model` + Example: `deepseek:deepseek-reasoner` +- `profile` + Example: `openrouter` +- plain model ids + Example: `deepseek-chat` + +Routing behavior: + +1. `profile:model` wins immediately. +2. Scenario routing may override the active profile when configured. +3. Plain model ids are matched against the configured OpenAI-compatible + profiles before falling back to the active profile. + +This means a Claude session launched through one compatible profile can still +request another compatible profile/model when the proxy can resolve it safely. + +## Scenario Routing + +Scenario routing is now supported through `proxy.routing` in your CCS config. + +Example `~/.ccs/config.yaml`: + +```yaml +proxy: + routing: + default: "deepseek:deepseek-chat" + background: "ollama:qwen2.5-coder:0.5b" + think: "deepseek:deepseek-reasoner" + longContext: "openrouter:google/gemini-2.5-pro" + longContextThreshold: 60000 + webSearch: "openrouter:perplexity/sonar-pro" +``` + +Current scenario detection: + +- `background`: requested model contains `haiku` +- `think`: Anthropic `thinking` is enabled +- `longContext`: estimated request tokens exceed `longContextThreshold` +- `webSearch`: tool list includes `web_search` +- `default`: fallback selector when the above do not apply + +Routing decisions are logged through CCS structured logs. + ## How Profile Detection Works CCS keeps these profiles in the normal API/settings-profile flow. @@ -102,6 +166,79 @@ OpenAI-compatible endpoints such as: are routed through the local proxy for Claude-target launches. +## Provider Setup + +### DeepSeek + +Use a settings profile whose env looks like: + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "https://api.deepseek.com/v1", + "ANTHROPIC_AUTH_TOKEN": "sk-...", + "ANTHROPIC_MODEL": "deepseek-chat", + "CCS_DROID_PROVIDER": "generic-chat-completion-api" + } +} +``` + +Typical override target: + +- `deepseek:deepseek-reasoner` + +### OpenRouter + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "https://openrouter.ai/api/v1", + "ANTHROPIC_AUTH_TOKEN": "sk-or-...", + "ANTHROPIC_MODEL": "openai/gpt-4.1-mini", + "CCS_DROID_PROVIDER": "generic-chat-completion-api" + } +} +``` + +Useful when you want: + +- model fan-out behind one provider profile +- long-context or web-search scenario targets + +### Ollama / Local Gateways + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "http://127.0.0.1:11434", + "ANTHROPIC_AUTH_TOKEN": "ollama", + "ANTHROPIC_MODEL": "qwen3-coder", + "CCS_DROID_PROVIDER": "generic-chat-completion-api" + } +} +``` + +For self-signed HTTPS gateways, add `CCS_OPENAI_PROXY_INSECURE=1`. + +### DashScope / Qwen Compatible Mode + +DashScope's compatible endpoint works even when older settings files still +carry a stale Anthropic-style provider hint: + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "https://dashscope-us.aliyuncs.com/compatible-mode/v1", + "ANTHROPIC_AUTH_TOKEN": "sk-...", + "ANTHROPIC_MODEL": "qwen3.6-plus", + "CCS_DROID_PROVIDER": "anthropic" + } +} +``` + +CCS now infers the OpenAI-compatible route from the base URL and does not let +that stale provider hint block proxy routing. + ## Self-Signed TLS If your upstream gateway uses a self-signed or privately issued certificate, @@ -124,29 +261,50 @@ That flag is respected by both: - `ccs ` with Claude target: auto-starts the local proxy when needed - `ccs proxy start `: starts the proxy explicitly +- `GET /`: proxy info and bound profile details - `GET /health`: proxy liveness check - `GET /v1/models`: local view of the configured model mapping - `POST /v1/messages`: Anthropic-compatible request entrypoint +## Troubleshooting + +### Missing or invalid local proxy token + +- Re-run `eval "$(ccs proxy activate)"` +- Check `ccs proxy status` and confirm the expected profile is running + +### Self-signed or private CA upstream + +- Add `CCS_OPENAI_PROXY_INSECURE=1` to the profile settings +- Restart the proxy after changing the setting + +### Port conflict on `3456` + +- Start with a fixed port: `ccs proxy start hf --port 3457` +- Re-run `ccs proxy activate` after changing the port + +### Provider returns `429` or empty upstream output + +- CCS now preserves upstream rate-limit errors and retry headers +- Empty or malformed provider JSON is returned as Anthropic-style `api_error` + +### Requests route to the wrong model/profile + +- Use an explicit selector such as `profile:model` +- Review `proxy.routing` if scenario routing is enabled +- Check CCS structured logs in `~/.ccs/logs/current.jsonl` for routing decisions + ## Validation The shipped coverage includes: - unit tests for OpenAI-compatible profile detection - unit tests for Anthropic -> OpenAI request translation +- unit tests for request-time profile/model routing and scenario routing - unit tests for multi-line SSE parsing - integration tests for `/v1/messages` request/response translation +- integration tests for rate limits, empty upstream responses, timeout handling, + thinking/tool-call chunk streaming, and request-time routing - integration tests for daemon lifecycle and `/health` / `/v1/models` - e2e tests for `ccs proxy` lifecycle - e2e tests for `ccs ` auto-routing through a mock upstream - -## Current Scope - -The current implementation focuses on the core routing path: - -- local proxy lifecycle -- Anthropic/OpenAI request-response translation -- Claude-target settings profile auto-routing - -Scenario-based routing and token-count-driven model switching remain follow-up -work if they are needed beyond the base provider-routing flow. diff --git a/docs/project-roadmap.md b/docs/project-roadmap.md index 3b82689d..c6271e38 100644 --- a/docs/project-roadmap.md +++ b/docs/project-roadmap.md @@ -41,7 +41,7 @@ All major modularization work is complete. The codebase evolved from monolithic ### Recent Fixes -- **2026-04-14**: **#991** CCS now auto-routes Claude-target settings profiles that use OpenAI-compatible endpoints through a local Anthropic-compatible proxy instead of sending raw Anthropic `/v1/messages` traffic directly to chat-completions backends. The new `ccs proxy` command supports `start`, `status`, `activate`, and `stop`, the runtime exposes `/health` and `/v1/models`, the shared SSE parser now handles multi-line `data:` payloads, and the feature ships with dedicated unit, integration, and e2e coverage. +- **2026-04-14**: **#991** CCS now auto-routes Claude-target settings profiles that use OpenAI-compatible endpoints through a local Anthropic-compatible proxy instead of sending raw Anthropic `/v1/messages` traffic directly to chat-completions backends. The `ccs proxy` command now supports `start`, `status`, `activate`, and `stop` with explicit host binding, shell-aware activation helpers, and a fuller local runtime env contract. The proxy surface now exposes `GET /`, `/health`, `/v1/models`, and `/v1/messages`, logs routing decisions into CCS structured logs, supports Anthropic image blocks plus request-time `profile:model` overrides, and adds config-driven scenario routing (`background`, `think`, `longContext`, `webSearch`) on top of the compatible-profile path. Coverage now includes request routing, rate-limit/timeout/empty-upstream failures, chunked tool-call streaming, and disconnect cleanup alongside the existing unit, integration, and e2e suites. - **2026-04-10**: **#765** `/providers` now includes a first-class Hugging Face preset for API Profiles. CCS exposes Hugging Face Inference Providers through the existing OpenAI-compatible profile flow with the official router endpoint `https://router.huggingface.co/v1`, a short `hf` default profile name, and `hf` preset alias support for both the dashboard chooser and `ccs api create --preset hf`. - **2026-04-10**: **#944** Image Analysis auth readiness no longer collapses to native Read when merged runtime-status dependency overrides include a missing initializer value. CCS now preserves default dependency functions when override entries are `undefined`, still reads token-backed auth status directly in the local readiness path, and includes regression coverage for the missing-initializer case that previously surfaced as `deps.initializeAccounts is not a function`. - **2026-04-10**: **#945** CCS now normalizes Gemini CLI and Antigravity tier signals around an explicit `free / pro / ultra / unknown` model, preserves raw tier ids such as `g1-pro-tier`, enriches Gemini quota responses with provider entitlement evidence, classifies `MODEL_CAPACITY_EXHAUSTED` separately from auth/entitlement failures, fixes the Antigravity CLI quota table so live quota-derived tiers no longer collapse back to stale `unknown`, adds Gemini tier ids to CLI quota output, extends Gemini Flash Lite grouping to cover `gemini-3.1-flash-lite-preview`, and allows Gemini account surfaces to render the same tier badge semantics as Antigravity. diff --git a/src/commands/proxy-command.ts b/src/commands/proxy-command.ts index 9110f3f1..51c406f9 100644 --- a/src/commands/proxy-command.ts +++ b/src/commands/proxy-command.ts @@ -36,12 +36,15 @@ function showHelp(): number { console.log(''); console.log('Options:'); console.log(' --port Override the local proxy port (default: 3456)'); + console.log(' --host Bind the proxy server to a specific host (default: 127.0.0.1)'); console.log(' --shell activate only: auto|bash|zsh|fish|powershell'); + console.log(' --fish activate only: shorthand for --shell fish'); console.log(' --insecure Disable upstream TLS verification'); console.log(''); console.log('Examples:'); console.log(' ccs proxy start hf'); console.log(' eval "$(ccs proxy activate)"'); + console.log(' ccs proxy activate --fish'); console.log(' ccs proxy status'); console.log(' ccs proxy stop'); console.log(''); @@ -61,11 +64,14 @@ function resolveProfile(profileName: string) { async function handleStart(args: string[]): Promise { const profileName = args.find((arg) => !arg.startsWith('-')); if (!profileName) { - console.error(fail('Usage: ccs proxy start [--port ] [--insecure]')); + console.error( + fail('Usage: ccs proxy start [--port ] [--host ] [--insecure]') + ); return 1; } const portValue = parseOptionValue(args, '--port'); + const host = parseOptionValue(args, '--host'); const port = portValue ? Number.parseInt(portValue, 10) || 3456 : undefined; let profile; try { @@ -77,6 +83,7 @@ async function handleStart(args: string[]): Promise { const result = await startOpenAICompatProxy(profile, { ...(port ? { port } : {}), + ...(host ? { host } : {}), insecure: args.includes('--insecure'), }); @@ -101,6 +108,10 @@ async function handleStatus(): Promise { } console.log(ok(`Proxy running on port ${status.port}`)); + if (status.host) { + console.log(` Host: ${status.host}`); + console.log(` Local URL: http://${status.host}:${status.port}`); + } console.log(` Profile: ${status.profileName}`); console.log(` Base URL: ${status.baseUrl}`); if (status.model) { @@ -119,7 +130,7 @@ async function handleActivate(args: string[]): Promise { return 1; } - const shell = detectShell(parseOptionValue(args, '--shell')); + const shell = detectShell(args.includes('--fish') ? 'fish' : parseOptionValue(args, '--shell')); let profile; try { profile = resolveProfile(status.profileName); @@ -128,7 +139,13 @@ async function handleActivate(args: string[]): Promise { return 1; } - const env = buildOpenAICompatProxyEnv(profile, status.port, status.authToken); + const env = buildOpenAICompatProxyEnv( + profile, + status.port, + status.authToken, + undefined, + status.host || '127.0.0.1' + ); Object.entries(env).forEach(([key, value]) => { console.log(formatExportLine(shell, key, value)); }); diff --git a/src/config/unified-config-loader.ts b/src/config/unified-config-loader.ts index 5865ba5d..5cd1254e 100644 --- a/src/config/unified-config-loader.ts +++ b/src/config/unified-config-loader.ts @@ -384,6 +384,18 @@ function mergeWithDefaults(partial: Partial): UnifiedConfig { : defaults.cliproxy.routing?.strategy, }, }, + proxy: { + routing: { + default: partial.proxy?.routing?.default ?? defaults.proxy?.routing?.default, + background: partial.proxy?.routing?.background ?? defaults.proxy?.routing?.background, + think: partial.proxy?.routing?.think ?? defaults.proxy?.routing?.think, + longContext: partial.proxy?.routing?.longContext ?? defaults.proxy?.routing?.longContext, + webSearch: partial.proxy?.routing?.webSearch ?? defaults.proxy?.routing?.webSearch, + longContextThreshold: + partial.proxy?.routing?.longContextThreshold ?? + defaults.proxy?.routing?.longContextThreshold, + }, + }, logging: { enabled: partial.logging?.enabled ?? DEFAULT_LOGGING_CONFIG.enabled, level: partial.logging?.level ?? DEFAULT_LOGGING_CONFIG.level, @@ -674,6 +686,17 @@ function generateYamlWithComments(config: UnifiedConfig): string { ); lines.push(''); + if (config.proxy?.routing) { + lines.push('# ----------------------------------------------------------------------------'); + lines.push('# Proxy Routing: OpenAI-compatible local proxy model selection rules'); + lines.push('# Use profile:model selectors to force a target profile and upstream model.'); + lines.push('# ----------------------------------------------------------------------------'); + lines.push( + yaml.dump({ proxy: config.proxy }, { indent: 2, lineWidth: -1, quotingType: '"' }).trim() + ); + lines.push(''); + } + if (config.logging) { lines.push('# ----------------------------------------------------------------------------'); lines.push('# Logging: CCS-owned structured runtime logs'); diff --git a/src/config/unified-config-types.ts b/src/config/unified-config-types.ts index 3ab0f545..afb7006c 100644 --- a/src/config/unified-config-types.ts +++ b/src/config/unified-config-types.ts @@ -498,6 +498,19 @@ export interface ProxyLocalConfig { auto_start: boolean; } +export interface OpenAICompatProxyRoutingConfig { + default?: string; + background?: string; + think?: string; + longContext?: string; + webSearch?: string; + longContextThreshold?: number; +} + +export interface OpenAICompatProxyConfig { + routing?: OpenAICompatProxyRoutingConfig; +} + /** * CLIProxy server configuration section. * Controls whether CCS uses local or remote CLIProxyAPI instance. @@ -856,6 +869,8 @@ export interface UnifiedConfig { profiles: Record; /** CLIProxy configuration */ cliproxy: CLIProxyConfig; + /** OpenAI-compatible local proxy configuration */ + proxy?: OpenAICompatProxyConfig; /** CCS-owned structured logging configuration */ logging?: LoggingConfig; /** User preferences */ @@ -934,6 +949,12 @@ export const DEFAULT_CLIPROXY_SERVER_CONFIG: CliproxyServerConfig = { }, }; +export const DEFAULT_OPENAI_COMPAT_PROXY_CONFIG: OpenAICompatProxyConfig = { + routing: { + longContextThreshold: 60_000, + }, +}; + /** * Create an empty unified config with defaults. */ @@ -958,6 +979,11 @@ export function createEmptyUnifiedConfig(): UnifiedConfig { strategy: 'round-robin', }, }, + proxy: { + routing: { + ...DEFAULT_OPENAI_COMPAT_PROXY_CONFIG.routing, + }, + }, logging: { ...DEFAULT_LOGGING_CONFIG }, preferences: { theme: 'system', diff --git a/src/proxy/proxy-daemon-entry.ts b/src/proxy/proxy-daemon-entry.ts index 226433bb..0013ef7e 100644 --- a/src/proxy/proxy-daemon-entry.ts +++ b/src/proxy/proxy-daemon-entry.ts @@ -4,6 +4,7 @@ import { startOpenAICompatProxyServer } from './server/proxy-server'; interface RuntimeOptions { port: number; + host: string; profileName: string; settingsPath: string; authToken: string; @@ -12,6 +13,7 @@ interface RuntimeOptions { function parseArgs(argv: string[]): RuntimeOptions { let port = 3456; + let host = '127.0.0.1'; let profileName = ''; let settingsPath = ''; let authToken = ''; @@ -23,6 +25,10 @@ function parseArgs(argv: string[]): RuntimeOptions { port = Number.parseInt(argv[++i] || '', 10) || port; continue; } + if (arg === '--host' && argv[i + 1]) { + host = argv[++i] || host; + continue; + } if (arg === '--profile' && argv[i + 1]) { profileName = argv[++i] || ''; continue; @@ -40,7 +46,7 @@ function parseArgs(argv: string[]): RuntimeOptions { } } - return { port, profileName, settingsPath, authToken, insecure }; + return { port, host, profileName, settingsPath, authToken, insecure }; } function startRuntime(options: RuntimeOptions): void { @@ -62,6 +68,7 @@ function startRuntime(options: RuntimeOptions): void { const server = startOpenAICompatProxyServer({ profile, + host: options.host, port: options.port, authToken: options.authToken, insecure: options.insecure, diff --git a/src/proxy/proxy-daemon-state.ts b/src/proxy/proxy-daemon-state.ts index c1921948..525c5a7e 100644 --- a/src/proxy/proxy-daemon-state.ts +++ b/src/proxy/proxy-daemon-state.ts @@ -9,6 +9,7 @@ import { export interface OpenAICompatProxySession { profileName: string; settingsPath: string; + host: string; port: number; baseUrl: string; authToken: string; diff --git a/src/proxy/proxy-daemon.ts b/src/proxy/proxy-daemon.ts index 603c3097..8a28382e 100644 --- a/src/proxy/proxy-daemon.ts +++ b/src/proxy/proxy-daemon.ts @@ -225,10 +225,11 @@ export async function stopOpenAICompatProxy(): Promise<{ success: boolean; error export async function startOpenAICompatProxy( profile: OpenAICompatProfileConfig, - options: { port?: number; insecure?: boolean } = {} + options: { port?: number; host?: string; insecure?: boolean } = {} ): Promise { return withOpenAICompatProxyLock(async () => { const status = await getOpenAICompatProxyStatus(); + const host = options.host?.trim() || status.host || '127.0.0.1'; const port = typeof options.port === 'number' ? options.port @@ -245,7 +246,12 @@ export async function startOpenAICompatProxy( if (!Number.isInteger(port) || port < 1 || port > 65535) { return { success: false, port, error: `Invalid port: ${port}` }; } - if (status.running && status.profileName === profile.profileName && status.port === port) { + if ( + status.running && + status.profileName === profile.profileName && + status.port === port && + (status.host || '127.0.0.1') === host + ) { return { success: true, alreadyRunning: true, @@ -304,6 +310,8 @@ export async function startOpenAICompatProxy( daemonEntry, '--port', String(port), + '--host', + host, '--profile', profile.profileName, '--settings-path', @@ -323,6 +331,7 @@ export async function startOpenAICompatProxy( writeOpenAICompatProxySession({ profileName: profile.profileName, settingsPath: profile.settingsPath, + host, port, baseUrl: profile.baseUrl, authToken, diff --git a/src/proxy/proxy-env.ts b/src/proxy/proxy-env.ts index ca11813a..ed99a87c 100644 --- a/src/proxy/proxy-env.ts +++ b/src/proxy/proxy-env.ts @@ -4,11 +4,17 @@ export function buildOpenAICompatProxyEnv( profile: OpenAICompatProfileConfig, port: number, authToken: string, - claudeConfigDir?: string + claudeConfigDir?: string, + host = '127.0.0.1' ): Record { + const localBaseUrl = `http://${host}:${port}`; return { - ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, + ANTHROPIC_BASE_URL: localBaseUrl, ANTHROPIC_AUTH_TOKEN: authToken, + DISABLE_TELEMETRY: '1', + DISABLE_COST_WARNINGS: '1', + API_TIMEOUT_MS: '600000', + NO_PROXY: host === '127.0.0.1' ? '127.0.0.1,localhost' : `${host},127.0.0.1,localhost`, ...(profile.model ? { ANTHROPIC_MODEL: profile.model } : {}), ...(profile.opusModel ? { ANTHROPIC_DEFAULT_OPUS_MODEL: profile.opusModel } : {}), ...(profile.sonnetModel ? { ANTHROPIC_DEFAULT_SONNET_MODEL: profile.sonnetModel } : {}), diff --git a/src/proxy/request-router.ts b/src/proxy/request-router.ts new file mode 100644 index 00000000..aa758280 --- /dev/null +++ b/src/proxy/request-router.ts @@ -0,0 +1,222 @@ +import { loadConfigSafe, loadSettings } from '../utils/config-manager'; +import { expandPath } from '../utils/helpers'; +import type { ProxyOpenAIRequest } from './transformers/request-transformer'; +import { + loadOpenAICompatProxyRoutingConfig, + type OpenAICompatProxyRoutingConfig, +} from './routing-config'; +import { resolveOpenAICompatProfileConfig, type OpenAICompatProfileConfig } from './profile-router'; + +export type ProxyRoutingScenario = 'default' | 'background' | 'think' | 'longContext' | 'webSearch'; + +export interface ProxyRequestRoute { + profile: OpenAICompatProfileConfig; + model?: string; + scenario?: ProxyRoutingScenario; + estimatedTokens: number; + source: + | 'explicit-profile' + | 'scenario' + | 'profile-model-match' + | 'profile-name' + | 'request-model' + | 'active-default'; +} + +function loadOpenAICompatProfiles( + activeProfile: OpenAICompatProfileConfig +): OpenAICompatProfileConfig[] { + const config = loadConfigSafe(); + const profiles = [activeProfile]; + + for (const [profileName, settingsPath] of Object.entries(config.profiles)) { + if (profileName === activeProfile.profileName) { + continue; + } + + try { + const expandedPath = expandPath(settingsPath); + const settings = loadSettings(expandedPath); + const profile = resolveOpenAICompatProfileConfig( + profileName, + expandedPath, + settings.env || {} + ); + if (profile) { + profiles.push(profile); + } + } catch { + // Ignore invalid profiles while routing a live request. + } + } + + return profiles; +} + +function resolveSelectorTarget( + selector: string, + activeProfile: OpenAICompatProfileConfig, + profiles: OpenAICompatProfileConfig[] +): { profile: OpenAICompatProfileConfig; model?: string; explicitProfile: boolean } | null { + const trimmed = selector.trim(); + if (!trimmed) { + return null; + } + + const colonIndex = trimmed.indexOf(':'); + if (colonIndex > 0) { + const profileName = trimmed.slice(0, colonIndex).trim(); + const profile = profiles.find((candidate) => candidate.profileName === profileName); + if (profile) { + const model = trimmed.slice(colonIndex + 1).trim() || profile.model; + return { profile, model, explicitProfile: true }; + } + } + + const namedProfile = profiles.find((candidate) => candidate.profileName === trimmed); + if (namedProfile) { + return { profile: namedProfile, model: namedProfile.model, explicitProfile: false }; + } + + return { + profile: activeProfile, + model: trimmed, + explicitProfile: false, + }; +} + +function profileSupportsModel(profile: OpenAICompatProfileConfig, model: string): boolean { + return [profile.model, profile.opusModel, profile.sonnetModel, profile.haikuModel].some( + (candidate) => typeof candidate === 'string' && candidate === model + ); +} + +function estimateTokens(request: ProxyOpenAIRequest): number { + let characters = 0; + for (const message of request.messages) { + if (typeof message.content === 'string') { + characters += message.content.length; + continue; + } + if (Array.isArray(message.content)) { + for (const part of message.content) { + characters += part.type === 'text' ? part.text.length : part.image_url.url.length; + } + } + if (Array.isArray(message.tool_calls)) { + for (const toolCall of message.tool_calls) { + characters += toolCall.function.name.length + toolCall.function.arguments.length; + } + } + } + + if (Array.isArray(request.tools)) { + characters += JSON.stringify(request.tools).length; + } + + return Math.max(1, Math.ceil(characters / 4)); +} + +function detectScenario( + request: ProxyOpenAIRequest, + requestedModel: string | undefined, + routing: OpenAICompatProxyRoutingConfig +): { scenario?: ProxyRoutingScenario; selector?: string; estimatedTokens: number } { + const estimatedTokens = estimateTokens(request); + const hasWebSearchTool = + request.tools?.some((tool) => tool.function.name === 'web_search') === true; + const thinkingEnabled = + request.reasoning?.enabled === true || typeof request.reasoning_effort === 'string'; + const modelId = requestedModel || ''; + const longContextThreshold = routing.longContextThreshold ?? 60_000; + + if (hasWebSearchTool && routing.webSearch) { + return { scenario: 'webSearch', selector: routing.webSearch, estimatedTokens }; + } + if (thinkingEnabled && routing.think) { + return { scenario: 'think', selector: routing.think, estimatedTokens }; + } + if (estimatedTokens > longContextThreshold && routing.longContext) { + return { scenario: 'longContext', selector: routing.longContext, estimatedTokens }; + } + if (modelId.toLowerCase().includes('haiku') && routing.background) { + return { scenario: 'background', selector: routing.background, estimatedTokens }; + } + if (!requestedModel && routing.default) { + return { scenario: 'default', selector: routing.default, estimatedTokens }; + } + + return { estimatedTokens }; +} + +export function resolveProxyRequestRoute( + activeProfile: OpenAICompatProfileConfig, + request: ProxyOpenAIRequest +): ProxyRequestRoute { + const profiles = loadOpenAICompatProfiles(activeProfile); + const requestedModel = request.model?.trim() || undefined; + const explicitTarget = requestedModel + ? resolveSelectorTarget(requestedModel, activeProfile, profiles) + : null; + const routing = loadOpenAICompatProxyRoutingConfig(); + + if (explicitTarget?.explicitProfile) { + return { + profile: explicitTarget.profile, + model: explicitTarget.model, + estimatedTokens: estimateTokens(request), + source: 'explicit-profile', + }; + } + + const scenario = detectScenario(request, requestedModel, routing); + if (scenario.selector) { + const scenarioTarget = resolveSelectorTarget(scenario.selector, activeProfile, profiles); + if (scenarioTarget) { + return { + profile: scenarioTarget.profile, + model: scenarioTarget.model, + scenario: scenario.scenario, + estimatedTokens: scenario.estimatedTokens, + source: 'scenario', + }; + } + } + + if (explicitTarget && explicitTarget.profile.profileName !== activeProfile.profileName) { + return { + profile: explicitTarget.profile, + model: explicitTarget.model, + estimatedTokens: scenario.estimatedTokens, + source: 'profile-name', + }; + } + + if (requestedModel) { + const matchedProfile = profiles.find((profile) => + profileSupportsModel(profile, requestedModel) + ); + if (matchedProfile) { + return { + profile: matchedProfile, + model: requestedModel, + estimatedTokens: scenario.estimatedTokens, + source: 'profile-model-match', + }; + } + + return { + profile: activeProfile, + model: requestedModel, + estimatedTokens: scenario.estimatedTokens, + source: 'request-model', + }; + } + + return { + profile: activeProfile, + model: activeProfile.model, + estimatedTokens: scenario.estimatedTokens, + source: 'active-default', + }; +} diff --git a/src/proxy/routing-config.ts b/src/proxy/routing-config.ts new file mode 100644 index 00000000..f069edd6 --- /dev/null +++ b/src/proxy/routing-config.ts @@ -0,0 +1,74 @@ +import * as fs from 'fs'; +import * as yaml from 'js-yaml'; +import { getActiveConfigPath, getConfigPath } from '../utils/config-manager'; + +export interface OpenAICompatProxyRoutingConfig { + default?: string; + background?: string; + think?: string; + longContext?: string; + webSearch?: string; + longContextThreshold?: number; +} + +function readRawConfigObject(configPath: string): Record | null { + if (!fs.existsSync(configPath)) { + return null; + } + + const raw = fs.readFileSync(configPath, 'utf8'); + const parsed = + configPath.endsWith('.yaml') || configPath.endsWith('.yml') ? yaml.load(raw) : JSON.parse(raw); + return typeof parsed === 'object' && parsed !== null ? (parsed as Record) : null; +} + +function normalizeRoutingConfig(value: unknown): OpenAICompatProxyRoutingConfig { + if (typeof value !== 'object' || value === null) { + return {}; + } + + const config = value as Record; + return { + default: + typeof config.default === 'string' && config.default.trim() + ? config.default.trim() + : undefined, + background: + typeof config.background === 'string' && config.background.trim() + ? config.background.trim() + : undefined, + think: + typeof config.think === 'string' && config.think.trim() ? config.think.trim() : undefined, + longContext: + typeof config.longContext === 'string' && config.longContext.trim() + ? config.longContext.trim() + : undefined, + webSearch: + typeof config.webSearch === 'string' && config.webSearch.trim() + ? config.webSearch.trim() + : undefined, + longContextThreshold: + typeof config.longContextThreshold === 'number' && + Number.isFinite(config.longContextThreshold) + ? config.longContextThreshold + : undefined, + }; +} + +export function loadOpenAICompatProxyRoutingConfig(): OpenAICompatProxyRoutingConfig { + const activePath = getActiveConfigPath(); + const rawConfig = readRawConfigObject(activePath); + if (rawConfig?.proxy && typeof rawConfig.proxy === 'object') { + return normalizeRoutingConfig((rawConfig.proxy as Record).routing); + } + + const legacyPath = getConfigPath(); + if (legacyPath !== activePath) { + const legacyConfig = readRawConfigObject(legacyPath); + if (legacyConfig?.proxy && typeof legacyConfig.proxy === 'object') { + return normalizeRoutingConfig((legacyConfig.proxy as Record).routing); + } + } + + return {}; +} diff --git a/src/proxy/server/messages-route.ts b/src/proxy/server/messages-route.ts index 49a66585..ae0d33d9 100644 --- a/src/proxy/server/messages-route.ts +++ b/src/proxy/server/messages-route.ts @@ -1,12 +1,15 @@ import * as http from 'http'; import type { Dispatcher } from 'undici'; import type { OpenAICompatProfileConfig } from '../profile-router'; +import { resolveProxyRequestRoute } from '../request-router'; import { ProxyRequestTransformer } from '../transformers/request-transformer'; import { ProxySseStreamTransformer } from '../transformers/sse-stream-transformer'; import { resolveOpenAIChatCompletionsUrl } from '../upstream-url'; +import { createLogger } from '../../services/logging'; import { pipeWebResponseToNode, readJsonBody, writeJson } from './http-helpers'; -const REQUEST_TIMEOUT_MS = 30_000; +const REQUEST_TIMEOUT_MS = 600_000; +const logger = createLogger('proxy:openai-compat:messages'); class ProxyInputError extends Error { constructor(message: string) { @@ -23,7 +26,10 @@ function buildUpstreamHeaders(profile: OpenAICompatProfileConfig): Record } { let transformed; try { const transformer = new ProxyRequestTransformer(); @@ -32,12 +38,13 @@ function buildUpstreamBody(profile: OpenAICompatProfileConfig, rawBody: unknown) const message = error instanceof Error ? error.message : 'Invalid Anthropic request'; throw new ProxyInputError(message); } + const route = resolveProxyRequestRoute(profile, transformed); const body = { ...transformed, - model: transformed.model || profile.model, + model: route.model || route.profile.model, stream: transformed.stream === true, }; - return JSON.stringify(body); + return { body: JSON.stringify(body), route }; } export function extractIncomingProxyToken(headers: http.IncomingHttpHeaders): string | null { @@ -88,6 +95,20 @@ function buildFetchInit( return init; } +function getRequestTimeoutMs(): number { + const rawValue = process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS; + if (!rawValue) { + return REQUEST_TIMEOUT_MS; + } + + const parsed = Number.parseInt(rawValue, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : REQUEST_TIMEOUT_MS; +} + +function formatTimeoutDuration(timeoutMs: number): string { + return timeoutMs % 1000 === 0 ? `${timeoutMs / 1000} seconds` : `${timeoutMs}ms`; +} + export async function handleProxyMessagesRequest( req: http.IncomingMessage, res: http.ServerResponse, @@ -98,6 +119,9 @@ export async function handleProxyMessagesRequest( const transformer = new ProxySseStreamTransformer(); if (!validateIncomingProxyAuth(req.headers, expectedAuthToken)) { + logger.warn('auth.invalid', 'Rejected proxy message request with invalid auth token', { + remoteAddress: req.socket.remoteAddress || null, + }); await pipeWebResponseToNode( transformer.error(401, 'authentication_error', 'Missing or invalid local proxy token'), res @@ -105,34 +129,77 @@ export async function handleProxyMessagesRequest( return; } + let timeoutMs = REQUEST_TIMEOUT_MS; try { const rawBody = await readJsonBody(req); - const upstreamBody = buildUpstreamBody(profile, rawBody); + const upstream = buildUpstreamRequest(profile, rawBody); + logger.info('request.forward', 'Forwarding Anthropic request to OpenAI-compatible upstream', { + profileName: upstream.route.profile.profileName, + provider: upstream.route.profile.provider, + baseUrl: upstream.route.profile.baseUrl, + model: upstream.route.model || upstream.route.profile.model || null, + routeSource: upstream.route.source, + scenario: upstream.route.scenario || null, + estimatedTokens: upstream.route.estimatedTokens, + }); const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); - const abortOnDisconnect = () => { + timeoutMs = getRequestTimeoutMs(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const abortOnDisconnect = (source: string) => { if (!controller.signal.aborted && !res.writableEnded) { + logger.info( + 'request.disconnect', + 'Aborting upstream request after local client disconnect', + { + profileName: profile.profileName, + source, + } + ); controller.abort(); } }; - req.on('aborted', abortOnDisconnect); - req.on('close', abortOnDisconnect); - res.on('close', abortOnDisconnect); + req.on('aborted', () => abortOnDisconnect('req.aborted')); + req.on('close', () => abortOnDisconnect('req.close')); + req.socket?.on('close', () => abortOnDisconnect('req.socket.close')); + res.on('close', () => abortOnDisconnect('res.close')); + res.socket?.on('close', () => abortOnDisconnect('res.socket.close')); + const disconnectPoll = setInterval(() => { + if ( + req.destroyed || + res.destroyed || + req.socket?.destroyed === true || + res.socket?.destroyed === true + ) { + abortOnDisconnect('poll.destroyed'); + } + }, 50); try { const upstreamResponse = await fetch( - resolveOpenAIChatCompletionsUrl(profile.baseUrl), - buildFetchInit(profile, upstreamBody, controller.signal, insecureDispatcher) + resolveOpenAIChatCompletionsUrl(upstream.route.profile.baseUrl), + buildFetchInit(upstream.route.profile, upstream.body, controller.signal, insecureDispatcher) ); clearTimeout(timeout); + clearInterval(disconnectPoll); + logger.info('response.received', 'Received upstream response', { + profileName: profile.profileName, + routedProfileName: upstream.route.profile.profileName, + status: upstreamResponse.status, + }); const response = await transformer.transform(upstreamResponse); await pipeWebResponseToNode(response, res); } finally { clearTimeout(timeout); + clearInterval(disconnectPoll); } } catch (error) { const message = error instanceof Error ? error.message : 'Unknown proxy error'; + logger.error('request.failed', 'Proxy message request failed', { + profileName: profile.profileName, + error: message, + abort: error instanceof Error && error.name === 'AbortError', + }); const status = error instanceof Error && error.name === 'AbortError' ? 502 @@ -149,7 +216,7 @@ export async function handleProxyMessagesRequest( status, type, error instanceof Error && error.name === 'AbortError' - ? 'The upstream provider did not respond within 30 seconds' + ? `The upstream provider did not respond within ${formatTimeoutDuration(timeoutMs)}` : message ), res diff --git a/src/proxy/server/proxy-server.ts b/src/proxy/server/proxy-server.ts index b808d9ee..4b68f2a1 100644 --- a/src/proxy/server/proxy-server.ts +++ b/src/proxy/server/proxy-server.ts @@ -2,6 +2,7 @@ import * as http from 'http'; import { Agent } from 'undici'; import type { OpenAICompatProfileConfig } from '../profile-router'; import { OPENAI_COMPAT_PROXY_SERVICE_NAME } from '../proxy-daemon-paths'; +import { createLogger } from '../../services/logging'; import { handleProxyMessagesRequest, handleProxyModelsRequest, @@ -11,12 +12,19 @@ import { writeJson } from './http-helpers'; export interface OpenAICompatProxyServerOptions { profile: OpenAICompatProfileConfig; + host?: string; port: number; authToken: string; insecure?: boolean; } export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOptions): http.Server { + const host = options.host?.trim() || '127.0.0.1'; + const logger = createLogger('proxy:openai-compat', { + profileName: options.profile.profileName, + host, + port: options.port, + }); const insecureDispatcher = options.insecure ? new Agent({ connect: { rejectUnauthorized: false } }) : undefined; @@ -31,12 +39,31 @@ export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOpt writeJson(res, 200, { ok: true, service: OPENAI_COMPAT_PROXY_SERVICE_NAME, + host, profile: options.profile.profileName, port: options.port, }); return; } + if (method === 'GET' && pathname === '/') { + writeJson(res, 200, { + ok: true, + service: OPENAI_COMPAT_PROXY_SERVICE_NAME, + bind: { + host, + port: options.port, + }, + profile: { + name: options.profile.profileName, + provider: options.profile.provider, + model: options.profile.model || null, + }, + endpoints: ['/health', '/v1/messages', '/v1/models'], + }); + return; + } + if (method === 'GET' && pathname === '/v1/models') { if (!validateIncomingProxyAuth(req.headers, options.authToken)) { writeJson(res, 401, { @@ -63,13 +90,21 @@ export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOpt return; } + logger.warn('http.not_found', 'Rejected unknown proxy route', { + method, + pathname, + }); writeJson(res, 404, { error: 'Not found' }); }); + logger.info('server.start', 'OpenAI-compatible proxy server listening', { + baseUrl: `http://${host}:${options.port}`, + }); server.on('close', () => { + logger.info('server.stop', 'OpenAI-compatible proxy server stopped'); void insecureDispatcher?.close(); }); - server.listen(options.port, '127.0.0.1'); + server.listen(options.port, host); return server; } diff --git a/src/proxy/transformers/request-transformer.ts b/src/proxy/transformers/request-transformer.ts index 1804d426..6b294dac 100644 --- a/src/proxy/transformers/request-transformer.ts +++ b/src/proxy/transformers/request-transformer.ts @@ -1,20 +1,97 @@ -import { translateAnthropicRequest } from '../../cursor/cursor-anthropic-translator'; +interface AnthropicThinking { + type?: 'enabled' | 'disabled' | string; + budget_tokens?: number; +} + +interface AnthropicTextBlock { + type: 'text'; + text?: string; +} + +interface AnthropicImageBlock { + type: 'image'; + source?: { + type?: string; + media_type?: string; + data?: string; + }; +} + +interface AnthropicToolUseBlock { + type: 'tool_use'; + id?: string; + name?: string; + input?: Record; +} + +interface AnthropicToolResultBlock { + type: 'tool_result'; + tool_use_id?: string; + content?: unknown; +} + +type AnthropicContentBlock = + | AnthropicTextBlock + | AnthropicImageBlock + | AnthropicToolUseBlock + | AnthropicToolResultBlock + | { type: string; [key: string]: unknown }; + +interface AnthropicMessage { + role?: 'user' | 'assistant' | string; + content?: string | AnthropicContentBlock[]; +} interface AnthropicProxyRequestShape { + model?: unknown; + system?: unknown; + messages?: unknown; max_tokens?: unknown; temperature?: unknown; top_p?: unknown; stop_sequences?: unknown; metadata?: unknown; tools?: unknown; + stream?: unknown; + thinking?: AnthropicThinking; } -export interface ProxyOpenAIRequest extends ReturnType { - max_tokens?: number; - temperature?: number; - top_p?: number; - stop?: string[]; - metadata?: Record; +interface OpenAITextPart { + type: 'text'; + text: string; +} + +interface OpenAIImagePart { + type: 'image_url'; + image_url: { + url: string; + }; +} + +type OpenAIContentPart = OpenAITextPart | OpenAIImagePart; + +interface OpenAIMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string | OpenAIContentPart[] | null; + tool_call_id?: string; + tool_calls?: Array<{ + id: string; + type: 'function'; + function: { + name: string; + arguments: string; + }; + }>; +} + +export interface ProxyOpenAIRequest { + model?: string; + stream: boolean; + reasoning_effort?: string; + reasoning?: { + enabled: boolean; + effort: string; + }; tools?: Array<{ type: 'function'; function: { @@ -23,6 +100,23 @@ export interface ProxyOpenAIRequest extends ReturnType; }; }>; + messages: OpenAIMessage[]; + max_tokens?: number; + temperature?: number; + top_p?: number; + stop?: string[]; + metadata?: Record; +} + +const TOOL_RESULT_SERIALIZATION_FALLBACK = '[unserializable content]'; +const TOOL_USE_ARGUMENTS_FALLBACK = '{}'; + +function assertObject(value: unknown, label: string): Record { + if (typeof value !== 'object' || value === null) { + throw new Error(`${label} must be an object`); + } + + return value as Record; } function asNumber(value: unknown): number | undefined { @@ -46,6 +140,90 @@ function asMetadata(value: unknown): Record | undefined { : undefined; } +function safeJsonStringify(value: unknown, fallback: string): string { + try { + const serialized = JSON.stringify(value); + return typeof serialized === 'string' ? serialized : fallback; + } catch { + return fallback; + } +} + +function flattenTextContent(content: unknown, label: string): string { + if (typeof content === 'string') { + return content; + } + if (!Array.isArray(content)) { + throw new Error(`${label} must be a string or content block array`); + } + + return content + .map((block, index) => { + const parsed = assertObject(block, `${label}[${index}]`); + if (parsed.type !== 'text') { + throw new Error(`${label}[${index}].type "${String(parsed.type)}" is not supported`); + } + return typeof parsed.text === 'string' ? parsed.text : ''; + }) + .join('\n'); +} + +function toToolResultContent(content: unknown, label: string): string { + if (content === undefined) { + return ''; + } + if (typeof content === 'string') { + return content; + } + if (Array.isArray(content)) { + return flattenTextContent(content, label); + } + return safeJsonStringify(content, TOOL_RESULT_SERIALIZATION_FALLBACK); +} + +function createFallbackToolId(messageIndex: number, blockIndex: number): string { + return `toolu_proxy_fallback_${messageIndex}_${blockIndex}`; +} + +function toImagePart(block: AnthropicImageBlock, label: string): OpenAIImagePart { + const source = block.source; + if (!source || source.type !== 'base64' || !source.media_type || !source.data) { + throw new Error(`${label}.source must be a base64 image payload`); + } + + return { + type: 'image_url', + image_url: { + url: `data:${source.media_type};base64,${source.data}`, + }, + }; +} + +function isImageBlock(block: AnthropicContentBlock): block is AnthropicImageBlock { + return block.type === 'image'; +} + +function isToolUseBlock(block: AnthropicContentBlock): block is AnthropicToolUseBlock { + return block.type === 'tool_use'; +} + +function isToolResultBlock(block: AnthropicContentBlock): block is AnthropicToolResultBlock { + return block.type === 'tool_result'; +} + +function flushUserContent(messages: OpenAIMessage[], parts: OpenAIContentPart[]): void { + if (parts.length === 0) { + return; + } + + const onlyText = parts.every((part) => part.type === 'text'); + messages.push({ + role: 'user', + content: onlyText ? parts.map((part) => (part as OpenAITextPart).text).join('\n') : [...parts], + }); + parts.length = 0; +} + function transformTools(value: unknown): ProxyOpenAIRequest['tools'] { if (!Array.isArray(value)) { return undefined; @@ -71,19 +249,179 @@ function transformTools(value: unknown): ProxyOpenAIRequest['tools'] { return tools.length > 0 ? tools : undefined; } +function mapThinkingToReasoning( + thinking: AnthropicThinking | undefined +): Pick { + if (!thinking || thinking.type === 'disabled') { + return {}; + } + + if (thinking.type !== 'enabled') { + throw new Error('thinking.type must be "enabled" or "disabled"'); + } + + const effort = + typeof thinking.budget_tokens === 'number' && thinking.budget_tokens >= 8192 + ? 'high' + : 'medium'; + + return { + reasoning_effort: effort, + reasoning: { + enabled: true, + effort, + }, + }; +} + +function transformMessages(messagesValue: unknown): OpenAIMessage[] { + if (!Array.isArray(messagesValue)) { + throw new Error('messages must be an array'); + } + + const translatedMessages: OpenAIMessage[] = []; + + messagesValue.forEach((message, messageIndex) => { + const parsedMessage = assertObject(message, `messages[${messageIndex}]`) as AnthropicMessage; + const role = parsedMessage.role; + if (role !== 'user' && role !== 'assistant') { + throw new Error(`messages[${messageIndex}].role must be "user" or "assistant"`); + } + + const content = parsedMessage.content; + if (typeof content === 'string') { + translatedMessages.push({ role, content }); + return; + } + + if (!Array.isArray(content)) { + throw new Error(`messages[${messageIndex}].content must be a string or array`); + } + + const userParts: OpenAIContentPart[] = []; + const assistantTextParts: string[] = []; + const toolCalls: NonNullable = []; + let sawToolResult = false; + + content.forEach((block, blockIndex) => { + const parsed = assertObject( + block, + `messages[${messageIndex}].content[${blockIndex}]` + ) as AnthropicContentBlock; + + if (parsed.type === 'text') { + const text = typeof parsed.text === 'string' ? parsed.text : ''; + if (role === 'user') { + userParts.push({ type: 'text', text }); + } else { + assistantTextParts.push(text); + } + return; + } + + if (isImageBlock(parsed)) { + if (role !== 'user') { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}] image requires user role` + ); + } + userParts.push(toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)); + return; + } + + if (isToolUseBlock(parsed)) { + if (role !== 'assistant') { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}] tool_use requires assistant role` + ); + } + toolCalls.push({ + id: + typeof parsed.id === 'string' && parsed.id.length > 0 + ? parsed.id + : createFallbackToolId(messageIndex, blockIndex), + type: 'function', + function: { + name: typeof parsed.name === 'string' ? parsed.name : 'tool', + arguments: safeJsonStringify(parsed.input ?? {}, TOOL_USE_ARGUMENTS_FALLBACK), + }, + }); + return; + } + + if (isToolResultBlock(parsed)) { + if (role !== 'user') { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}] tool_result requires user role` + ); + } + if (typeof parsed.tool_use_id !== 'string' || parsed.tool_use_id.trim().length === 0) { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}].tool_use_id must be a non-empty string` + ); + } + sawToolResult = true; + flushUserContent(translatedMessages, userParts); + translatedMessages.push({ + role: 'tool', + tool_call_id: parsed.tool_use_id, + content: toToolResultContent( + parsed.content, + `messages[${messageIndex}].content[${blockIndex}].content` + ), + }); + return; + } + + throw new Error( + `messages[${messageIndex}].content[${blockIndex}].type "${String(parsed.type)}" is not supported` + ); + }); + + if (role === 'assistant') { + translatedMessages.push({ + role: 'assistant', + content: assistantTextParts.join('\n'), + tool_calls: toolCalls.length > 0 ? toolCalls : undefined, + }); + return; + } + + if (userParts.length > 0 || !sawToolResult) { + flushUserContent(translatedMessages, userParts); + } + }); + + return translatedMessages; +} + export class ProxyRequestTransformer { transform(raw: unknown): ProxyOpenAIRequest { - const translated = translateAnthropicRequest(raw); - const source = (raw || {}) as AnthropicProxyRequestShape; + const source = assertObject(raw || {}, 'request') as AnthropicProxyRequestShape; + const messages = transformMessages(source.messages); + const system = source.system; + const allMessages = + system !== undefined + ? [ + { role: 'system', content: flattenTextContent(system, 'system') } as OpenAIMessage, + ...messages, + ] + : messages; return { - ...translated, + model: + typeof source.model === 'string' && source.model.trim().length > 0 + ? source.model.trim() + : undefined, + stream: source.stream === true, + messages: allMessages, max_tokens: asNumber(source.max_tokens), temperature: asNumber(source.temperature), top_p: asNumber(source.top_p), stop: asStringArray(source.stop_sequences), metadata: asMetadata(source.metadata), tools: transformTools(source.tools), + ...mapThinkingToReasoning(source.thinking), }; } } diff --git a/src/types/config.ts b/src/types/config.ts index b84658e1..47de4f0d 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -40,6 +40,19 @@ export interface CLIProxyVariantsConfig { [profileName: string]: CLIProxyVariantConfig; } +export interface OpenAICompatProxyRoutingConfig { + default?: string; + background?: string; + think?: string; + longContext?: string; + webSearch?: string; + longContextThreshold?: number; +} + +export interface OpenAICompatProxyConfig { + routing?: OpenAICompatProxyRoutingConfig; +} + /** * Main CCS configuration * Located at: ~/.ccs/config.json @@ -51,6 +64,8 @@ export interface Config { profile_targets?: Record; /** User-defined CLIProxy profile variants (optional) */ cliproxy?: CLIProxyVariantsConfig; + /** OpenAI-compatible local proxy configuration (optional) */ + proxy?: OpenAICompatProxyConfig; /** Legacy continuity inheritance mapping (profile -> source account) */ continuity_inherit_from_account?: Record; } diff --git a/src/types/index.ts b/src/types/index.ts index f88f447d..934f18ae 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -14,6 +14,8 @@ export type { ProfilesRegistry, CLIProxyVariantConfig, CLIProxyVariantsConfig, + OpenAICompatProxyConfig, + OpenAICompatProxyRoutingConfig, } from './config'; export { isConfig, isSettings } from './config'; diff --git a/src/web-server/routes/config-routes.ts b/src/web-server/routes/config-routes.ts index 60b0cc44..3cfa87e6 100644 --- a/src/web-server/routes/config-routes.ts +++ b/src/web-server/routes/config-routes.ts @@ -673,6 +673,10 @@ router.put('/', (req: Request, res: Response): void => { currentConfig.cliproxy = mergeCliproxyConfig(currentConfig, config); } + if (config.proxy !== undefined) { + currentConfig.proxy = config.proxy; + } + if (config.preferences !== undefined) { currentConfig.preferences = config.preferences; } diff --git a/tests/e2e/proxy-command.e2e.test.ts b/tests/e2e/proxy-command.e2e.test.ts index 550ee8a2..021a0919 100644 --- a/tests/e2e/proxy-command.e2e.test.ts +++ b/tests/e2e/proxy-command.e2e.test.ts @@ -68,20 +68,42 @@ describe('proxy command e2e', () => { 'utf8' ); - const started = runCli(['proxy', 'start', 'hf', '--port', String(port)]); + const started = runCli(['proxy', 'start', 'hf', '--port', String(port), '--host', '127.0.0.1']); expect(started.status).toBe(0); const status = runCli(['proxy', 'status']); expect(status.stdout).toContain(`Proxy running on port ${port}`); + expect(status.stdout).toContain('Host: 127.0.0.1'); expect(status.stdout).toContain('Profile: hf'); const activate = runCli(['proxy', 'activate', '--shell', 'bash']); expect(activate.stdout).toContain(`export ANTHROPIC_BASE_URL='http://127.0.0.1:${port}'`); expect(activate.stdout).toMatch(/export ANTHROPIC_AUTH_TOKEN='[a-f0-9]{48}'/); + expect(activate.stdout).toContain("export DISABLE_TELEMETRY='1'"); + expect(activate.stdout).toContain("export DISABLE_COST_WARNINGS='1'"); + expect(activate.stdout).toContain("export API_TIMEOUT_MS='600000'"); + expect(activate.stdout).toContain("export NO_PROXY='127.0.0.1,localhost'"); + + const activateFish = runCli(['proxy', 'activate', '--fish']); + expect(activateFish.stdout).toContain(`set -gx ANTHROPIC_BASE_URL 'http://127.0.0.1:${port}'`); const health = await fetch(`http://127.0.0.1:${port}/health`); expect(health.status).toBe(200); + const info = await fetch(`http://127.0.0.1:${port}/`); + expect(info.status).toBe(200); + await expect(info.json()).resolves.toMatchObject({ + ok: true, + service: 'ccs-openai-compat-proxy', + bind: { + host: '127.0.0.1', + port, + }, + profile: { + name: 'hf', + }, + }); + const stopped = runCli(['proxy', 'stop']); expect(stopped.status).toBe(0); }, 35000); diff --git a/tests/integration/proxy/messages-edge-cases.test.ts b/tests/integration/proxy/messages-edge-cases.test.ts new file mode 100644 index 00000000..9b9898a8 --- /dev/null +++ b/tests/integration/proxy/messages-edge-cases.test.ts @@ -0,0 +1,259 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import getPort from 'get-port'; +import * as fs from 'fs'; +import * as http from 'http'; +import * as os from 'os'; +import * as path from 'path'; +import { startOpenAICompatProxyServer } from '../../../src/proxy/server/proxy-server'; +import type { OpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; + +let proxyServer: http.Server; +let upstreamServer: http.Server; +let upstreamSockets = new Set(); +let upstreamPort: number; +let proxyPort: number; +let tempDir: string; +let originalTimeoutEnv: string | undefined; +let originalCcsHome: string | undefined; + +async function startUpstream( + handler: (req: http.IncomingMessage, res: http.ServerResponse) => Promise | void +): Promise { + upstreamServer = http.createServer((req, res) => { + void Promise.resolve(handler(req, res)); + }); + upstreamServer.on('connection', (socket) => { + upstreamSockets.add(socket); + socket.on('close', () => { + upstreamSockets.delete(socket); + }); + }); + await new Promise((resolve) => + upstreamServer.listen(upstreamPort, '127.0.0.1', () => resolve()) + ); +} + +async function requestProxy(payload: unknown, signal?: AbortSignal): Promise { + return fetch(`http://127.0.0.1:${proxyPort}/v1/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': 'test-proxy-token', + }, + body: JSON.stringify(payload), + signal, + }); +} + +beforeEach(async () => { + upstreamPort = await getPort(); + proxyPort = await getPort(); + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-proxy-edge-')); + originalTimeoutEnv = process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS; + originalCcsHome = process.env.CCS_HOME; + process.env.CCS_HOME = tempDir; +}); + +afterEach(async () => { + if (originalTimeoutEnv !== undefined) { + process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS = originalTimeoutEnv; + } else { + delete process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS; + } + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + + if (proxyServer) { + await new Promise((resolve) => proxyServer.close(() => resolve())); + } + if (upstreamServer) { + for (const socket of upstreamSockets) { + socket.destroy(); + } + upstreamSockets = new Set(); + await new Promise((resolve) => upstreamServer.close(() => resolve())); + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('openai proxy message edge cases', () => { + async function startProxyWithHandler( + handler: (req: http.IncomingMessage, res: http.ServerResponse) => Promise | void + ) { + await startUpstream(handler); + const profile: OpenAICompatProfileConfig = { + profileName: 'hf', + settingsPath: '/tmp/hf.settings.json', + baseUrl: `http://127.0.0.1:${upstreamPort}`, + apiKey: 'hf_token', + provider: 'generic-chat-completion-api', + model: 'hf-model', + }; + proxyServer = startOpenAICompatProxyServer({ + profile, + port: proxyPort, + authToken: 'test-proxy-token', + }); + } + + it('preserves rate-limit errors from the upstream provider', async () => { + await startProxyWithHandler((_req, res) => { + res.writeHead(429, { + 'Content-Type': 'application/json', + 'Retry-After': '9', + }); + res.end(JSON.stringify({ error: { message: 'rate limited' } })); + }); + + const response = await requestProxy({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(response.status).toBe(429); + expect(response.headers.get('retry-after')).toBe('9'); + await expect(response.json()).resolves.toMatchObject({ + type: 'error', + error: { + type: 'rate_limit_error', + message: 'rate limited', + }, + }); + }); + + it('returns api_error when the upstream JSON response has no usable choices', async () => { + await startProxyWithHandler((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ id: 'chatcmpl_empty', choices: [] })); + }); + + const response = await requestProxy({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toMatchObject({ + type: 'error', + error: { + type: 'api_error', + message: 'Failed to translate Cursor JSON response', + }, + }); + }); + + it('streams thinking deltas and chunked tool-call arguments back as Anthropic SSE', async () => { + await startProxyWithHandler((_req, res) => { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant"}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"reasoning_content":"Need to search first."}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search","arguments":"{\\"q\\":\\""}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"docs\\"}"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":8,"completion_tokens":6}}\n\n' + ); + res.end('data: [DONE]\n\n'); + }); + + const response = await requestProxy({ + model: 'hf-model', + stream: true, + messages: [{ role: 'user', content: 'search docs' }], + }); + + const body = await response.text(); + expect(response.status).toBe(200); + expect(body).toContain('"type":"thinking_delta"'); + expect(body).toContain('"type":"tool_use"'); + expect(body).toContain('"partial_json":"{\\"q\\":\\""'); + expect(body).toContain('"partial_json":"docs\\"}"'); + expect(body).toContain('event: message_stop'); + }); + + it('returns a timeout error when the upstream does not respond in time', async () => { + process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS = '50'; + await startProxyWithHandler(async (req, _res) => { + await new Promise((resolve) => req.on('close', () => resolve())); + }); + + const response = await requestProxy({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toMatchObject({ + type: 'error', + error: { + type: 'api_error', + message: 'The upstream provider did not respond within 50ms', + }, + }); + }); + + it('aborts the upstream request when the client disconnects mid-flight', async () => { + await startProxyWithHandler(() => {}); + + await new Promise((resolve) => { + const request = http.request( + { + hostname: '127.0.0.1', + port: proxyPort, + path: '/v1/messages', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': 'test-proxy-token', + }, + }, + () => resolve() + ); + + request.write( + JSON.stringify({ + model: 'hf-model', + messages: [{ role: 'user', content: 'hello' }], + }) + ); + request.end(); + + setTimeout(() => { + request.destroy(new Error('client aborted')); + resolve(); + }, 50); + }); + + const logPath = path.join(tempDir, '.ccs', 'logs', 'current.jsonl'); + await Promise.race([ + new Promise((resolve, reject) => { + const startedAt = Date.now(); + const timer = setInterval(() => { + if (fs.existsSync(logPath)) { + const content = fs.readFileSync(logPath, 'utf8'); + if (content.includes('"event":"request.disconnect"')) { + clearInterval(timer); + resolve(); + return; + } + } + + if (Date.now() - startedAt > 1500) { + clearInterval(timer); + reject(new Error('proxy did not log disconnect cleanup')); + } + }, 50); + }), + ]); + }); +}); diff --git a/tests/integration/proxy/messages-endpoint.test.ts b/tests/integration/proxy/messages-endpoint.test.ts index ee96ce4c..d6ce7469 100644 --- a/tests/integration/proxy/messages-endpoint.test.ts +++ b/tests/integration/proxy/messages-endpoint.test.ts @@ -173,16 +173,44 @@ describe('openai proxy messages endpoint', () => { expect(body.error?.type).toBe('authentication_error'); }); - it('returns invalid_request_error for unsupported Anthropic content blocks', async () => { + it('translates supported Anthropic image blocks into OpenAI image_url content', async () => { const response = await requestProxy({ model: 'hf-model', - messages: [{ role: 'user', content: [{ type: 'image' }] }], + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'ZmFrZQ==', + }, + }, + ], + }, + ], }); - const body = (await response.json()) as { error?: { type?: string; message?: string } }; - expect(response.status).toBe(400); - expect(body.error?.type).toBe('invalid_request_error'); - expect(body.error?.message).toContain('is not supported'); + expect(response.status).toBe(200); + const parsedUpstream = upstreamBody as { + messages?: Array<{ + role: string; + content: Array<{ type: string; text?: string; image_url?: { url?: string } }>; + }>; + }; + expect(parsedUpstream.messages?.[0]).toEqual({ + role: 'user', + content: [ + { type: 'text', text: 'Describe this' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,ZmFrZQ==' }, + }, + ], + }); }); it('accepts query strings and trailing slashes on the messages route', async () => { diff --git a/tests/integration/proxy/request-routing.test.ts b/tests/integration/proxy/request-routing.test.ts new file mode 100644 index 00000000..d5a8ce0d --- /dev/null +++ b/tests/integration/proxy/request-routing.test.ts @@ -0,0 +1,217 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import getPort from 'get-port'; +import * as fs from 'fs'; +import * as http from 'http'; +import * as os from 'os'; +import * as path from 'path'; +import { startOpenAICompatProxyServer } from '../../../src/proxy/server/proxy-server'; +import type { OpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; + +let originalCcsHome: string | undefined; +let tempDir: string; +let proxyServer: http.Server; +let upstreamServers: http.Server[] = []; +let proxyPort: number; + +function startMockUpstream( + port: number, + hitLabel: string, + hits: string[], + bodies: Array<{ label: string; body: unknown }> +): Promise { + return new Promise((resolve) => { + const server = http.createServer(async (req, res) => { + let body = ''; + for await (const chunk of req) { + body += chunk.toString(); + } + hits.push(hitLabel); + bodies.push({ label: hitLabel, body: JSON.parse(body) }); + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + id: `chatcmpl_${hitLabel}`, + model: hitLabel, + choices: [ + { + index: 0, + message: { role: 'assistant', content: `Reply from ${hitLabel}` }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 2, completion_tokens: 3 }, + }) + ); + }); + upstreamServers.push(server); + server.listen(port, '127.0.0.1', () => resolve()); + }); +} + +function writeSettings(profileName: string, env: Record): string { + const settingsPath = path.join(tempDir, '.ccs', `${profileName}.settings.json`); + fs.writeFileSync(settingsPath, JSON.stringify({ env }, null, 2), 'utf8'); + return settingsPath; +} + +async function requestProxy(payload: unknown): Promise { + return fetch(`http://127.0.0.1:${proxyPort}/v1/messages`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': 'test-proxy-token', + }, + body: JSON.stringify(payload), + }); +} + +beforeEach(async () => { + originalCcsHome = process.env.CCS_HOME; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-proxy-routing-')); + fs.mkdirSync(path.join(tempDir, '.ccs'), { recursive: true }); + process.env.CCS_HOME = tempDir; + proxyPort = await getPort(); +}); + +afterEach(async () => { + await Promise.all( + upstreamServers.map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + }) + ) + ); + upstreamServers = []; + if (proxyServer) { + await new Promise((resolve) => proxyServer.close(() => resolve())); + } + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('openai proxy request routing', () => { + it('routes explicit profile:model selectors to the matching upstream profile', async () => { + const primaryPort = await getPort(); + const secondaryPort = await getPort(); + const hits: string[] = []; + const bodies: Array<{ label: string; body: unknown }> = []; + await startMockUpstream(primaryPort, 'primary', hits, bodies); + await startMockUpstream(secondaryPort, 'secondary', hits, bodies); + + const primarySettings = writeSettings('hf', { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${primaryPort}`, + ANTHROPIC_AUTH_TOKEN: 'hf_token', + ANTHROPIC_MODEL: 'hf-default', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + const secondarySettings = writeSettings('deepseek', { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${secondaryPort}`, + ANTHROPIC_AUTH_TOKEN: 'deepseek_token', + ANTHROPIC_MODEL: 'deepseek-chat', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + + fs.writeFileSync( + path.join(tempDir, '.ccs', 'config.json'), + JSON.stringify({ profiles: { hf: primarySettings, deepseek: secondarySettings } }, null, 2), + 'utf8' + ); + + const profile: OpenAICompatProfileConfig = { + profileName: 'hf', + settingsPath: primarySettings, + baseUrl: `http://127.0.0.1:${primaryPort}`, + apiKey: 'hf_token', + provider: 'generic-chat-completion-api', + model: 'hf-default', + }; + proxyServer = startOpenAICompatProxyServer({ + profile, + port: proxyPort, + authToken: 'test-proxy-token', + }); + + const response = await requestProxy({ + model: 'deepseek:deepseek-reasoner', + messages: [{ role: 'user', content: 'hello' }], + }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + content: [{ type: 'text', text: 'Reply from secondary' }], + }); + expect(hits).toEqual(['secondary']); + expect(bodies[0]?.body).toMatchObject({ model: 'deepseek-reasoner' }); + }); + + it('routes thinking requests through the configured think scenario', async () => { + const primaryPort = await getPort(); + const thinkPort = await getPort(); + const hits: string[] = []; + const bodies: Array<{ label: string; body: unknown }> = []; + await startMockUpstream(primaryPort, 'primary', hits, bodies); + await startMockUpstream(thinkPort, 'thinker', hits, bodies); + + const primarySettings = writeSettings('hf', { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${primaryPort}`, + ANTHROPIC_AUTH_TOKEN: 'hf_token', + ANTHROPIC_MODEL: 'hf-default', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + const thinkSettings = writeSettings('thinker', { + ANTHROPIC_BASE_URL: `http://127.0.0.1:${thinkPort}`, + ANTHROPIC_AUTH_TOKEN: 'think_token', + ANTHROPIC_MODEL: 'deepseek-reasoner', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }); + + fs.writeFileSync( + path.join(tempDir, '.ccs', 'config.json'), + JSON.stringify( + { + profiles: { hf: primarySettings, thinker: thinkSettings }, + proxy: { + routing: { + think: 'thinker:deepseek-reasoner', + }, + }, + }, + null, + 2 + ), + 'utf8' + ); + + const profile: OpenAICompatProfileConfig = { + profileName: 'hf', + settingsPath: primarySettings, + baseUrl: `http://127.0.0.1:${primaryPort}`, + apiKey: 'hf_token', + provider: 'generic-chat-completion-api', + model: 'hf-default', + }; + proxyServer = startOpenAICompatProxyServer({ + profile, + port: proxyPort, + authToken: 'test-proxy-token', + }); + + const response = await requestProxy({ + model: 'hf-default', + thinking: { type: 'enabled', budget_tokens: 9000 }, + messages: [{ role: 'user', content: 'think hard' }], + }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + content: [{ type: 'text', text: 'Reply from thinker' }], + }); + expect(hits).toEqual(['thinker']); + expect(bodies[0]?.body).toMatchObject({ model: 'deepseek-reasoner' }); + }); +}); diff --git a/tests/unit/proxy/request-router.test.ts b/tests/unit/proxy/request-router.test.ts new file mode 100644 index 00000000..eb780493 --- /dev/null +++ b/tests/unit/proxy/request-router.test.ts @@ -0,0 +1,194 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { resolveProxyRequestRoute } from '../../../src/proxy/request-router'; +import { resolveOpenAICompatProfileConfig } from '../../../src/proxy/profile-router'; +import { loadSettings } from '../../../src/utils/config-manager'; + +let originalCcsHome: string | undefined; +let tempDir: string; + +function writeSettings(profileName: string, env: Record): string { + const settingsPath = path.join(tempDir, '.ccs', `${profileName}.settings.json`); + fs.writeFileSync(settingsPath, JSON.stringify({ env }, null, 2), 'utf8'); + return settingsPath; +} + +function buildProfile(profileName: string) { + const settingsPath = path.join(tempDir, '.ccs', `${profileName}.settings.json`); + const profile = resolveOpenAICompatProfileConfig( + profileName, + settingsPath, + loadSettings(settingsPath).env || {} + ); + expect(profile).toBeTruthy(); + return profile!; +} + +beforeEach(() => { + originalCcsHome = process.env.CCS_HOME; + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-proxy-router-')); + fs.mkdirSync(path.join(tempDir, '.ccs'), { recursive: true }); +}); + +afterEach(() => { + if (originalCcsHome !== undefined) { + process.env.CCS_HOME = originalCcsHome; + } else { + delete process.env.CCS_HOME; + } + fs.rmSync(tempDir, { recursive: true, force: true }); +}); + +describe('resolveProxyRequestRoute', () => { + beforeEach(() => { + process.env.CCS_HOME = tempDir; + + const profiles = { + hf: writeSettings('hf', { + ANTHROPIC_BASE_URL: 'https://router.huggingface.co/v1', + ANTHROPIC_AUTH_TOKEN: 'hf_token', + ANTHROPIC_MODEL: 'hf-default', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + deepseek: writeSettings('deepseek', { + ANTHROPIC_BASE_URL: 'https://api.deepseek.com/v1', + ANTHROPIC_AUTH_TOKEN: 'deepseek_token', + ANTHROPIC_MODEL: 'deepseek-chat', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + thinker: writeSettings('thinker', { + ANTHROPIC_BASE_URL: 'https://thinking.example.com/v1', + ANTHROPIC_AUTH_TOKEN: 'think_token', + ANTHROPIC_MODEL: 'deepseek-reasoner', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + search: writeSettings('search', { + ANTHROPIC_BASE_URL: 'https://search.example.com/v1', + ANTHROPIC_AUTH_TOKEN: 'search_token', + ANTHROPIC_MODEL: 'sonar-pro', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + background: writeSettings('background', { + ANTHROPIC_BASE_URL: 'https://background.example.com/v1', + ANTHROPIC_AUTH_TOKEN: 'background_token', + ANTHROPIC_MODEL: 'qwen-small', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + long: writeSettings('long', { + ANTHROPIC_BASE_URL: 'https://long.example.com/v1', + ANTHROPIC_AUTH_TOKEN: 'long_token', + ANTHROPIC_MODEL: 'gemini-2.5-pro', + CCS_DROID_PROVIDER: 'generic-chat-completion-api', + }), + }; + + fs.writeFileSync( + path.join(tempDir, '.ccs', 'config.json'), + JSON.stringify( + { + profiles, + proxy: { + routing: { + default: 'hf:hf-default', + background: 'background:qwen-small', + think: 'thinker:deepseek-reasoner', + longContext: 'long:gemini-2.5-pro', + longContextThreshold: 10, + webSearch: 'search:sonar-pro', + }, + }, + }, + null, + 2 + ), + 'utf8' + ); + }); + + it('uses explicit profile:model selectors', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'deepseek:deepseek-reasoner', + stream: true, + messages: [{ role: 'user', content: 'plan this' }], + }); + + expect(route.profile.profileName).toBe('deepseek'); + expect(route.model).toBe('deepseek-reasoner'); + expect(route.source).toBe('explicit-profile'); + }); + + it('matches plain model ids to the profile that owns them', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'deepseek-chat', + stream: true, + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(route.profile.profileName).toBe('deepseek'); + expect(route.model).toBe('deepseek-chat'); + expect(route.source).toBe('profile-model-match'); + }); + + it('routes thinking requests through the configured think scenario', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'hf-default', + stream: true, + reasoning_effort: 'high', + reasoning: { enabled: true, effort: 'high' }, + messages: [{ role: 'user', content: 'work this out carefully' }], + }); + + expect(route.profile.profileName).toBe('thinker'); + expect(route.model).toBe('deepseek-reasoner'); + expect(route.scenario).toBe('think'); + expect(route.source).toBe('scenario'); + }); + + it('routes long-context requests through the configured longContext scenario', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'hf-default', + stream: true, + messages: [{ role: 'user', content: 'x'.repeat(120) }], + }); + + expect(route.profile.profileName).toBe('long'); + expect(route.model).toBe('gemini-2.5-pro'); + expect(route.scenario).toBe('longContext'); + expect(route.estimatedTokens).toBeGreaterThan(10); + }); + + it('routes web_search tool requests through the configured webSearch scenario', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'hf-default', + stream: true, + messages: [{ role: 'user', content: 'search the docs' }], + tools: [ + { + type: 'function', + function: { + name: 'web_search', + parameters: { type: 'object' }, + }, + }, + ], + }); + + expect(route.profile.profileName).toBe('search'); + expect(route.model).toBe('sonar-pro'); + expect(route.scenario).toBe('webSearch'); + }); + + it('routes haiku requests through the configured background scenario', () => { + const route = resolveProxyRequestRoute(buildProfile('hf'), { + model: 'claude-3-haiku', + stream: true, + messages: [{ role: 'user', content: 'run this in the background' }], + }); + + expect(route.profile.profileName).toBe('background'); + expect(route.model).toBe('qwen-small'); + expect(route.scenario).toBe('background'); + }); +}); diff --git a/tests/unit/proxy/transformers/request-transformer.test.ts b/tests/unit/proxy/transformers/request-transformer.test.ts index e3d6afad..25687c20 100644 --- a/tests/unit/proxy/transformers/request-transformer.test.ts +++ b/tests/unit/proxy/transformers/request-transformer.test.ts @@ -28,6 +28,7 @@ describe('ProxyRequestTransformer', () => { expect(result.stream).toBe(true); expect(result.reasoning_effort).toBe('high'); + expect(result.reasoning).toEqual({ enabled: true, effort: 'high' }); expect(result.max_tokens).toBe(1024); expect(result.temperature).toBe(0.2); expect(result.top_p).toBe(0.9); @@ -42,6 +43,43 @@ describe('ProxyRequestTransformer', () => { }); }); + it('translates base64 image blocks into OpenAI image_url parts', () => { + const transformer = new ProxyRequestTransformer(); + const result = transformer.transform({ + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this image' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'ZmFrZS1pbWFnZS1ieXRlcw==', + }, + }, + ], + }, + ], + }); + + expect(result.messages).toEqual([ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this image' }, + { + type: 'image_url', + image_url: { + url: 'data:image/png;base64,ZmFrZS1pbWFnZS1ieXRlcw==', + }, + }, + ], + }, + ]); + }); + it('drops malformed optional fields but preserves the translated core request', () => { const transformer = new ProxyRequestTransformer(); const result = transformer.transform({ From a3407093d70bef44874cdf26a06436e0d287c1a7 Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Wed, 15 Apr 2026 01:44:01 -0400 Subject: [PATCH 7/8] refactor(proxy): internalize sse translation and cleanup handlers --- src/cursor/cursor-anthropic-response.ts | 253 +---------------- src/glmt/sse-parser.ts | 2 +- src/proxy/server/messages-route.ts | 85 ++++-- .../transformers/sse-stream-transformer.ts | 254 +++++++++++++++++- tests/unit/glmt/sse-parser.test.ts | 18 +- tests/unit/proxy/messages-route.test.ts | 72 +++++ .../sse-stream-transformer.test.ts | 70 +++++ 7 files changed, 476 insertions(+), 278 deletions(-) create mode 100644 tests/unit/proxy/messages-route.test.ts create mode 100644 tests/unit/proxy/transformers/sse-stream-transformer.test.ts diff --git a/src/cursor/cursor-anthropic-response.ts b/src/cursor/cursor-anthropic-response.ts index 6575fd14..c5c5a0e2 100644 --- a/src/cursor/cursor-anthropic-response.ts +++ b/src/cursor/cursor-anthropic-response.ts @@ -1,249 +1,4 @@ -import { DeltaAccumulator } from '../glmt/delta-accumulator'; -import { GlmtTransformer } from '../glmt/glmt-transformer'; -import { SSEParser } from '../glmt/sse-parser'; -import type { OpenAIResponse, SSEEvent } from '../glmt/pipeline'; - -const JSON_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor JSON response'; -const STREAM_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor SSE response'; -type ResponseHeaders = Headers | Record | Array<[string, string]>; - -interface AnthropicErrorPayload { - type: 'error'; - error: { - type: string; - message: string; - }; -} - -function createAnthropicErrorPayload(type: string, message: string): AnthropicErrorPayload { - return { - type: 'error', - error: { - type, - message, - }, - }; -} - -function formatErrorForLog(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - - try { - return JSON.stringify(error); - } catch { - return String(error); - } -} - -function logTranslationError(context: string, error: unknown): void { - console.error(`[cursor-anthropic-response] ${context}: ${formatErrorForLog(error)}`); -} - -export function createAnthropicErrorResponse( - status: number, - type: string, - message: string, - headers?: ResponseHeaders -): Response { - const responseHeaders = new Headers(headers); - responseHeaders.set('Content-Type', 'application/json'); - responseHeaders.delete('Content-Length'); - - return new Response(JSON.stringify(createAnthropicErrorPayload(type, message)), { - status, - headers: responseHeaders, - }); -} - -function formatSseEvent(event: string, data: unknown): string { - return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; -} - -function hasTranslatableChoices(value: unknown): value is OpenAIResponse { - if (typeof value !== 'object' || value === null) { - return false; - } - - const { choices } = value as OpenAIResponse; - if (!Array.isArray(choices) || choices.length === 0) { - return false; - } - - const firstChoice = choices[0]; - if (typeof firstChoice !== 'object' || firstChoice === null) { - return false; - } - - const message = (firstChoice as { message?: unknown }).message; - return typeof message === 'object' && message !== null; -} - -function isSyntheticTransformationFallback(value: unknown): boolean { - return ( - typeof value === 'object' && - value !== null && - typeof (value as { id?: unknown }).id === 'string' && - (value as { id: string }).id.startsWith('msg_error_') - ); -} - -async function createAnthropicErrorProxyResponse(response: Response): Promise { - const headers = new Headers(response.headers); - headers.delete('Content-Type'); - headers.delete('Content-Length'); - - let type = - response.status === 401 - ? 'authentication_error' - : response.status === 429 - ? 'rate_limit_error' - : response.status >= 400 && response.status < 500 - ? 'invalid_request_error' - : 'api_error'; - let message = `Cursor request failed with status ${response.status}`; - - try { - const contentType = (response.headers.get('content-type') || '').toLowerCase(); - if (contentType.includes('application/json')) { - const payload = (await response.json()) as { - error?: { type?: string; message?: string }; - message?: string; - }; - - if (typeof payload?.error?.type === 'string' && payload.error.type.trim().length > 0) { - type = payload.error.type; - } - - if (typeof payload?.error?.message === 'string' && payload.error.message.trim().length > 0) { - message = payload.error.message; - } else if (typeof payload?.message === 'string' && payload.message.trim().length > 0) { - message = payload.message; - } - } else { - const text = (await response.text()).trim(); - if (text.length > 0) { - message = text; - } - } - } catch (error) { - logTranslationError('Failed to parse Cursor error response', error); - } - - return createAnthropicErrorResponse(response.status, type, message, headers); -} - -async function createAnthropicJsonResponse(response: Response): Promise { - try { - const openAiResponse = await response.json(); - if (!hasTranslatableChoices(openAiResponse)) { - return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); - } - - const anthropicResponse = new GlmtTransformer().transformResponse(openAiResponse); - if (isSyntheticTransformationFallback(anthropicResponse)) { - logTranslationError( - 'Cursor JSON translation produced synthetic fallback response', - anthropicResponse - ); - return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); - } - - return new Response(JSON.stringify(anthropicResponse), { - status: response.status, - headers: { 'Content-Type': 'application/json' }, - }); - } catch (error) { - logTranslationError('Cursor JSON translation failed', error); - return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); - } -} - -function createAnthropicStreamingResponse(response: Response): Response { - const body = response.body; - if (!body) { - return createAnthropicErrorResponse( - 502, - 'api_error', - 'Cursor stream ended before a response body was available' - ); - } - - const parser = new SSEParser({ throwOnMalformedJson: true }); - const transformer = new GlmtTransformer(); - const accumulator = new DeltaAccumulator({}); - const encoder = new TextEncoder(); - - const readable = new ReadableStream({ - async start(controller) { - const reader = body.getReader(); - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - if (!value) { - continue; - } - - const events = parser.parse(Buffer.from(value)); - events.forEach((event) => { - const anthropicEvents = transformer.transformDelta(event as SSEEvent, accumulator); - anthropicEvents.forEach((anthropicEvent) => { - controller.enqueue( - encoder.encode(formatSseEvent(anthropicEvent.event, anthropicEvent.data)) - ); - }); - }); - } - - if (!accumulator.isFinalized() && accumulator.isMessageStarted()) { - transformer.finalizeDelta(accumulator).forEach((anthropicEvent) => { - controller.enqueue( - encoder.encode(formatSseEvent(anthropicEvent.event, anthropicEvent.data)) - ); - }); - } - } catch (error) { - logTranslationError('Cursor SSE translation failed', error); - controller.enqueue( - encoder.encode( - formatSseEvent( - 'error', - createAnthropicErrorPayload('api_error', STREAM_TRANSLATION_ERROR_MESSAGE) - ) - ) - ); - } finally { - reader.releaseLock(); - controller.close(); - } - }, - }); - - return new Response(readable, { - status: response.status, - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }); -} - -export async function createAnthropicProxyResponse(response: Response): Promise { - if (!response.ok) { - return createAnthropicErrorProxyResponse(response); - } - - const contentType = (response.headers.get('content-type') || '').toLowerCase(); - const isEventStream = - contentType === 'text/event-stream' || contentType.startsWith('text/event-stream;'); - - return isEventStream - ? createAnthropicStreamingResponse(response) - : createAnthropicJsonResponse(response); -} +export { + createAnthropicErrorResponse, + createAnthropicProxyResponse, +} from '../proxy/transformers/sse-stream-transformer'; diff --git a/src/glmt/sse-parser.ts b/src/glmt/sse-parser.ts index c161a7df..d6fff5d6 100644 --- a/src/glmt/sse-parser.ts +++ b/src/glmt/sse-parser.ts @@ -49,7 +49,7 @@ export class SSEParser { * @returns Array of parsed events */ parse(chunk: Buffer | string): SSEEvent[] { - this.buffer += chunk.toString().replace(/\r\n/g, '\n'); + this.buffer += chunk.toString().replace(/\r\n?/g, '\n'); // C-01 Fix: Prevent unbounded buffer growth (DoS protection) if (this.buffer.length > this.maxBufferSize) { diff --git a/src/proxy/server/messages-route.ts b/src/proxy/server/messages-route.ts index ae0d33d9..13c329e8 100644 --- a/src/proxy/server/messages-route.ts +++ b/src/proxy/server/messages-route.ts @@ -109,6 +109,61 @@ function formatTimeoutDuration(timeoutMs: number): string { return timeoutMs % 1000 === 0 ? `${timeoutMs / 1000} seconds` : `${timeoutMs}ms`; } +function registerOnceListener( + emitter: NodeJS.EventEmitter | null | undefined, + event: string, + handler: () => void +): () => void { + if (!emitter) { + return () => {}; + } + + emitter.once(event, handler); + return () => { + emitter.removeListener(event, handler); + }; +} + +export function attachDisconnectAbortHandlers( + req: http.IncomingMessage, + res: http.ServerResponse, + controller: AbortController, + onDisconnect: (source: string) => void +): () => void { + const abortOnDisconnect = (source: string) => { + if (!controller.signal.aborted && !res.writableEnded) { + onDisconnect(source); + controller.abort(); + } + }; + + const cleanupFns = [ + registerOnceListener(req, 'aborted', () => abortOnDisconnect('req.aborted')), + registerOnceListener(req, 'close', () => abortOnDisconnect('req.close')), + registerOnceListener(req.socket, 'close', () => abortOnDisconnect('req.socket.close')), + registerOnceListener(res, 'close', () => abortOnDisconnect('res.close')), + registerOnceListener(res.socket, 'close', () => abortOnDisconnect('res.socket.close')), + ]; + + const disconnectPoll = setInterval(() => { + if ( + req.destroyed || + res.destroyed || + req.socket?.destroyed === true || + res.socket?.destroyed === true + ) { + abortOnDisconnect('poll.destroyed'); + } + }, 50); + + return () => { + clearInterval(disconnectPoll); + for (const cleanup of cleanupFns) { + cleanup(); + } + }; +} + export async function handleProxyMessagesRequest( req: http.IncomingMessage, res: http.ServerResponse, @@ -145,8 +200,11 @@ export async function handleProxyMessagesRequest( const controller = new AbortController(); timeoutMs = getRequestTimeoutMs(); const timeout = setTimeout(() => controller.abort(), timeoutMs); - const abortOnDisconnect = (source: string) => { - if (!controller.signal.aborted && !res.writableEnded) { + const cleanupDisconnectHandlers = attachDisconnectAbortHandlers( + req, + res, + controller, + (source) => { logger.info( 'request.disconnect', 'Aborting upstream request after local client disconnect', @@ -155,25 +213,8 @@ export async function handleProxyMessagesRequest( source, } ); - controller.abort(); } - }; - - req.on('aborted', () => abortOnDisconnect('req.aborted')); - req.on('close', () => abortOnDisconnect('req.close')); - req.socket?.on('close', () => abortOnDisconnect('req.socket.close')); - res.on('close', () => abortOnDisconnect('res.close')); - res.socket?.on('close', () => abortOnDisconnect('res.socket.close')); - const disconnectPoll = setInterval(() => { - if ( - req.destroyed || - res.destroyed || - req.socket?.destroyed === true || - res.socket?.destroyed === true - ) { - abortOnDisconnect('poll.destroyed'); - } - }, 50); + ); try { const upstreamResponse = await fetch( @@ -181,7 +222,7 @@ export async function handleProxyMessagesRequest( buildFetchInit(upstream.route.profile, upstream.body, controller.signal, insecureDispatcher) ); clearTimeout(timeout); - clearInterval(disconnectPoll); + cleanupDisconnectHandlers(); logger.info('response.received', 'Received upstream response', { profileName: profile.profileName, routedProfileName: upstream.route.profile.profileName, @@ -191,7 +232,7 @@ export async function handleProxyMessagesRequest( await pipeWebResponseToNode(response, res); } finally { clearTimeout(timeout); - clearInterval(disconnectPoll); + cleanupDisconnectHandlers(); } } catch (error) { const message = error instanceof Error ? error.message : 'Unknown proxy error'; diff --git a/src/proxy/transformers/sse-stream-transformer.ts b/src/proxy/transformers/sse-stream-transformer.ts index 53774c21..0211e211 100644 --- a/src/proxy/transformers/sse-stream-transformer.ts +++ b/src/proxy/transformers/sse-stream-transformer.ts @@ -1,7 +1,253 @@ -import { - createAnthropicErrorResponse, - createAnthropicProxyResponse, -} from '../../cursor/cursor-anthropic-response'; +import { DeltaAccumulator } from '../../glmt/delta-accumulator'; +import { GlmtTransformer } from '../../glmt/glmt-transformer'; +import { SSEParser } from '../../glmt/sse-parser'; +import type { OpenAIResponse, SSEEvent } from '../../glmt/pipeline'; + +const JSON_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor JSON response'; +const STREAM_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor SSE response'; + +type ResponseHeaders = Headers | Record | Array<[string, string]>; + +interface AnthropicErrorPayload { + type: 'error'; + error: { + type: string; + message: string; + }; +} + +function createAnthropicErrorPayload(type: string, message: string): AnthropicErrorPayload { + return { + type: 'error', + error: { + type, + message, + }, + }; +} + +function formatErrorForLog(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + +function logTranslationError(context: string, error: unknown): void { + console.error(`[proxy-sse-transformer] ${context}: ${formatErrorForLog(error)}`); +} + +export function createAnthropicErrorResponse( + status: number, + type: string, + message: string, + headers?: ResponseHeaders +): Response { + const responseHeaders = new Headers(headers); + responseHeaders.set('Content-Type', 'application/json'); + responseHeaders.delete('Content-Length'); + + return new Response(JSON.stringify(createAnthropicErrorPayload(type, message)), { + status, + headers: responseHeaders, + }); +} + +function formatSseEvent(event: string, data: unknown): string { + return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; +} + +function hasTranslatableChoices(value: unknown): value is OpenAIResponse { + if (typeof value !== 'object' || value === null) { + return false; + } + + const { choices } = value as OpenAIResponse; + if (!Array.isArray(choices) || choices.length === 0) { + return false; + } + + const firstChoice = choices[0]; + if (typeof firstChoice !== 'object' || firstChoice === null) { + return false; + } + + const message = (firstChoice as { message?: unknown }).message; + return typeof message === 'object' && message !== null; +} + +function isSyntheticTransformationFallback(value: unknown): boolean { + return ( + typeof value === 'object' && + value !== null && + typeof (value as { id?: unknown }).id === 'string' && + (value as { id: string }).id.startsWith('msg_error_') + ); +} + +async function createAnthropicErrorProxyResponse(response: Response): Promise { + const headers = new Headers(response.headers); + headers.delete('Content-Type'); + headers.delete('Content-Length'); + + let type = + response.status === 401 + ? 'authentication_error' + : response.status === 429 + ? 'rate_limit_error' + : response.status >= 400 && response.status < 500 + ? 'invalid_request_error' + : 'api_error'; + let message = `Cursor request failed with status ${response.status}`; + + try { + const contentType = (response.headers.get('content-type') || '').toLowerCase(); + if (contentType.includes('application/json')) { + const payload = (await response.json()) as { + error?: { type?: string; message?: string }; + message?: string; + }; + + if (typeof payload?.error?.type === 'string' && payload.error.type.trim().length > 0) { + type = payload.error.type; + } + + if (typeof payload?.error?.message === 'string' && payload.error.message.trim().length > 0) { + message = payload.error.message; + } else if (typeof payload?.message === 'string' && payload.message.trim().length > 0) { + message = payload.message; + } + } else { + const text = (await response.text()).trim(); + if (text.length > 0) { + message = text; + } + } + } catch (error) { + logTranslationError('Failed to parse upstream error response', error); + } + + return createAnthropicErrorResponse(response.status, type, message, headers); +} + +async function createAnthropicJsonResponse(response: Response): Promise { + try { + const openAIResponse = await response.json(); + if (!hasTranslatableChoices(openAIResponse)) { + return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); + } + + const anthropicResponse = new GlmtTransformer().transformResponse(openAIResponse); + if (isSyntheticTransformationFallback(anthropicResponse)) { + logTranslationError( + 'Cursor JSON translation produced synthetic fallback response', + anthropicResponse + ); + return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); + } + + return new Response(JSON.stringify(anthropicResponse), { + status: response.status, + headers: { 'Content-Type': 'application/json' }, + }); + } catch (error) { + logTranslationError('Cursor JSON translation failed', error); + return createAnthropicErrorResponse(502, 'api_error', JSON_TRANSLATION_ERROR_MESSAGE); + } +} + +function createAnthropicStreamingResponse(response: Response): Response { + const body = response.body; + if (!body) { + return createAnthropicErrorResponse( + 502, + 'api_error', + 'Cursor stream ended before a response body was available' + ); + } + + const parser = new SSEParser({ throwOnMalformedJson: true }); + const transformer = new GlmtTransformer(); + const accumulator = new DeltaAccumulator({}); + const encoder = new TextEncoder(); + + const readable = new ReadableStream({ + async start(controller) { + const reader = body.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + if (!value) { + continue; + } + + const events = parser.parse(Buffer.from(value)); + for (const event of events) { + const anthropicEvents = transformer.transformDelta(event as SSEEvent, accumulator); + for (const anthropicEvent of anthropicEvents) { + controller.enqueue( + encoder.encode(formatSseEvent(anthropicEvent.event, anthropicEvent.data)) + ); + } + } + } + + if (!accumulator.isFinalized() && accumulator.isMessageStarted()) { + for (const anthropicEvent of transformer.finalizeDelta(accumulator)) { + controller.enqueue( + encoder.encode(formatSseEvent(anthropicEvent.event, anthropicEvent.data)) + ); + } + } + } catch (error) { + logTranslationError('Cursor SSE translation failed', error); + controller.enqueue( + encoder.encode( + formatSseEvent( + 'error', + createAnthropicErrorPayload('api_error', STREAM_TRANSLATION_ERROR_MESSAGE) + ) + ) + ); + } finally { + reader.releaseLock(); + controller.close(); + } + }, + }); + + return new Response(readable, { + status: response.status, + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); +} + +export async function createAnthropicProxyResponse(response: Response): Promise { + if (!response.ok) { + return createAnthropicErrorProxyResponse(response); + } + + const contentType = (response.headers.get('content-type') || '').toLowerCase(); + const isEventStream = + contentType === 'text/event-stream' || contentType.startsWith('text/event-stream;'); + + return isEventStream + ? createAnthropicStreamingResponse(response) + : createAnthropicJsonResponse(response); +} export class ProxySseStreamTransformer { async transform(response: Response): Promise { diff --git a/tests/unit/glmt/sse-parser.test.ts b/tests/unit/glmt/sse-parser.test.ts index 10f14177..740fc0e1 100644 --- a/tests/unit/glmt/sse-parser.test.ts +++ b/tests/unit/glmt/sse-parser.test.ts @@ -28,8 +28,22 @@ describe('SSEParser', () => { const events = parser.parse('\n\n'); expect(events).toHaveLength(1); - expect((events[0]?.data as { choices?: Array<{ delta?: { content?: string } }> })?.choices?.[0]?.delta?.content).toBe( - 'Hi' + expect( + (events[0]?.data as { choices?: Array<{ delta?: { content?: string } }> })?.choices?.[0] + ?.delta?.content + ).toBe('Hi'); + }); + + it('accepts standalone carriage-return line endings', () => { + const parser = new SSEParser({ throwOnMalformedJson: true }); + const events = parser.parse( + ['event: message', 'data: {"choices":[{"delta":{"content":"Legacy"}}]}', '', ''].join('\r') ); + + expect(events).toHaveLength(1); + expect( + (events[0]?.data as { choices?: Array<{ delta?: { content?: string } }> })?.choices?.[0] + ?.delta?.content + ).toBe('Legacy'); }); }); diff --git a/tests/unit/proxy/messages-route.test.ts b/tests/unit/proxy/messages-route.test.ts new file mode 100644 index 00000000..3ac40437 --- /dev/null +++ b/tests/unit/proxy/messages-route.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'bun:test'; +import { EventEmitter } from 'events'; +import { attachDisconnectAbortHandlers } from '../../../src/proxy/server/messages-route'; + +class FakeSocket extends EventEmitter { + destroyed = false; +} + +class FakeRequest extends EventEmitter { + destroyed = false; + socket = new FakeSocket(); +} + +class FakeResponse extends EventEmitter { + destroyed = false; + writableEnded = false; + socket = new FakeSocket(); +} + +describe('attachDisconnectAbortHandlers', () => { + it('cleans up registered listeners after the request completes', () => { + const req = new FakeRequest(); + const res = new FakeResponse(); + const controller = new AbortController(); + + const cleanup = attachDisconnectAbortHandlers( + req as never, + res as never, + controller, + () => {} + ); + + expect(req.listenerCount('aborted')).toBe(1); + expect(req.listenerCount('close')).toBe(1); + expect(req.socket.listenerCount('close')).toBe(1); + expect(res.listenerCount('close')).toBe(1); + expect(res.socket.listenerCount('close')).toBe(1); + + cleanup(); + + expect(req.listenerCount('aborted')).toBe(0); + expect(req.listenerCount('close')).toBe(0); + expect(req.socket.listenerCount('close')).toBe(0); + expect(res.listenerCount('close')).toBe(0); + expect(res.socket.listenerCount('close')).toBe(0); + }); + + it('aborts at most once when disconnect signals race each other', () => { + const req = new FakeRequest(); + const res = new FakeResponse(); + const controller = new AbortController(); + let disconnectCount = 0; + + const cleanup = attachDisconnectAbortHandlers( + req as never, + res as never, + controller, + () => { + disconnectCount += 1; + } + ); + + req.emit('close'); + req.socket.emit('close'); + res.emit('close'); + + expect(controller.signal.aborted).toBe(true); + expect(disconnectCount).toBe(1); + + cleanup(); + }); +}); diff --git a/tests/unit/proxy/transformers/sse-stream-transformer.test.ts b/tests/unit/proxy/transformers/sse-stream-transformer.test.ts new file mode 100644 index 00000000..9faebabc --- /dev/null +++ b/tests/unit/proxy/transformers/sse-stream-transformer.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'bun:test'; +import { createAnthropicProxyResponse } from '../../../../src/proxy/transformers/sse-stream-transformer'; + +describe('proxy SSE stream transformer', () => { + it('converts OpenAI JSON into Anthropic message JSON', async () => { + const response = new Response( + JSON.stringify({ + id: 'chatcmpl_1', + model: 'claude-sonnet-4.5', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Here is the result.', + reasoning_content: 'Need to call the tool first.', + tool_calls: [ + { + id: 'toolu_2', + type: 'function', + function: { name: 'search', arguments: '{"q":"proxy"}' }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 }, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + } + ); + + const transformed = await createAnthropicProxyResponse(response); + const body = (await transformed.json()) as { + type: string; + stop_reason: string; + content: Array<{ type: string; thinking?: string; name?: string }>; + }; + + expect(body.type).toBe('message'); + expect(body.stop_reason).toBe('tool_use'); + expect(body.content.map((block) => block.type)).toEqual(['thinking', 'text', 'tool_use']); + expect(body.content[0]?.thinking).toContain('Need to call the tool first'); + expect(body.content[2]?.name).toBe('search'); + }); + + it('converts OpenAI SSE chunks into Anthropic SSE events', async () => { + const openAISse = [ + 'data: {"id":"chatcmpl_2","object":"chat.completion.chunk","created":1,"model":"claude-sonnet-4.5","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}\n\n', + 'data: {"id":"chatcmpl_2","object":"chat.completion.chunk","created":1,"model":"claude-sonnet-4.5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":1,"total_tokens":6}}\n\n', + 'data: [DONE]\n\n', + ].join(''); + + const transformed = await createAnthropicProxyResponse( + new Response(openAISse, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + ); + + const body = await transformed.text(); + expect(body).toContain('event: message_start'); + expect(body).toContain('event: content_block_start'); + expect(body).toContain('"type":"text_delta"'); + expect(body).toContain('event: message_stop'); + }); +}); From 841eeb497c274ed38ec2c653ec7c9cd02c1ea0ed Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Wed, 15 Apr 2026 02:52:29 -0400 Subject: [PATCH 8/8] fix(proxy): keep stream guards active through sse piping --- src/proxy/server/messages-route.ts | 2 -- .../transformers/sse-stream-transformer.ts | 14 +++++------ .../proxy/messages-edge-cases.test.ts | 24 ++++++++++++++++++- .../cursor-anthropic-translator.test.ts | 12 +++++----- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/proxy/server/messages-route.ts b/src/proxy/server/messages-route.ts index 13c329e8..95388e00 100644 --- a/src/proxy/server/messages-route.ts +++ b/src/proxy/server/messages-route.ts @@ -221,8 +221,6 @@ export async function handleProxyMessagesRequest( resolveOpenAIChatCompletionsUrl(upstream.route.profile.baseUrl), buildFetchInit(upstream.route.profile, upstream.body, controller.signal, insecureDispatcher) ); - clearTimeout(timeout); - cleanupDisconnectHandlers(); logger.info('response.received', 'Received upstream response', { profileName: profile.profileName, routedProfileName: upstream.route.profile.profileName, diff --git a/src/proxy/transformers/sse-stream-transformer.ts b/src/proxy/transformers/sse-stream-transformer.ts index 0211e211..dff2b7ce 100644 --- a/src/proxy/transformers/sse-stream-transformer.ts +++ b/src/proxy/transformers/sse-stream-transformer.ts @@ -3,8 +3,8 @@ import { GlmtTransformer } from '../../glmt/glmt-transformer'; import { SSEParser } from '../../glmt/sse-parser'; import type { OpenAIResponse, SSEEvent } from '../../glmt/pipeline'; -const JSON_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor JSON response'; -const STREAM_TRANSLATION_ERROR_MESSAGE = 'Failed to translate Cursor SSE response'; +const JSON_TRANSLATION_ERROR_MESSAGE = 'Failed to translate OpenAI-compatible JSON response'; +const STREAM_TRANSLATION_ERROR_MESSAGE = 'Failed to translate OpenAI-compatible SSE response'; type ResponseHeaders = Headers | Record | Array<[string, string]>; @@ -103,7 +103,7 @@ async function createAnthropicErrorProxyResponse(response: Response): Promise= 400 && response.status < 500 ? 'invalid_request_error' : 'api_error'; - let message = `Cursor request failed with status ${response.status}`; + let message = `Upstream request failed with status ${response.status}`; try { const contentType = (response.headers.get('content-type') || '').toLowerCase(); @@ -145,7 +145,7 @@ async function createAnthropicJsonResponse(response: Response): Promise { type: 'error', error: { type: 'api_error', - message: 'Failed to translate Cursor JSON response', + message: 'Failed to translate OpenAI-compatible JSON response', }, }); }); @@ -202,6 +202,28 @@ describe('openai proxy message edge cases', () => { }); }); + it('emits an SSE error if the upstream stalls after response headers are sent', async () => { + process.env.CCS_OPENAI_PROXY_REQUEST_TIMEOUT_MS = '50'; + await startProxyWithHandler((_req, res) => { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"partial"}}]}\n\n' + ); + }); + + const response = await requestProxy({ + model: 'hf-model', + stream: true, + messages: [{ role: 'user', content: 'hello' }], + }); + + expect(response.status).toBe(200); + const body = await response.text(); + expect(body).toContain('event: message_start'); + expect(body).toContain('event: error'); + expect(body).toContain('"message":"Failed to translate OpenAI-compatible SSE response"'); + }); + it('aborts the upstream request when the client disconnects mid-flight', async () => { await startProxyWithHandler(() => {}); diff --git a/tests/unit/cursor/cursor-anthropic-translator.test.ts b/tests/unit/cursor/cursor-anthropic-translator.test.ts index 0abaec2b..9da161fe 100644 --- a/tests/unit/cursor/cursor-anthropic-translator.test.ts +++ b/tests/unit/cursor/cursor-anthropic-translator.test.ts @@ -281,7 +281,7 @@ describe('createAnthropicProxyResponse', () => { }; expect(body.type).toBe('error'); expect(body.error?.type).toBe('api_error'); - expect(body.error?.message).toBe('Failed to translate Cursor JSON response'); + expect(body.error?.message).toBe('Failed to translate OpenAI-compatible JSON response'); }); it('returns 502 when Cursor response is missing choices', async () => { @@ -305,7 +305,7 @@ describe('createAnthropicProxyResponse', () => { }; expect(body.type).toBe('error'); expect(body.error?.type).toBe('api_error'); - expect(body.error?.message).toBe('Failed to translate Cursor JSON response'); + expect(body.error?.message).toBe('Failed to translate OpenAI-compatible JSON response'); }); it('returns 502 when Cursor response has empty choices', async () => { @@ -330,7 +330,7 @@ describe('createAnthropicProxyResponse', () => { }; expect(body.type).toBe('error'); expect(body.error?.type).toBe('api_error'); - expect(body.error?.message).toBe('Failed to translate Cursor JSON response'); + expect(body.error?.message).toBe('Failed to translate OpenAI-compatible JSON response'); }); it('returns Anthropic error envelopes for non-OK upstream JSON errors', async () => { @@ -382,7 +382,7 @@ describe('createAnthropicProxyResponse', () => { }; expect(body.type).toBe('error'); expect(body.error?.type).toBe('api_error'); - expect(body.error?.message).toBe('Failed to translate Cursor JSON response'); + expect(body.error?.message).toBe('Failed to translate OpenAI-compatible JSON response'); }); it('converts OpenAI SSE chunks into Anthropic SSE events', async () => { @@ -420,7 +420,7 @@ describe('createAnthropicProxyResponse', () => { expect(body).toContain('event: error'); expect(body).toContain('"type":"error"'); expect(body).toContain('"error":{"type":"api_error"'); - expect(body).toContain('Failed to translate Cursor SSE response'); + expect(body).toContain('Failed to translate OpenAI-compatible SSE response'); }); it('emits Anthropic-style error events when SSE JSON is malformed', async () => { @@ -435,6 +435,6 @@ describe('createAnthropicProxyResponse', () => { expect(body).toContain('event: error'); expect(body).toContain('"type":"error"'); expect(body).toContain('"error":{"type":"api_error"'); - expect(body).toContain('Failed to translate Cursor SSE response'); + expect(body).toContain('Failed to translate OpenAI-compatible SSE response'); }); });