fix(proxy): stabilize upstream fetch transport

Refs #1686

src/proxy/proxy-daemon-entry.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import { resolveOpenAICompatProfileConfig } from './profile-router';
 import { OPENAI_COMPAT_PROXY_DEFAULT_PORT } from './proxy-daemon-paths';
-import { startOpenAICompatProxyServer } from './server/proxy-server';
+import { closeOpenAICompatProxyServer, startOpenAICompatProxyServer } from './server/proxy-server';
 import { loadSettings } from '../config/config-loader-facade';
 
 interface RuntimeOptions {
@@ -99,9 +99,17 @@ function startRuntime(options: RuntimeOptions): void {
   });
   server.once('error', (error) => {
     process.stderr.write(String((error as Error).message) + '\n');
-    process.exit(1);
+    void closeOpenAICompatProxyServer(server).finally(() => {
+      process.exitCode = 1;
+    });
   });
-  const shutdown = () => server.close();
+  let shutdownPromise: Promise<void> | undefined;
+  const shutdown = () => {
+    shutdownPromise ??= closeOpenAICompatProxyServer(server).catch((error) => {
+      process.stderr.write(`Proxy shutdown failed: ${String((error as Error).message)}\n`);
+      process.exitCode = 1;
+    });
+  };
   process.on('SIGTERM', shutdown);
   process.on('SIGINT', shutdown);
 }

src/proxy/server/messages-route.ts

@@ -1,5 +1,4 @@
 import * as http from 'http';
-import { Agent } from 'undici';
 import type { Dispatcher } from 'undici';
 import type { OpenAICompatProfileConfig } from '../profile-router';
 import { resolveProxyRequestRoute } from '../request-router';
@@ -10,11 +9,14 @@ import {
 import { ProxySseStreamTransformer } from '../transformers/sse-stream-transformer';
 import { isAnthropicPassthroughProfile, resolveOpenAIChatCompletionsUrl } from '../upstream-url';
 import { createLogger } from '../../services/logging';
-import {
-  createGlobalFetchProxyDispatcher,
-  type UpstreamAgentTimeoutOptions,
-} from '../../utils/fetch-proxy-setup';
+import type { UpstreamAgentTimeoutOptions } from '../../utils/fetch-proxy-setup';
 import { pipeWebResponseToNode, readRawBody, writeJson } from './http-helpers';
+import {
+  closeUpstreamDispatcher,
+  createUpstreamDispatcher,
+  fetchWithUpstreamTransport,
+  toLogErrorInfo,
+} from './upstream-transport';
 
 const REQUEST_TIMEOUT_MS = 600_000;
 // Keep undici's per-phase timeouts above the explicit request timeout so the
@@ -421,17 +423,6 @@ export function buildUpstreamAgentTimeouts(): UpstreamAgentTimeoutOptions {
   return { headersTimeout: ceiling, bodyTimeout: ceiling };
 }
 
-let defaultUpstreamDispatcher: Dispatcher | null = null;
-
-function getDefaultUpstreamDispatcher(): Dispatcher {
-  if (!defaultUpstreamDispatcher) {
-    const timeouts = buildUpstreamAgentTimeouts();
-    // Honor HTTP(S)_PROXY routing when configured; otherwise a plain Agent.
-    defaultUpstreamDispatcher = createGlobalFetchProxyDispatcher(timeouts) ?? new Agent(timeouts);
-  }
-  return defaultUpstreamDispatcher;
-}
-
 function formatTimeoutDuration(timeoutMs: number): string {
   return timeoutMs % 1000 === 0 ? `${timeoutMs / 1000} seconds` : `${timeoutMs}ms`;
 }
@@ -489,7 +480,10 @@ export async function handleProxyMessagesRequest(
   res: http.ServerResponse,
   profile: OpenAICompatProfileConfig,
   expectedAuthToken: string,
-  insecureDispatcher?: Dispatcher
+  insecureDispatcher?: Dispatcher,
+  sharedUpstreamDispatcher?: Dispatcher,
+  upstreamFetch?: typeof globalThis.fetch,
+  forceInsecureTls = false
 ): Promise<void> {
   const transformer = new ProxySseStreamTransformer();
   const startedAt = Date.now();
@@ -563,20 +557,26 @@ export async function handleProxyMessagesRequest(
       }
     );
 
+    const routeStaysOnActiveProfile = upstream.route.profile.profileName === profile.profileName;
     const useSharedInsecureDispatcher =
-      insecureDispatcher !== undefined &&
-      upstream.route.profile.profileName === profile.profileName;
+      insecureDispatcher !== undefined && routeStaysOnActiveProfile;
+    const useServerInsecureTls = forceInsecureTls && routeStaysOnActiveProfile;
     const useProfileInsecureTls = upstream.route.profile.insecure === true;
     const ephemeralInsecureDispatcher =
-      useProfileInsecureTls && !useSharedInsecureDispatcher
-        ? new Agent({ connect: { rejectUnauthorized: false }, ...buildUpstreamAgentTimeouts() })
+      useProfileInsecureTls && !useSharedInsecureDispatcher && !useServerInsecureTls
+        ? createUpstreamDispatcher(buildUpstreamAgentTimeouts(), true)
         : undefined;
-    const insecureTls = useSharedInsecureDispatcher || useProfileInsecureTls;
+    const insecureTls =
+      useSharedInsecureDispatcher || useServerInsecureTls || useProfileInsecureTls;
+    const ownedUpstreamDispatcher =
+      sharedUpstreamDispatcher || insecureTls
+        ? undefined
+        : createUpstreamDispatcher(buildUpstreamAgentTimeouts());
     const dispatcher = useSharedInsecureDispatcher
       ? insecureDispatcher
       : useProfileInsecureTls
         ? ephemeralInsecureDispatcher
-        : getDefaultUpstreamDispatcher();
+        : (sharedUpstreamDispatcher ?? ownedUpstreamDispatcher);
 
     try {
       logger.stage('dispatch', 'upstream.dispatch', 'Dispatching upstream fetch', {
@@ -588,17 +588,20 @@ export async function handleProxyMessagesRequest(
       const upstreamUrl = resolveOpenAIChatCompletionsUrl(upstream.route.profile.baseUrl, {
         passthrough,
       });
-      const upstreamResponse = await fetch(
-        upstreamUrl,
-        buildFetchInit(
-          upstream.route.profile,
-          upstream.body,
-          controller.signal,
-          req.headers,
-          passthrough,
-          dispatcher
-        )
+      const fetchInit = buildFetchInit(
+        upstream.route.profile,
+        upstream.body,
+        controller.signal,
+        req.headers,
+        passthrough,
+        dispatcher
       );
+      const upstreamResponse = upstreamFetch
+        ? await upstreamFetch(upstreamUrl, fetchInit)
+        : await fetchWithUpstreamTransport(upstreamUrl, fetchInit, {
+            dispatcher,
+            insecureTls,
+          });
       logger.stage('upstream', 'upstream.response', 'Received upstream response', {
         profileName: profile.profileName,
         routedProfileName: upstream.route.profile.profileName,
@@ -620,7 +623,7 @@ export async function handleProxyMessagesRequest(
       cleanupDisconnectHandlers();
       if (ephemeralInsecureDispatcher) {
         try {
-          await ephemeralInsecureDispatcher.close();
+          await closeUpstreamDispatcher(ephemeralInsecureDispatcher);
         } catch (closeError) {
           logger.stage(
             'cleanup',
@@ -635,13 +638,27 @@ export async function handleProxyMessagesRequest(
           );
         }
       }
+      if (ownedUpstreamDispatcher) {
+        try {
+          await closeUpstreamDispatcher(ownedUpstreamDispatcher);
+        } catch (closeError) {
+          logger.stage(
+            'cleanup',
+            'request.dispatcher_close_failed',
+            'Failed to close per-request upstream dispatcher',
+            {
+              profileName: profile.profileName,
+              routedProfileName: upstream.route.profile.profileName,
+              error: closeError instanceof Error ? closeError.message : String(closeError),
+            },
+            { level: 'warn' }
+          );
+        }
+      }
     }
   } catch (error) {
     const message = error instanceof Error ? error.message : 'Unknown proxy error';
-    const errInfo = {
-      name: error instanceof Error ? error.name : 'Error',
-      message,
-    };
+    const errInfo = toLogErrorInfo(error);
     logger.stage(
       'cleanup',
       'request.failed',

src/proxy/server/proxy-server.ts

@@ -1,6 +1,5 @@
 import * as http from 'http';
 import { randomUUID } from 'crypto';
-import { Agent } from 'undici';
 import type { OpenAICompatProfileConfig } from '../profile-router';
 import { OPENAI_COMPAT_PROXY_SERVICE_NAME } from '../proxy-daemon-paths';
 import { createLogger, withRequestContext } from '../../services/logging';
@@ -11,10 +10,19 @@ import {
   validateIncomingProxyAuth,
 } from './messages-route';
 import { writeJson } from './http-helpers';
+import { closeUpstreamDispatcher, createUpstreamDispatcher } from './upstream-transport';
 
 const REQUEST_ID_HEADER = 'x-ccs-request-id';
 // Loose UUID-ish guard: accepts UUIDs and similar opaque ids; rejects empty / control chars.
 const REQUEST_ID_PATTERN = /^[A-Za-z0-9._-]{8,128}$/;
+const DEFAULT_SHUTDOWN_FORCE_MS = 1_000;
+
+type ProxyServerLifecycle = {
+  cleanup: () => Promise<void>;
+  shutdownPromise?: Promise<void>;
+};
+
+const proxyServerLifecycles = new WeakMap<http.Server, ProxyServerLifecycle>();
 
 function resolveInboundRequestId(headers: http.IncomingHttpHeaders): string {
   const raw = headers[REQUEST_ID_HEADER];
@@ -40,8 +48,17 @@ export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOpt
     port: options.port,
   });
   const insecureDispatcher = options.insecure
-    ? new Agent({ connect: { rejectUnauthorized: false }, ...buildUpstreamAgentTimeouts() })
+    ? createUpstreamDispatcher(buildUpstreamAgentTimeouts(), true)
     : undefined;
+  const upstreamDispatcher = createUpstreamDispatcher(buildUpstreamAgentTimeouts());
+  let cleanupPromise: Promise<void> | undefined;
+  const cleanup = () => {
+    cleanupPromise ??= Promise.all([
+      closeUpstreamDispatcher(upstreamDispatcher),
+      closeUpstreamDispatcher(insecureDispatcher),
+    ]).then(() => undefined);
+    return cleanupPromise;
+  };
   const server = http.createServer((req, res) => {
     const requestId = resolveInboundRequestId(req.headers);
     res.setHeader(REQUEST_ID_HEADER, requestId);
@@ -121,7 +138,10 @@ export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOpt
         res,
         options.profile,
         options.authToken,
-        insecureDispatcher
+        insecureDispatcher,
+        upstreamDispatcher,
+        undefined,
+        options.insecure === true
       );
       return;
     }
@@ -138,9 +158,55 @@ export function startOpenAICompatProxyServer(options: OpenAICompatProxyServerOpt
   });
   server.on('close', () => {
     logger.info('server.stop', 'OpenAI-compatible proxy server stopped');
-    void insecureDispatcher?.close();
+    void cleanup().catch((error) => {
+      logger.warn('server.dispatcher_close_failed', 'Failed to close upstream transport', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    });
   });
 
+  proxyServerLifecycles.set(server, { cleanup });
   server.listen(options.port, host);
   return server;
 }
+
+export function closeOpenAICompatProxyServer(
+  server: http.Server,
+  forceAfterMs = DEFAULT_SHUTDOWN_FORCE_MS
+): Promise<void> {
+  const lifecycle = proxyServerLifecycles.get(server);
+  if (lifecycle?.shutdownPromise) {
+    return lifecycle.shutdownPromise;
+  }
+
+  const shutdownPromise = (async () => {
+    if (server.listening) {
+      await new Promise<void>((resolve, reject) => {
+        let forceTimer: NodeJS.Timeout | undefined;
+        server.close((error) => {
+          if (forceTimer) {
+            clearTimeout(forceTimer);
+          }
+          if (error) {
+            reject(error);
+          } else {
+            resolve();
+          }
+        });
+
+        // Node 18 does not reap idle keep-alive sockets as part of close().
+        server.closeIdleConnections?.();
+        if (forceAfterMs > 0) {
+          forceTimer = setTimeout(() => server.closeAllConnections?.(), forceAfterMs);
+          forceTimer.unref();
+        }
+      });
+    }
+    await lifecycle?.cleanup();
+  })();
+
+  if (lifecycle) {
+    lifecycle.shutdownPromise = shutdownPromise;
+  }
+  return shutdownPromise;
+}

src/proxy/server/upstream-transport.ts

@@ -0,0 +1,93 @@
+import { Agent, fetch as undiciFetch, type Dispatcher } from 'undici';
+import type { LogErrorInfo } from '../../services/logging';
+import {
+  createGlobalFetchProxyDispatcher,
+  type UpstreamAgentTimeoutOptions,
+} from '../../utils/fetch-proxy-setup';
+import { getProxyResolution, shouldBypassProxy } from '../../utils/proxy-env';
+
+type BunRequestInit = RequestInit & {
+  proxy?: string;
+  tls?: { rejectUnauthorized: boolean };
+};
+
+const dispatcherClosePromises = new WeakMap<object, Promise<void>>();
+
+export function isBunRuntime(): boolean {
+  return typeof process.versions.bun === 'string';
+}
+
+export function createUpstreamDispatcher(
+  options: UpstreamAgentTimeoutOptions,
+  insecure = false
+): Dispatcher | undefined {
+  if (isBunRuntime()) {
+    return undefined;
+  }
+  if (insecure) {
+    return new Agent({ connect: { rejectUnauthorized: false }, ...options });
+  }
+  return createGlobalFetchProxyDispatcher(options) ?? new Agent(options);
+}
+
+export function closeUpstreamDispatcher(dispatcher?: Dispatcher): Promise<void> {
+  if (!dispatcher) {
+    return Promise.resolve();
+  }
+  const existing = dispatcherClosePromises.get(dispatcher);
+  if (existing) {
+    return existing;
+  }
+
+  const close = (dispatcher as unknown as { close?: () => Promise<void> }).close;
+  const closing =
+    typeof close === 'function' ? Promise.resolve(close.call(dispatcher)) : Promise.resolve();
+  dispatcherClosePromises.set(dispatcher, closing);
+  return closing;
+}
+
+export async function fetchWithUpstreamTransport(
+  input: string | URL | Request,
+  init: RequestInit,
+  options: { dispatcher?: Dispatcher; insecureTls?: boolean } = {}
+): Promise<Response> {
+  if (!isBunRuntime()) {
+    return undiciFetch(input, init as Parameters<typeof undiciFetch>[1]) as Promise<Response>;
+  }
+
+  const requestUrl = new URL(
+    typeof input === 'string' || input instanceof URL ? input.toString() : input.url
+  );
+  const bunInit: BunRequestInit = { ...init };
+  delete (bunInit as Record<string, unknown>).dispatcher;
+
+  if (!shouldBypassProxy(requestUrl.hostname)) {
+    const proxy = getProxyResolution(requestUrl.protocol === 'https:', process.env, {
+      allowedProtocols: ['http:', 'https:'],
+    });
+    if (proxy.url) {
+      bunInit.proxy = proxy.url;
+    }
+  }
+  if (options.insecureTls) {
+    bunInit.tls = { rejectUnauthorized: false };
+  }
+
+  return globalThis.fetch(input, bunInit);
+}
+
+export function toLogErrorInfo(error: unknown, depth = 0): LogErrorInfo {
+  if (!(error instanceof Error)) {
+    return { name: 'Error', message: String(error) };
+  }
+
+  const errorWithDetails = error as Error & { code?: unknown; cause?: unknown };
+  const info: LogErrorInfo = { name: error.name, message: error.message };
+  if (typeof errorWithDetails.code === 'string') {
+    info.code = errorWithDetails.code;
+  }
+  if (depth < 3 && errorWithDetails.cause !== undefined && errorWithDetails.cause !== error) {
+    info.cause = toLogErrorInfo(errorWithDetails.cause, depth + 1);
+  }
+  return info;
+}