fix(proxy): stabilize upstream fetch transport

Refs #1686

tests/integration/proxy/fixtures/runtime-probe.cjs

@@ -0,0 +1,100 @@
+const http = require('node:http');
+const path = require('node:path');
+
+const repoRoot = process.env.CCS_RUNTIME_PROBE_ROOT || process.cwd();
+const {
+  closeOpenAICompatProxyServer,
+  startOpenAICompatProxyServer,
+} = require(path.join(repoRoot, 'dist/proxy/server/proxy-server.js'));
+
+function waitForListening(server) {
+  if (server.listening) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.once('listening', resolve);
+  });
+}
+
+function request(port, requestPath, options = {}) {
+  return new Promise((resolve, reject) => {
+    const req = http.request(
+      {
+        hostname: '127.0.0.1',
+        port,
+        path: requestPath,
+        method: options.method || 'GET',
+        headers: options.headers,
+        agent: options.agent,
+      },
+      (res) => {
+        let body = '';
+        res.setEncoding('utf8');
+        res.on('data', (chunk) => (body += chunk));
+        res.on('end', () => resolve({ status: res.statusCode || 0, body }));
+      }
+    );
+    req.once('error', reject);
+    req.end(options.body);
+  });
+}
+
+async function startProbeServer() {
+  const server = startOpenAICompatProxyServer({
+    profile: {
+      profileName: 'sentinel',
+      settingsPath: '/tmp/sentinel.settings.json',
+      baseUrl: process.env.CCS_RUNTIME_PROBE_UPSTREAM,
+      apiKey: 'sentinel-api-key-not-real',
+      provider: 'generic-chat-completion-api',
+      model: 'sentinel-model',
+      passthrough: false,
+    },
+    port: 0,
+    authToken: 'sentinel-local-token',
+  });
+  await waitForListening(server);
+  return server;
+}
+
+async function runTransportProbe() {
+  const server = await startProbeServer();
+  const port = server.address().port;
+  const response = await request(port, '/v1/messages', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-api-key': 'sentinel-local-token',
+    },
+    body: JSON.stringify({
+      model: 'sentinel-model',
+      messages: [{ role: 'user', content: 'hi' }],
+    }),
+  });
+  await closeOpenAICompatProxyServer(server, 250);
+  const parsed = JSON.parse(response.body);
+  console.log(
+    JSON.stringify({
+      status: response.status,
+      text: parsed.content?.[0]?.text,
+      error: parsed.error?.message,
+    })
+  );
+}
+
+async function runShutdownProbe() {
+  const server = await startProbeServer();
+  const port = server.address().port;
+  const agent = new http.Agent({ keepAlive: true });
+  await request(port, '/health', { agent });
+  const startedAt = Date.now();
+  await closeOpenAICompatProxyServer(server, 250);
+  const elapsedMs = Date.now() - startedAt;
+  agent.destroy();
+  console.log(JSON.stringify({ closed: !server.listening, elapsedMs }));
+}
+
+const mode = process.env.CCS_RUNTIME_PROBE_MODE || 'transport';
+Promise.resolve(mode === 'shutdown' ? runShutdownProbe() : runTransportProbe()).catch((error) => {
+  console.error(JSON.stringify({ name: error.name, message: error.message }));
+  process.exitCode = 1;
+});

tests/integration/proxy/runtime-transport-matrix.test.ts

@@ -0,0 +1,200 @@
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { spawn, spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as http from 'node:http';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+const matrixEnabled = process.env.CCS_RUNTIME_MATRIX === '1';
+const fixturePath = path.join(process.cwd(), 'tests/integration/proxy/fixtures/runtime-probe.cjs');
+const proxyKeys = [
+  'HTTP_PROXY',
+  'HTTPS_PROXY',
+  'ALL_PROXY',
+  'NO_PROXY',
+  'http_proxy',
+  'https_proxy',
+  'all_proxy',
+  'no_proxy',
+];
+
+type ProbeResult = {
+  status?: number;
+  text?: string;
+  error?: string;
+  closed?: boolean;
+  elapsedMs?: number;
+};
+
+function resolveNodeBinary(version: 18 | 22 | 26): string {
+  const result = spawnSync('npx', ['-y', `node@${version}`, '-p', 'process.execPath'], {
+    encoding: 'utf8',
+    env: Object.fromEntries(
+      Object.entries(process.env).filter(([key]) => !proxyKeys.includes(key))
+    ),
+  });
+  if (result.status !== 0) {
+    throw new Error(`Unable to resolve Node ${version}: ${result.stderr.trim()}`);
+  }
+  return result.stdout.trim();
+}
+
+function runProbe(
+  binary: string,
+  upstream: string,
+  env: Record<string, string> = {},
+  mode = 'transport'
+): Promise<ProbeResult> {
+  const cleanEnv = Object.fromEntries(
+    Object.entries(process.env).filter(([key]) => !proxyKeys.includes(key))
+  );
+  const ccsHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-runtime-matrix-'));
+  return new Promise((resolve, reject) => {
+    const child = spawn(binary, [fixturePath], {
+      cwd: process.cwd(),
+      env: {
+        ...cleanEnv,
+        ...env,
+        CCS_RUNTIME_PROBE_ROOT: process.cwd(),
+        CCS_RUNTIME_PROBE_UPSTREAM: upstream,
+        CCS_RUNTIME_PROBE_MODE: mode,
+        CCS_HOME: ccsHome,
+      },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => (stdout += chunk));
+    child.stderr.on('data', (chunk) => (stderr += chunk));
+    child.once('error', (error) => {
+      fs.rmSync(ccsHome, { recursive: true, force: true });
+      reject(error);
+    });
+    child.once('close', (code) => {
+      fs.rmSync(ccsHome, { recursive: true, force: true });
+      if (code !== 0) {
+        reject(new Error(`Probe exited ${code}: ${stderr || stdout}`));
+        return;
+      }
+      const lastLine = stdout.trim().split('\n').pop();
+      if (!lastLine) {
+        reject(new Error(`Probe returned no output: ${stderr}`));
+        return;
+      }
+      resolve(JSON.parse(lastLine) as ProbeResult);
+    });
+  });
+}
+
+describe.skipIf(!matrixEnabled)('real runtime upstream transport matrix', () => {
+  let node18 = '';
+  let node22 = '';
+  let node26 = '';
+  let upstreamServer: http.Server | undefined;
+  let upstreamPort = 0;
+  let corporateProxy: http.Server | undefined;
+  let corporateProxyPort = 0;
+  let corporateProxyHits = 0;
+  let sawProxyAuthorization = false;
+
+  beforeAll(async () => {
+    node18 = resolveNodeBinary(18);
+    node22 = resolveNodeBinary(22);
+    node26 = resolveNodeBinary(26);
+    const payload = JSON.stringify({
+      id: 'chatcmpl_matrix',
+      model: 'sentinel-model',
+      choices: [
+        {
+          index: 0,
+          message: { role: 'assistant', content: 'matrix-ok' },
+          finish_reason: 'stop',
+        },
+      ],
+      usage: { prompt_tokens: 1, completion_tokens: 1 },
+    });
+    upstreamServer = http.createServer((_req, res) => {
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end(payload);
+    });
+    corporateProxy = http.createServer((req, res) => {
+      corporateProxyHits += 1;
+      sawProxyAuthorization = String(req.headers['proxy-authorization'] || '').startsWith('Basic ');
+      res.writeHead(200, {
+        'content-type': 'application/json',
+        'content-length': Buffer.byteLength(payload),
+      });
+      res.end(payload);
+    });
+    corporateProxy.on('connect', (req, socket) => {
+      corporateProxyHits += 1;
+      sawProxyAuthorization = String(req.headers['proxy-authorization'] || '').startsWith('Basic ');
+      socket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+      socket.once('data', () => {
+        socket.end(
+          `HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(payload)}\r\nConnection: close\r\n\r\n${payload}`
+        );
+      });
+    });
+    await Promise.all([
+      new Promise<void>((resolve) => upstreamServer.listen(0, '127.0.0.1', resolve)),
+      new Promise<void>((resolve) => corporateProxy.listen(0, '127.0.0.1', resolve)),
+    ]);
+    upstreamPort = (upstreamServer.address() as { port: number }).port;
+    corporateProxyPort = (corporateProxy.address() as { port: number }).port;
+  }, 60_000);
+
+  afterAll(async () => {
+    const servers = [upstreamServer, corporateProxy].filter(
+      (server): server is http.Server => server !== undefined
+    );
+    await Promise.all(
+      servers.map((server) => new Promise<void>((resolve) => server.close(() => resolve())))
+    );
+  }, 10_000);
+
+  it('uses the matching fetch transport on Node 18, 22, 26, and Bun', async () => {
+    const upstream = `http://127.0.0.1:${upstreamPort}/v1`;
+    const results = await Promise.all([
+      runProbe(node18, upstream),
+      runProbe(node22, upstream),
+      runProbe(node26, upstream),
+      runProbe(process.execPath, upstream),
+    ]);
+    for (const result of results) {
+      expect(result).toMatchObject({ status: 200, text: 'matrix-ok' });
+    }
+  });
+
+  it('routes Node 26 and Bun through a real credentialed proxy', async () => {
+    const proxyUrl = `http://sentinel-user:sentinel-password@127.0.0.1:${corporateProxyPort}`;
+    const nodeResult = await runProbe(node26, 'http://upstream.invalid/v1', {
+      HTTP_PROXY: proxyUrl,
+      NO_PROXY: '127.0.0.1,localhost',
+    });
+    const bunResult = await runProbe(process.execPath, 'http://upstream.invalid/v1', {
+      HTTP_PROXY: proxyUrl,
+      NO_PROXY: '127.0.0.1,localhost',
+    });
+    expect(nodeResult).toMatchObject({ status: 200, text: 'matrix-ok' });
+    expect(bunResult).toMatchObject({ status: 200, text: 'matrix-ok' });
+    expect(corporateProxyHits).toBeGreaterThanOrEqual(2);
+    expect(sawProxyAuthorization).toBe(true);
+  });
+
+  it('honors NO_PROXY without touching the proxy', async () => {
+    const hitsBefore = corporateProxyHits;
+    const result = await runProbe(node26, 'http://upstream.invalid/v1', {
+      HTTP_PROXY: `http://sentinel-user:sentinel-password@127.0.0.1:${corporateProxyPort}`,
+      NO_PROXY: 'upstream.invalid',
+    });
+    expect(result.status).toBe(502);
+    expect(corporateProxyHits).toBe(hitsBefore);
+  });
+
+  it('shuts down Node 18 with an idle keep-alive connection', async () => {
+    const result = await runProbe(node18, `http://127.0.0.1:${upstreamPort}/v1`, {}, 'shutdown');
+    expect(result.closed).toBe(true);
+    expect(result.elapsedMs).toBeLessThan(2_000);
+  });
+});

tests/integration/proxy/runtime-transport.test.ts

@@ -0,0 +1,195 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import * as fs from 'node:fs';
+import * as http from 'node:http';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { getGlobalDispatcher } from 'undici';
+import {
+  closeOpenAICompatProxyServer,
+  startOpenAICompatProxyServer,
+} from '../../../src/proxy/server/proxy-server';
+import type { OpenAICompatProfileConfig } from '../../../src/proxy/profile-router';
+
+const PROXY_ENV_KEYS = ['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY'] as const;
+
+function listen(server: http.Server): Promise<number> {
+  return new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string') {
+        reject(new Error('Failed to resolve listening port'));
+        return;
+      }
+      resolve(address.port);
+    });
+  });
+}
+
+function requestMessages(port: number): Promise<{ status: number; body: string }> {
+  return new Promise((resolve, reject) => {
+    const req = http.request(
+      {
+        hostname: '127.0.0.1',
+        port,
+        path: '/v1/messages',
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          'x-api-key': 'sentinel-local-token',
+        },
+      },
+      (res) => {
+        let body = '';
+        res.setEncoding('utf8');
+        res.on('data', (chunk) => (body += chunk));
+        res.on('end', () => resolve({ status: res.statusCode ?? 0, body }));
+      }
+    );
+    req.once('error', reject);
+    req.end(
+      JSON.stringify({
+        model: 'sentinel-model',
+        messages: [{ role: 'user', content: 'hi' }],
+      })
+    );
+  });
+}
+
+describe('runtime-aware upstream transport', () => {
+  const originalEnv = new Map<string, string | undefined>();
+  let ccsHome = '';
+  let corporateProxy: http.Server | undefined;
+  let corporateProxyHits = 0;
+  let sawProxyAuthorization = false;
+  let proxyServer: http.Server | undefined;
+
+  beforeEach(() => {
+    for (const key of PROXY_ENV_KEYS) {
+      originalEnv.set(key, process.env[key]);
+      delete process.env[key];
+    }
+    originalEnv.set('CCS_HOME', process.env.CCS_HOME);
+    ccsHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-runtime-transport-'));
+    process.env.CCS_HOME = ccsHome;
+    corporateProxy = undefined;
+    proxyServer = undefined;
+    corporateProxyHits = 0;
+    sawProxyAuthorization = false;
+  });
+
+  afterEach(async () => {
+    if (proxyServer) {
+      await closeOpenAICompatProxyServer(proxyServer, 50);
+    }
+    if (corporateProxy) {
+      await new Promise<void>((resolve) => corporateProxy.close(() => resolve()));
+    }
+    for (const key of [...PROXY_ENV_KEYS, 'CCS_HOME'] as const) {
+      const value = originalEnv.get(key);
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+    fs.rmSync(ccsHome, { recursive: true, force: true });
+  });
+
+  it('uses a credentialed proxy and honors NO_PROXY without global fetch mutation', async () => {
+    const originalFetch = globalThis.fetch;
+    const upstreamPayload = JSON.stringify({
+      id: 'chatcmpl_proxy',
+      model: 'sentinel-model',
+      choices: [
+        {
+          index: 0,
+          message: { role: 'assistant', content: 'proxied-ok' },
+          finish_reason: 'stop',
+        },
+      ],
+      usage: { prompt_tokens: 1, completion_tokens: 1 },
+    });
+    corporateProxy = http.createServer((req, res) => {
+      corporateProxyHits += 1;
+      sawProxyAuthorization = String(req.headers['proxy-authorization'] || '').startsWith('Basic ');
+      res.writeHead(200, {
+        'content-type': 'application/json',
+        'content-length': Buffer.byteLength(upstreamPayload),
+      });
+      res.end(upstreamPayload);
+    });
+    const corporateProxyPort = await listen(corporateProxy);
+    process.env.HTTP_PROXY = `http://sentinel-user:sentinel-password@127.0.0.1:${corporateProxyPort}`;
+
+    const profile: OpenAICompatProfileConfig = {
+      profileName: 'sentinel',
+      settingsPath: '/tmp/sentinel.settings.json',
+      baseUrl: 'http://upstream.invalid/v1',
+      apiKey: 'sentinel-api-key-not-real',
+      provider: 'generic-chat-completion-api',
+      model: 'sentinel-model',
+    };
+    proxyServer = startOpenAICompatProxyServer({
+      profile,
+      port: 0,
+      authToken: 'sentinel-local-token',
+    });
+    const proxyPort = await new Promise<number>((resolve, reject) => {
+      proxyServer.once('error', reject);
+      proxyServer.once('listening', () => {
+        const address = proxyServer.address();
+        if (!address || typeof address === 'string') reject(new Error('Missing proxy port'));
+        else resolve(address.port);
+      });
+    });
+
+    const proxied = await requestMessages(proxyPort);
+    expect(proxied.status).toBe(200);
+    expect(JSON.parse(proxied.body)).toMatchObject({
+      content: [{ type: 'text', text: 'proxied-ok' }],
+    });
+    expect(corporateProxyHits).toBe(1);
+    expect(sawProxyAuthorization).toBe(true);
+
+    process.env.NO_PROXY = 'upstream.invalid';
+    const hitsBeforeBypass = corporateProxyHits;
+    const bypassed = await requestMessages(proxyPort);
+    expect(bypassed.status).toBe(502);
+    expect(corporateProxyHits).toBe(hitsBeforeBypass);
+    expect(globalThis.fetch).toBe(originalFetch);
+  });
+
+  it('keeps global transport ownership unchanged across overlapping server lifecycles', async () => {
+    const originalFetch = globalThis.fetch;
+    const originalDispatcher = getGlobalDispatcher();
+    const profile: OpenAICompatProfileConfig = {
+      profileName: 'sentinel',
+      settingsPath: '/tmp/sentinel.settings.json',
+      baseUrl: 'http://127.0.0.1:1/v1',
+      apiKey: 'sentinel-api-key-not-real',
+      provider: 'generic-chat-completion-api',
+      model: 'sentinel-model',
+    };
+    const serverA = startOpenAICompatProxyServer({
+      profile,
+      port: 0,
+      authToken: 'sentinel-local-token-a',
+    });
+    const serverB = startOpenAICompatProxyServer({
+      profile,
+      port: 0,
+      authToken: 'sentinel-local-token-b',
+    });
+
+    await Promise.all([
+      new Promise<void>((resolve) => serverA.once('listening', resolve)),
+      new Promise<void>((resolve) => serverB.once('listening', resolve)),
+    ]);
+    await closeOpenAICompatProxyServer(serverA, 50);
+    expect(serverB.listening).toBe(true);
+    expect(globalThis.fetch).toBe(originalFetch);
+    expect(getGlobalDispatcher()).toBe(originalDispatcher);
+
+    await closeOpenAICompatProxyServer(serverB, 50);
+    expect(globalThis.fetch).toBe(originalFetch);
+    expect(getGlobalDispatcher()).toBe(originalDispatcher);
+  });
+});