Merge pull request #708 from kaitranntt/kai/fix/codex-quota-diagnostics

fix: improve quota diagnostics and oauth refresh handling

tests/unit/cliproxy/quota-caching-integration.test.ts

@@ -17,6 +17,7 @@ import {
   getQuotaCacheStats,
   QUOTA_CACHE_TTL_MS,
 } from '../../../src/cliproxy/quota-response-cache';
+import { shouldCacheQuotaResult } from '../../../src/web-server/routes/cliproxy-stats-routes';
 import type { GeminiCliQuotaResult, CodexQuotaResult } from '../../../src/cliproxy/quota-types';
 
 describe('Quota Caching Integration', () => {
@@ -308,6 +309,60 @@ describe('Quota Caching Integration', () => {
 
       expect(cached?.error).toBe('API error: 503');
     });
+
+    it('should cache stable auth and workspace failures', () => {
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          needsReauth: true,
+          error: 'Token expired',
+        })
+      ).toBe(true);
+
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          httpStatus: 402,
+          error: 'Workspace deactivated (HTTP 402)',
+        })
+      ).toBe(true);
+    });
+
+    it('should skip transient failures marked retryable or temporary by status', () => {
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          retryable: true,
+          error: 'Rate limited - try again later',
+        })
+      ).toBe(false);
+
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          httpStatus: 429,
+          error: 'Rate limited - try again later',
+        })
+      ).toBe(false);
+
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          httpStatus: 503,
+          error: 'Codex quota service unavailable (HTTP 503)',
+        })
+      ).toBe(false);
+    });
+
+    it('should respect explicit non-retryable failures even without message pattern matches', () => {
+      expect(
+        shouldCacheQuotaResult({
+          success: false,
+          retryable: false,
+          error: 'Unknown upstream error',
+        })
+      ).toBe(true);
+    });
   });
 
   describe('high-volume scenarios', () => {

tests/unit/cliproxy/quota-fetcher-codex.test.ts

@@ -4,13 +4,48 @@
  * Tests for Codex quota window parsing and transformation logic
  */
 
-import { describe, it, expect } from 'bun:test';
+import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
 import {
   buildCodexQuotaWindows,
   buildCodexCoreUsageSummary,
+  fetchCodexQuota,
   getUnknownCodexWindowLabels,
 } from '../../../src/cliproxy/quota-fetcher-codex';
 
+let tmpDir: string;
+let originalCcsHome: string | undefined;
+let originalFetch: typeof fetch;
+
+function createCodexAccount(
+  accountId: string,
+  tokenPayload: Record<string, unknown>,
+  tokenFile = `codex-${accountId.replace(/[@.]/g, '_')}.json`
+): void {
+  const authDir = path.join(tmpDir, '.ccs', 'cliproxy', 'auth');
+  fs.mkdirSync(authDir, { recursive: true });
+  fs.writeFileSync(path.join(authDir, tokenFile), JSON.stringify(tokenPayload));
+}
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-codex-quota-test-'));
+  originalCcsHome = process.env.CCS_HOME;
+  process.env.CCS_HOME = tmpDir;
+  originalFetch = global.fetch;
+});
+
+afterEach(() => {
+  global.fetch = originalFetch;
+  if (originalCcsHome !== undefined) {
+    process.env.CCS_HOME = originalCcsHome;
+  } else {
+    delete process.env.CCS_HOME;
+  }
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+});
+
 describe('Codex Quota Fetcher', () => {
   describe('buildCodexQuotaWindows', () => {
     it('should parse snake_case API response', () => {
@@ -307,4 +342,184 @@ describe('Codex Quota Fetcher', () => {
       expect(labels).toEqual([]);
     });
   });
+
+  describe('fetchCodexQuota failure mapping', () => {
+    function createValidCodexAccount(email: string, accountId = `workspace-${email}`): void {
+      createCodexAccount(email, {
+        access_token: 'test-token',
+        account_id: accountId,
+        expired: '2099-01-01T00:00:00.000Z',
+        email,
+        type: 'codex',
+      });
+    }
+
+    it('maps deactivated workspace 402 responses to structured metadata', async () => {
+      createValidCodexAccount('workspace@example.com', 'workspace-123');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({ detail: { code: 'deactivated_workspace' } }), {
+            status: 402,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('workspace@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(402);
+      expect(result.errorCode).toBe('deactivated_workspace');
+      expect(result.error).toContain('Workspace deactivated');
+      expect(result.actionHint).toContain('active ChatGPT workspace');
+      expect(result.retryable).toBe(false);
+    });
+
+    it('maps 401 responses to reauth-required metadata', async () => {
+      createValidCodexAccount('reauth@example.com', 'workspace-reauth');
+
+      global.fetch = mock(() => Promise.resolve(new Response('', { status: 401 }))) as typeof fetch;
+
+      const result = await fetchCodexQuota('reauth@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(401);
+      expect(result.errorCode).toBe('reauth_required');
+      expect(result.needsReauth).toBe(true);
+      expect(result.actionHint).toContain('ccs cliproxy auth codex');
+    });
+
+    it('maps 403 responses to forbidden metadata', async () => {
+      createValidCodexAccount('forbidden@example.com', 'workspace-forbidden');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({ detail: { code: 'quota_api_forbidden' } }), {
+            status: 403,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('forbidden@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(403);
+      expect(result.errorCode).toBe('quota_api_forbidden');
+      expect(result.isForbidden).toBe(true);
+      expect(result.retryable).toBe(false);
+    });
+
+    it('maps 429 responses to retryable rate-limit metadata', async () => {
+      createValidCodexAccount('rate-limit@example.com', 'workspace-rate-limit');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({ detail: { code: 'rate_limited' } }), {
+            status: 429,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('rate-limit@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(429);
+      expect(result.errorCode).toBe('rate_limited');
+      expect(result.retryable).toBe(true);
+      expect(result.actionHint).toContain('Retry');
+    });
+
+    it('maps 5xx responses to retryable provider-unavailable metadata', async () => {
+      createValidCodexAccount('outage@example.com', 'workspace-outage');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(JSON.stringify({ detail: { code: 'upstream_failure' } }), {
+            status: 503,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('outage@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(503);
+      expect(result.errorCode).toBe('upstream_failure');
+      expect(result.retryable).toBe(true);
+      expect(result.error).toContain('service unavailable');
+    });
+
+    it('maps unknown upstream statuses to a non-retryable structured error', async () => {
+      createValidCodexAccount('teapot@example.com', 'workspace-teapot');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response('{"message":"Strange upstream response"}', {
+            status: 418,
+            headers: { 'Content-Type': 'application/json' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('teapot@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.httpStatus).toBe(418);
+      expect(result.errorCode).toBe('unknown_upstream_error');
+      expect(result.retryable).toBe(false);
+      expect(result.error).toBe('Strange upstream response');
+    });
+
+    it('sanitizes and truncates raw upstream error detail before returning it', async () => {
+      createValidCodexAccount('sanitized@example.com', 'workspace-sanitized');
+      const leakedToken = 'secret-token-value-123';
+      const oversizedMessage = 'x'.repeat(400);
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              message: 'Upstream failure',
+              access_token: leakedToken,
+              extra: oversizedMessage,
+            }),
+            {
+              status: 418,
+              headers: { 'Content-Type': 'application/json' },
+            }
+          )
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('sanitized@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.errorDetail).toBeDefined();
+      expect(result.errorDetail).not.toContain(leakedToken);
+      expect(result.errorDetail).toContain('[redacted]');
+      expect(result.errorDetail?.endsWith('...[truncated]')).toBe(true);
+    });
+
+    it('omits raw HTML upstream bodies from the returned error detail', async () => {
+      createValidCodexAccount('html@example.com', 'workspace-html');
+
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response('<html><body>bad gateway</body></html>', {
+            status: 503,
+            headers: { 'Content-Type': 'text/html' },
+          })
+        )
+      ) as typeof fetch;
+
+      const result = await fetchCodexQuota('html@example.com');
+
+      expect(result.success).toBe(false);
+      expect(result.errorDetail).toBe('[HTML error response omitted]');
+    });
+  });
 });

tests/unit/cliproxy/quota-fetcher-gemini-cli.test.ts

@@ -4,13 +4,75 @@
  * Tests for Gemini CLI bucket parsing and transformation logic
  */
 
-import { describe, it, expect } from 'bun:test';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import {
   buildGeminiCliBuckets,
   resolveGeminiCliProjectId,
 } from '../../../src/cliproxy/quota-fetcher-gemini-cli';
+import { refreshGeminiToken } from '../../../src/cliproxy/auth/gemini-token-refresh';
+import { getProviderAuthDir } from '../../../src/cliproxy/config-generator';
+import { getCapturedFetchRequests, mockFetch, restoreFetch } from '../../mocks';
 
 describe('Gemini CLI Quota Fetcher', () => {
+  let tempHome: string;
+  let originalHome: string | undefined;
+  let originalCcsHome: string | undefined;
+  let originalGeminiClientId: string | undefined;
+  let originalGeminiClientSecret: string | undefined;
+
+  function writeGeminiToken(token: Record<string, unknown>): string {
+    const authDir = getProviderAuthDir('gemini');
+    fs.mkdirSync(authDir, { recursive: true });
+    const tokenPath = path.join(authDir, 'gemini-test.json');
+    fs.writeFileSync(tokenPath, JSON.stringify(token, null, 2));
+    return tokenPath;
+  }
+
+  beforeEach(() => {
+    tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-gemini-refresh-'));
+    originalHome = process.env.HOME;
+    originalCcsHome = process.env.CCS_HOME;
+    originalGeminiClientId = process.env.CCS_GEMINI_OAUTH_CLIENT_ID;
+    originalGeminiClientSecret = process.env.CCS_GEMINI_OAUTH_CLIENT_SECRET;
+
+    process.env.HOME = tempHome;
+    process.env.CCS_HOME = tempHome;
+    delete process.env.CCS_GEMINI_OAUTH_CLIENT_ID;
+    delete process.env.CCS_GEMINI_OAUTH_CLIENT_SECRET;
+  });
+
+  afterEach(() => {
+    restoreFetch();
+    fs.rmSync(tempHome, { recursive: true, force: true });
+
+    if (originalHome === undefined) {
+      delete process.env.HOME;
+    } else {
+      process.env.HOME = originalHome;
+    }
+
+    if (originalCcsHome === undefined) {
+      delete process.env.CCS_HOME;
+    } else {
+      process.env.CCS_HOME = originalCcsHome;
+    }
+
+    if (originalGeminiClientId === undefined) {
+      delete process.env.CCS_GEMINI_OAUTH_CLIENT_ID;
+    } else {
+      process.env.CCS_GEMINI_OAUTH_CLIENT_ID = originalGeminiClientId;
+    }
+
+    if (originalGeminiClientSecret === undefined) {
+      delete process.env.CCS_GEMINI_OAUTH_CLIENT_SECRET;
+    } else {
+      process.env.CCS_GEMINI_OAUTH_CLIENT_SECRET = originalGeminiClientSecret;
+    }
+  });
+
   describe('resolveGeminiCliProjectId', () => {
     it('should extract project ID from account field', () => {
       const account = 'user@example.com (cloudaicompanion-abc-123)';
@@ -191,4 +253,85 @@ describe('Gemini CLI Quota Fetcher', () => {
       expect(flashBucket!.modelIds).toContain('gemini-2.5-flash');
     });
   });
+
+  describe('refreshGeminiToken', () => {
+    it('uses OAuth client metadata stored in the token file', async () => {
+      writeGeminiToken({
+        type: 'gemini',
+        email: 'file@example.com',
+        token: {
+          access_token: 'old-token',
+          refresh_token: 'refresh-from-file',
+          expiry: Date.now() - 1000,
+          client_id: 'file-client-id',
+          client_secret: 'file-client-secret',
+          token_uri: 'https://oauth2.googleapis.com/token',
+        },
+      });
+
+      mockFetch([
+        {
+          url: 'https://oauth2.googleapis.com/token',
+          method: 'POST',
+          response: { access_token: 'fresh-token', expires_in: 1800 },
+        },
+      ]);
+
+      const result = await refreshGeminiToken();
+
+      expect(result.success).toBe(true);
+      const [request] = getCapturedFetchRequests();
+      expect(request.body).toContain('client_id=file-client-id');
+      expect(request.body).toContain('client_secret=file-client-secret');
+      expect(request.body).toContain('refresh_token=refresh-from-file');
+    });
+
+    it('falls back to CCS_GEMINI_OAUTH_CLIENT_* env vars when token metadata is missing', async () => {
+      process.env.CCS_GEMINI_OAUTH_CLIENT_ID = 'env-client-id';
+      process.env.CCS_GEMINI_OAUTH_CLIENT_SECRET = 'env-client-secret';
+
+      writeGeminiToken({
+        type: 'gemini',
+        email: 'env@example.com',
+        token: {
+          access_token: 'old-token',
+          refresh_token: 'refresh-from-file',
+          expiry: Date.now() - 1000,
+        },
+      });
+
+      mockFetch([
+        {
+          url: 'https://oauth2.googleapis.com/token',
+          method: 'POST',
+          response: { access_token: 'fresh-token', expires_in: 1800 },
+        },
+      ]);
+
+      const result = await refreshGeminiToken();
+
+      expect(result.success).toBe(true);
+      const [request] = getCapturedFetchRequests();
+      expect(request.body).toContain('client_id=env-client-id');
+      expect(request.body).toContain('client_secret=env-client-secret');
+    });
+
+    it('returns a clear error when no refresh client credentials are available', async () => {
+      writeGeminiToken({
+        type: 'gemini',
+        email: 'missing@example.com',
+        token: {
+          access_token: 'old-token',
+          refresh_token: 'refresh-from-file',
+          expiry: Date.now() - 1000,
+        },
+      });
+
+      const result = await refreshGeminiToken();
+
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('CCS_GEMINI_OAUTH_CLIENT_ID');
+      expect(result.error).toContain('CCS_GEMINI_OAUTH_CLIENT_SECRET');
+    });
+  });
 });