feat: integrate models.dev pricing metadata

This commit is contained in:
Tam Nhu Tran committed 2026-04-28 14:50:26 -04:00
1 parent 55ddb69104
commit a38c1a75ba
14 files changed
+851 -88

No files matched your search

+89 -1
View File
@@ -1,7 +1,10 @@
/**
* Unit tests for model-pricing.ts
*/
import { describe, it, expect } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { afterEach, beforeEach, describe, it, expect } from 'bun:test';
import {
getModelPricing,
calculateCost,
@@ -9,6 +12,10 @@ import {
hasCustomPricing,
type TokenUsage,
} from '../../src/web-server/model-pricing';
import {
clearModelsDevRegistryCache,
setCachedModelsDevRegistry,
} from '../../src/web-server/models-dev/registry-cache';
describe('model-pricing', () => {
describe('getModelPricing', () => {
@@ -297,4 +304,85 @@ describe('model-pricing', () => {
expect(hasCustomPricing('unknown-model-xyz')).toBe(false);
});
});
describe('models.dev cache integration', () => {
let tempRoot = '';
let originalCcsHome: string | undefined;
let originalCcsDir: string | undefined;
beforeEach(() => {
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-pricing-'));
originalCcsHome = process.env.CCS_HOME;
originalCcsDir = process.env.CCS_DIR;
process.env.CCS_HOME = tempRoot;
delete process.env.CCS_DIR;
clearModelsDevRegistryCache();
setCachedModelsDevRegistry({
openai: {
id: 'openai',
name: 'OpenAI',
models: {
'gpt-5.5': {
id: 'gpt-5.5',
name: 'GPT-5.5',
cost: { input: 5, output: 30, cache_read: 0.5 },
},
},
},
'github-copilot': {
id: 'github-copilot',
name: 'GitHub Copilot',
models: {
'gpt-5.5': {
id: 'gpt-5.5',
name: 'GPT-5.5',
cost: { input: 0, output: 0 },
},
},
},
});
});
afterEach(() => {
clearModelsDevRegistryCache();
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
else delete process.env.CCS_HOME;
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
else delete process.env.CCS_DIR;
fs.rmSync(tempRoot, { recursive: true, force: true });
});
it('resolves provider-prefixed paid API pricing from models.dev', () => {
const pricing = getModelPricing('openai/gpt-5.5');
expect(pricing.inputPerMillion).toBe(5);
expect(pricing.outputPerMillion).toBe(30);
expect(pricing.cacheReadPerMillion).toBe(0.5);
expect(pricing.cacheCreationPerMillion).toBe(0);
});
it('keeps subscription-backed provider pricing distinct from paid API pricing', () => {
const pricing = getModelPricing('gpt-5.5', { provider: 'github-copilot' });
expect(pricing.inputPerMillion).toBe(0);
expect(pricing.outputPerMillion).toBe(0);
});
it('does not use ambiguous model-only models.dev matches', () => {
const pricing = getModelPricing('gpt-5.5');
expect(pricing).toEqual(getModelPricing('unknown-model-xyz'));
expect(hasCustomPricing('gpt-5.5')).toBe(false);
expect(hasCustomPricing('gpt-5.5', { provider: 'openai' })).toBe(true);
});
it('calculates cost with provider-aware models.dev pricing', () => {
const usage: TokenUsage = {
inputTokens: 1_000_000,
outputTokens: 1_000_000,
cacheCreationTokens: 1_000_000,
cacheReadTokens: 1_000_000,
};
expect(calculateCost(usage, 'gpt-5.5', { provider: 'openai' })).toBe(35.5);
expect(calculateCost(usage, 'gpt-5.5', { provider: 'ghcp' })).toBe(0);
});
});
});
@@ -0,0 +1,93 @@
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import { getCcsDir } from '../../src/utils/config-manager';
import {
clearModelsDevRegistryCache,
getCachedModelsDevRegistry,
refreshModelsDevRegistry,
setCachedModelsDevRegistry,
} from '../../src/web-server/models-dev/registry-cache';
describe('models.dev registry cache', () => {
let tempRoot = '';
let originalCcsHome: string | undefined;
let originalCcsDir: string | undefined;
beforeEach(() => {
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-cache-'));
originalCcsHome = process.env.CCS_HOME;
originalCcsDir = process.env.CCS_DIR;
process.env.CCS_HOME = tempRoot;
delete process.env.CCS_DIR;
clearModelsDevRegistryCache();
});
afterEach(() => {
clearModelsDevRegistryCache();
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
else delete process.env.CCS_HOME;
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
else delete process.env.CCS_DIR;
fs.rmSync(tempRoot, { recursive: true, force: true });
});
it('normalizes and stores provider-keyed models.dev payloads', async () => {
const fetchImpl: typeof fetch = async () =>
new Response(
JSON.stringify({
openai: {
id: 'openai',
name: 'OpenAI',
models: {
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } },
},
},
ignored: { id: 'ignored' },
}),
{ status: 200 }
);
const registry = await refreshModelsDevRegistry({
force: true,
fetchImpl,
now: () => 123,
});
expect(registry?.openai.models?.['gpt-5.5']?.cost?.input).toBe(5);
expect(getCachedModelsDevRegistry({ allowStale: false, now: 123 })?.openai.id).toBe('openai');
});
it('uses stale cache when live refresh fails', async () => {
setCachedModelsDevRegistry(
{
openai: {
id: 'openai',
models: {
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } },
},
},
},
1
);
const fetchImpl: typeof fetch = async () => {
throw new Error('offline');
};
const registry = await refreshModelsDevRegistry({
force: true,
fetchImpl,
now: () => 1_000_000_000,
});
expect(registry?.openai.models?.['gpt-5.5']?.cost?.output).toBe(30);
});
it('ignores malformed cache files', () => {
fs.mkdirSync(getCcsDir(), { recursive: true });
fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json');
expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull();
});
});
@@ -1,5 +1,12 @@
import { describe, expect, it } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import type { CliproxyUsageApiResponse } from '../../../src/cliproxy/stats-fetcher';
import {
clearModelsDevRegistryCache,
setCachedModelsDevRegistry,
} from '../../../src/web-server/models-dev/registry-cache';
import {
buildCliproxyUsageHistoryAggregates,
extractCliproxyUsageHistoryDetails,
@@ -101,6 +108,7 @@ describe('cliproxy usage transformer', () => {
it('retains failed requests when they carry usage and skips zero-usage failures', () => {
const flat = extractCliproxyUsageHistoryDetails(sampleResponse);
expect(flat).toHaveLength(4);
expect(flat[0].provider).toBe('gemini');
expect(
flat.some(
(entry) =>
@@ -191,4 +199,102 @@ describe('cliproxy usage transformer', () => {
expect(monthly[0].outputTokens).toBe(110);
expect(monthly[0].cacheReadTokens).toBe(35);
});
describe('provider-aware pricing', () => {
let tempRoot = '';
let originalCcsHome: string | undefined;
let originalCcsDir: string | undefined;
beforeEach(() => {
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-cliproxy-models-dev-'));
originalCcsHome = process.env.CCS_HOME;
originalCcsDir = process.env.CCS_DIR;
process.env.CCS_HOME = tempRoot;
delete process.env.CCS_DIR;
setCachedModelsDevRegistry({
openai: {
id: 'openai',
models: {
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30, cache_read: 0.5 } },
},
},
'github-copilot': {
id: 'github-copilot',
models: {
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 0, output: 0 } },
},
},
});
});
afterEach(() => {
clearModelsDevRegistryCache();
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
else delete process.env.CCS_HOME;
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
else delete process.env.CCS_DIR;
fs.rmSync(tempRoot, { recursive: true, force: true });
});
it('keeps same model IDs separated by provider in CLIProxy usage', () => {
const response: CliproxyUsageApiResponse = {
usage: {
apis: {
openai: {
models: {
'gpt-5.5': {
details: [
{
timestamp: '2026-03-03T10:00:00.000Z',
source: 'api-account',
auth_index: 0,
tokens: {
input_tokens: 1_000_000,
output_tokens: 1_000_000,
reasoning_tokens: 0,
cached_tokens: 1_000_000,
total_tokens: 3_000_000,
},
failed: false,
},
],
},
},
},
'github-copilot': {
models: {
'gpt-5.5': {
details: [
{
timestamp: '2026-03-03T11:00:00.000Z',
source: 'copilot-account',
auth_index: 1,
tokens: {
input_tokens: 1_000_000,
output_tokens: 1_000_000,
reasoning_tokens: 0,
cached_tokens: 1_000_000,
total_tokens: 3_000_000,
},
failed: false,
},
],
},
},
},
},
},
};
const [daily] = transformCliproxyToDailyUsage(response);
const paid = daily.modelBreakdowns.find((breakdown) => breakdown.provider === 'openai');
const subscription = daily.modelBreakdowns.find(
(breakdown) => breakdown.provider === 'github-copilot'
);
expect(daily.totalCost).toBe(35.5);
expect(paid?.cost).toBe(35.5);
expect(subscription?.cost).toBe(0);
});
});
});