mirror of
https://github.com/tiennm99/ccs.git
synced 2026-10-19 20:14:43 +00:00
feat: integrate models.dev pricing metadata
This commit is contained in:
1 parent
55ddb69104
commit
a38c1a75ba
14 files changed
+851
-88
No files matched your search
@@ -1,7 +1,10 @@
|
||||
/**
|
||||
* Unit tests for model-pricing.ts
|
||||
*/
|
||||
import { describe, it, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { afterEach, beforeEach, describe, it, expect } from 'bun:test';
|
||||
import {
|
||||
getModelPricing,
|
||||
calculateCost,
|
||||
@@ -9,6 +12,10 @@ import {
|
||||
hasCustomPricing,
|
||||
type TokenUsage,
|
||||
} from '../../src/web-server/model-pricing';
|
||||
import {
|
||||
clearModelsDevRegistryCache,
|
||||
setCachedModelsDevRegistry,
|
||||
} from '../../src/web-server/models-dev/registry-cache';
|
||||
|
||||
describe('model-pricing', () => {
|
||||
describe('getModelPricing', () => {
|
||||
@@ -297,4 +304,85 @@ describe('model-pricing', () => {
|
||||
expect(hasCustomPricing('unknown-model-xyz')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('models.dev cache integration', () => {
|
||||
let tempRoot = '';
|
||||
let originalCcsHome: string | undefined;
|
||||
let originalCcsDir: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-pricing-'));
|
||||
originalCcsHome = process.env.CCS_HOME;
|
||||
originalCcsDir = process.env.CCS_DIR;
|
||||
process.env.CCS_HOME = tempRoot;
|
||||
delete process.env.CCS_DIR;
|
||||
clearModelsDevRegistryCache();
|
||||
setCachedModelsDevRegistry({
|
||||
openai: {
|
||||
id: 'openai',
|
||||
name: 'OpenAI',
|
||||
models: {
|
||||
'gpt-5.5': {
|
||||
id: 'gpt-5.5',
|
||||
name: 'GPT-5.5',
|
||||
cost: { input: 5, output: 30, cache_read: 0.5 },
|
||||
},
|
||||
},
|
||||
},
|
||||
'github-copilot': {
|
||||
id: 'github-copilot',
|
||||
name: 'GitHub Copilot',
|
||||
models: {
|
||||
'gpt-5.5': {
|
||||
id: 'gpt-5.5',
|
||||
name: 'GPT-5.5',
|
||||
cost: { input: 0, output: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearModelsDevRegistryCache();
|
||||
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
|
||||
else delete process.env.CCS_HOME;
|
||||
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
|
||||
else delete process.env.CCS_DIR;
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('resolves provider-prefixed paid API pricing from models.dev', () => {
|
||||
const pricing = getModelPricing('openai/gpt-5.5');
|
||||
expect(pricing.inputPerMillion).toBe(5);
|
||||
expect(pricing.outputPerMillion).toBe(30);
|
||||
expect(pricing.cacheReadPerMillion).toBe(0.5);
|
||||
expect(pricing.cacheCreationPerMillion).toBe(0);
|
||||
});
|
||||
|
||||
it('keeps subscription-backed provider pricing distinct from paid API pricing', () => {
|
||||
const pricing = getModelPricing('gpt-5.5', { provider: 'github-copilot' });
|
||||
expect(pricing.inputPerMillion).toBe(0);
|
||||
expect(pricing.outputPerMillion).toBe(0);
|
||||
});
|
||||
|
||||
it('does not use ambiguous model-only models.dev matches', () => {
|
||||
const pricing = getModelPricing('gpt-5.5');
|
||||
expect(pricing).toEqual(getModelPricing('unknown-model-xyz'));
|
||||
expect(hasCustomPricing('gpt-5.5')).toBe(false);
|
||||
expect(hasCustomPricing('gpt-5.5', { provider: 'openai' })).toBe(true);
|
||||
});
|
||||
|
||||
it('calculates cost with provider-aware models.dev pricing', () => {
|
||||
const usage: TokenUsage = {
|
||||
inputTokens: 1_000_000,
|
||||
outputTokens: 1_000_000,
|
||||
cacheCreationTokens: 1_000_000,
|
||||
cacheReadTokens: 1_000_000,
|
||||
};
|
||||
|
||||
expect(calculateCost(usage, 'gpt-5.5', { provider: 'openai' })).toBe(35.5);
|
||||
expect(calculateCost(usage, 'gpt-5.5', { provider: 'ghcp' })).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
||||
import { getCcsDir } from '../../src/utils/config-manager';
|
||||
import {
|
||||
clearModelsDevRegistryCache,
|
||||
getCachedModelsDevRegistry,
|
||||
refreshModelsDevRegistry,
|
||||
setCachedModelsDevRegistry,
|
||||
} from '../../src/web-server/models-dev/registry-cache';
|
||||
|
||||
describe('models.dev registry cache', () => {
|
||||
let tempRoot = '';
|
||||
let originalCcsHome: string | undefined;
|
||||
let originalCcsDir: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-models-dev-cache-'));
|
||||
originalCcsHome = process.env.CCS_HOME;
|
||||
originalCcsDir = process.env.CCS_DIR;
|
||||
process.env.CCS_HOME = tempRoot;
|
||||
delete process.env.CCS_DIR;
|
||||
clearModelsDevRegistryCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearModelsDevRegistryCache();
|
||||
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
|
||||
else delete process.env.CCS_HOME;
|
||||
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
|
||||
else delete process.env.CCS_DIR;
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('normalizes and stores provider-keyed models.dev payloads', async () => {
|
||||
const fetchImpl: typeof fetch = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
openai: {
|
||||
id: 'openai',
|
||||
name: 'OpenAI',
|
||||
models: {
|
||||
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } },
|
||||
},
|
||||
},
|
||||
ignored: { id: 'ignored' },
|
||||
}),
|
||||
{ status: 200 }
|
||||
);
|
||||
|
||||
const registry = await refreshModelsDevRegistry({
|
||||
force: true,
|
||||
fetchImpl,
|
||||
now: () => 123,
|
||||
});
|
||||
|
||||
expect(registry?.openai.models?.['gpt-5.5']?.cost?.input).toBe(5);
|
||||
expect(getCachedModelsDevRegistry({ allowStale: false, now: 123 })?.openai.id).toBe('openai');
|
||||
});
|
||||
|
||||
it('uses stale cache when live refresh fails', async () => {
|
||||
setCachedModelsDevRegistry(
|
||||
{
|
||||
openai: {
|
||||
id: 'openai',
|
||||
models: {
|
||||
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30 } },
|
||||
},
|
||||
},
|
||||
},
|
||||
1
|
||||
);
|
||||
|
||||
const fetchImpl: typeof fetch = async () => {
|
||||
throw new Error('offline');
|
||||
};
|
||||
|
||||
const registry = await refreshModelsDevRegistry({
|
||||
force: true,
|
||||
fetchImpl,
|
||||
now: () => 1_000_000_000,
|
||||
});
|
||||
|
||||
expect(registry?.openai.models?.['gpt-5.5']?.cost?.output).toBe(30);
|
||||
});
|
||||
|
||||
it('ignores malformed cache files', () => {
|
||||
fs.mkdirSync(getCcsDir(), { recursive: true });
|
||||
fs.writeFileSync(path.join(getCcsDir(), 'models-dev-registry-cache.json'), '{not json');
|
||||
expect(getCachedModelsDevRegistry({ allowStale: true })).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,12 @@
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
||||
import type { CliproxyUsageApiResponse } from '../../../src/cliproxy/stats-fetcher';
|
||||
import {
|
||||
clearModelsDevRegistryCache,
|
||||
setCachedModelsDevRegistry,
|
||||
} from '../../../src/web-server/models-dev/registry-cache';
|
||||
import {
|
||||
buildCliproxyUsageHistoryAggregates,
|
||||
extractCliproxyUsageHistoryDetails,
|
||||
@@ -101,6 +108,7 @@ describe('cliproxy usage transformer', () => {
|
||||
it('retains failed requests when they carry usage and skips zero-usage failures', () => {
|
||||
const flat = extractCliproxyUsageHistoryDetails(sampleResponse);
|
||||
expect(flat).toHaveLength(4);
|
||||
expect(flat[0].provider).toBe('gemini');
|
||||
expect(
|
||||
flat.some(
|
||||
(entry) =>
|
||||
@@ -191,4 +199,102 @@ describe('cliproxy usage transformer', () => {
|
||||
expect(monthly[0].outputTokens).toBe(110);
|
||||
expect(monthly[0].cacheReadTokens).toBe(35);
|
||||
});
|
||||
|
||||
describe('provider-aware pricing', () => {
|
||||
let tempRoot = '';
|
||||
let originalCcsHome: string | undefined;
|
||||
let originalCcsDir: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-cliproxy-models-dev-'));
|
||||
originalCcsHome = process.env.CCS_HOME;
|
||||
originalCcsDir = process.env.CCS_DIR;
|
||||
process.env.CCS_HOME = tempRoot;
|
||||
delete process.env.CCS_DIR;
|
||||
setCachedModelsDevRegistry({
|
||||
openai: {
|
||||
id: 'openai',
|
||||
models: {
|
||||
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 5, output: 30, cache_read: 0.5 } },
|
||||
},
|
||||
},
|
||||
'github-copilot': {
|
||||
id: 'github-copilot',
|
||||
models: {
|
||||
'gpt-5.5': { id: 'gpt-5.5', cost: { input: 0, output: 0 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearModelsDevRegistryCache();
|
||||
if (originalCcsHome !== undefined) process.env.CCS_HOME = originalCcsHome;
|
||||
else delete process.env.CCS_HOME;
|
||||
if (originalCcsDir !== undefined) process.env.CCS_DIR = originalCcsDir;
|
||||
else delete process.env.CCS_DIR;
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('keeps same model IDs separated by provider in CLIProxy usage', () => {
|
||||
const response: CliproxyUsageApiResponse = {
|
||||
usage: {
|
||||
apis: {
|
||||
openai: {
|
||||
models: {
|
||||
'gpt-5.5': {
|
||||
details: [
|
||||
{
|
||||
timestamp: '2026-03-03T10:00:00.000Z',
|
||||
source: 'api-account',
|
||||
auth_index: 0,
|
||||
tokens: {
|
||||
input_tokens: 1_000_000,
|
||||
output_tokens: 1_000_000,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 1_000_000,
|
||||
total_tokens: 3_000_000,
|
||||
},
|
||||
failed: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
'github-copilot': {
|
||||
models: {
|
||||
'gpt-5.5': {
|
||||
details: [
|
||||
{
|
||||
timestamp: '2026-03-03T11:00:00.000Z',
|
||||
source: 'copilot-account',
|
||||
auth_index: 1,
|
||||
tokens: {
|
||||
input_tokens: 1_000_000,
|
||||
output_tokens: 1_000_000,
|
||||
reasoning_tokens: 0,
|
||||
cached_tokens: 1_000_000,
|
||||
total_tokens: 3_000_000,
|
||||
},
|
||||
failed: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const [daily] = transformCliproxyToDailyUsage(response);
|
||||
const paid = daily.modelBreakdowns.find((breakdown) => breakdown.provider === 'openai');
|
||||
const subscription = daily.modelBreakdowns.find(
|
||||
(breakdown) => breakdown.provider === 'github-copilot'
|
||||
);
|
||||
|
||||
expect(daily.totalCost).toBe(35.5);
|
||||
expect(paid?.cost).toBe(35.5);
|
||||
expect(subscription?.cost).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user