tiennm99/ccs
1
0 Fork 0
mirror of https://github.com/tiennm99/ccs.git synced 2026-10-18 12:13:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(image-analysis): add provider-backed runtime

Browse Source
This commit is contained in:
Tam Nhu Tran committed 2026-04-02 15:44:13 -04:00
1 parent 2c006afbb9
commit 813b2dd4d0
24 files changed
+1715 -487
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/hooks/image-analyzer-transformer.cjs
+27 -418
View File
@@ -24,7 +24,7 @@
const fs = require('fs');
const path = require('path');
const http = require('http');
const { analyzeFile, isAnalyzableFile, parseProviderModels } = require('./image-analysis-runtime.cjs');
// ============================================================================
// PLATFORM DETECTION
@@ -36,19 +36,8 @@ const isWindows = process.platform === 'win32';
// CONFIGURATION
// ============================================================================
const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.heic', '.bmp', '.tiff'];
const PDF_EXTENSIONS = ['.pdf'];
const DEFAULT_MODEL = 'gemini-2.5-flash';
const DEFAULT_TIMEOUT_SEC = 60;
const MAX_FILE_SIZE_MB = 10;
const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
const CLIPROXY_HOST = '127.0.0.1';
const CLIPROXY_PORT = parseInt(process.env.CCS_CLIPROXY_PORT || '8317', 10);
const CLIPROXY_PATH = '/v1/messages';
// API key passed via env from cliproxy-executor, defaults to CCS internal key
const CLIPROXY_API_KEY = process.env.CCS_CLIPROXY_API_KEY || 'ccs-internal-managed';
// ============================================================================
// ERROR CODES (for categorization)
@@ -65,23 +54,6 @@ const ERROR_CODES = {
UNKNOWN: 'UNKNOWN',
};
// Default analysis prompt
const DEFAULT_PROMPT = `Analyze this image/document thoroughly and provide a detailed description.
Include:
1. Overall content and purpose
2. Text content (if any) - transcribe important text
3. Visual elements (diagrams, charts, UI components)
4. Layout and structure
5. Colors, styling, notable design elements
6. Any actionable information (buttons, links, code)
Be comprehensive - this description replaces direct visual access.`;
// ============================================================================
// HELPER FUNCTIONS
// ============================================================================
/**
* Output debug information to stderr
* Only outputs when CCS_DEBUG=1
@@ -105,19 +77,19 @@ function debugLog(message, data = {}) {
*/
function getDebugContext(filePath, stats) {
const currentProvider = process.env.CCS_CURRENT_PROVIDER || 'unknown';
const providerModels = parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS);
const model = providerModels[currentProvider] || DEFAULT_MODEL;
const model =
process.env.CCS_IMAGE_ANALYSIS_MODEL ||
parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS)[currentProvider] ||
DEFAULT_MODEL;
const timeout = parseInt(process.env.CCS_IMAGE_ANALYSIS_TIMEOUT || DEFAULT_TIMEOUT_SEC, 10);
const modelsToTry = getModelsToTry();
return {
file: path.basename(filePath),
size: stats ? `${(stats.size / 1024).toFixed(1)} KB` : 'unknown',
provider: currentProvider,
model: model,
modelsToTry: modelsToTry.length > 1 ? modelsToTry.join(' -> ') : model,
timeout: `${timeout}s`,
endpoint: `http://${CLIPROXY_HOST}:${CLIPROXY_PORT}${CLIPROXY_PATH}`,
endpoint: process.env.CCS_IMAGE_ANALYSIS_RUNTIME_BASE_URL || '(runtime fallback)',
};
}
@@ -126,319 +98,12 @@ function getDebugContext(filePath, stats) {
*/
function getProviderContext() {
const provider = process.env.CCS_CURRENT_PROVIDER || 'unknown';
const providerModels = parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS);
const model = providerModels[provider] || DEFAULT_MODEL;
const model =
process.env.CCS_IMAGE_ANALYSIS_MODEL ||
parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS)[provider] ||
DEFAULT_MODEL;
return { provider, model };
}
/**
* Parse provider_models env var to object
* Format: provider:model,provider:model
*/
function parseProviderModels(envValue) {
if (!envValue) return {};
const result = {};
envValue.split(',').forEach((pair) => {
const [provider, model] = pair.split(':');
if (provider && model && model.trim()) {
result[provider.trim()] = model.trim();
}
});
return result;
}
/**
* Extract concatenated text content from CLIProxy response blocks.
* Skips thinking and other non-text blocks.
*/
function extractTextContent(response) {
if (!response || !Array.isArray(response.content)) {
return null;
}
const textBlocks = response.content
.filter((block) => block && block.type === 'text' && typeof block.text === 'string')
.map((block) => block.text)
.filter((text) => text.trim());
if (textBlocks.length === 0) {
return null;
}
return textBlocks.join('\n\n');
}
/**
* Parse raw CLIProxy response body and extract text content.
*/
function parseCliProxyResponse(data) {
let response;
try {
response = JSON.parse(data);
} catch (err) {
throw new Error(`Failed to parse response: ${err.message}`);
}
const text = extractTextContent(response);
if (!text) {
throw new Error('No text content in response');
}
return text;
}
/**
* Get model for current provider from provider_models mapping
* Returns primary model only (for display/logging)
*/
function getModelForProvider() {
const currentProvider = process.env.CCS_CURRENT_PROVIDER || '';
const providerModels = parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS);
return providerModels[currentProvider] || DEFAULT_MODEL;
}
/**
* Get list of models to try in order:
* 1. provider_models[current_provider] (if exists)
* 2. DEFAULT_MODEL when no provider-specific vision model is configured
*
* ANTHROPIC_MODEL is intentionally ignored here because the chat model may
* not be vision-capable on the current provider route.
*/
function getModelsToTry() {
const currentProvider = process.env.CCS_CURRENT_PROVIDER || '';
const providerModels = parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS);
const models = [];
const seen = new Set();
// 1. Provider-specific model
if (providerModels[currentProvider]) {
models.push(providerModels[currentProvider]);
seen.add(providerModels[currentProvider]);
}
// 2. Default model — only when no provider-specific model is configured,
// since DEFAULT_MODEL (gemini-2.5-flash) may not be routable on the
// current provider's CLIProxy endpoint (e.g. codex, claude)
if (models.length === 0 && !seen.has(DEFAULT_MODEL)) {
models.push(DEFAULT_MODEL);
seen.add(DEFAULT_MODEL);
}
return models;
}
/**
* Analyze with retry logic - tries models in order until one succeeds
*/
async function analyzeWithRetry(base64Data, mediaType, timeoutMs) {
const models = getModelsToTry();
// Defensive check - should never happen but provides clear error
if (models.length === 0) {
throw new Error('No models configured for image analysis');
}
let lastError = null;
for (let i = 0; i < models.length; i++) {
const model = models[i];
try {
debugLog(`Trying model ${i + 1}/${models.length}`, { model });
const result = await analyzeViaCliProxy(base64Data, mediaType, model, timeoutMs);
if (i > 0) {
debugLog('Retry succeeded', { model, attempt: i + 1 });
}
return { description: result, model };
} catch (err) {
lastError = err;
const isLastModel = i === models.length - 1;
// Don't retry on certain errors (auth, rate limit, timeout, file access, network)
const errMsg = err.message || '';
const noRetryPatterns = [
'AUTH_ERROR', 'RATE_LIMIT', 'TIMEOUT',
'EACCES', 'EPERM', 'ECONNREFUSED',
'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' // Network errors - no point retrying
];
const shouldNotRetry = noRetryPatterns.some(p => errMsg.includes(p));
if (shouldNotRetry || isLastModel) {
debugLog('Analysis failed, no more retries', {
model,
error: errMsg,
reason: shouldNotRetry ? 'non-retryable error' : 'last model'
});
throw err;
}
debugLog('Model failed, trying next', {
model,
error: errMsg.substring(0, 100),
nextModel: models[i + 1]
});
}
}
throw lastError || new Error('No models available');
}
/**
* Check if file is an analyzable image or PDF
*/
function isAnalyzableFile(filePath) {
const ext = path.extname(filePath).toLowerCase();
return IMAGE_EXTENSIONS.includes(ext) || PDF_EXTENSIONS.includes(ext);
}
/**
* Get MIME type from file extension
*/
function getMediaType(filePath) {
const ext = path.extname(filePath).toLowerCase();
const mimeTypes = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.webp': 'image/webp',
'.heic': 'image/heic',
'.bmp': 'image/bmp',
'.tiff': 'image/tiff',
'.pdf': 'application/pdf',
};
return mimeTypes[ext] || 'application/octet-stream';
}
/**
* Encode file to base64
*/
function encodeFileToBase64(filePath) {
const content = fs.readFileSync(filePath);
return content.toString('base64');
}
/**
* Check if CLIProxy is available
*/
function isCliProxyAvailable() {
return new Promise((resolve) => {
const req = http.request(
{
hostname: CLIPROXY_HOST,
port: CLIPROXY_PORT,
path: '/',
method: 'GET',
timeout: 2000,
},
(res) => {
resolve(res.statusCode >= 200 && res.statusCode < 500);
}
);
req.on('error', () => resolve(false));
req.on('timeout', () => {
req.destroy();
resolve(false);
});
req.end();
});
}
/**
* Analyze file via CLIProxy vision API
*/
function analyzeViaCliProxy(base64Data, mediaType, model, timeoutMs) {
return new Promise((resolve, reject) => {
const requestBody = JSON.stringify({
model: model,
max_tokens: 4096,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: DEFAULT_PROMPT },
{
type: 'image',
source: {
type: 'base64',
media_type: mediaType,
data: base64Data,
},
},
],
},
],
});
const req = http.request(
{
hostname: CLIPROXY_HOST,
port: CLIPROXY_PORT,
path: CLIPROXY_PATH,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(requestBody),
'x-api-key': CLIPROXY_API_KEY,
},
timeout: timeoutMs,
},
(res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('error', (err) => {
reject(err);
});
res.on('end', () => {
// Categorize by status code
if (res.statusCode === 401 || res.statusCode === 403) {
reject(new Error(`AUTH_ERROR:${res.statusCode}`));
return;
}
if (res.statusCode === 429) {
const retryAfter = res.headers['retry-after'];
reject(new Error(`RATE_LIMIT:${retryAfter || ''}`));
return;
}
if (res.statusCode !== 200) {
reject(new Error(`API_ERROR:${res.statusCode}:${data}`));
return;
}
if (!data || !data.trim()) {
reject(new Error('Empty response from CLIProxy'));
return;
}
try {
const text = parseCliProxyResponse(data);
resolve(text);
} catch (err) {
reject(err);
}
});
}
);
req.on('error', (err) => reject(err));
req.on('timeout', () => {
req.destroy();
reject(new Error('TIMEOUT'));
});
req.write(requestBody);
req.end();
});
}
/**
* Format analysis description for Claude (matches websearch format)
*/
@@ -520,29 +185,6 @@ function outputFileTooLargeError(filePath, actualSizeMB, maxSizeMB) {
process.exit(2);
}
/**
* CLIProxy unavailable error
*/
function outputCliProxyUnavailableError(filePath, endpoint) {
const output = formatErrorOutput(
filePath,
ERROR_CODES.CLIPROXY_UNAVAILABLE,
`CLIProxy not available at ${endpoint}`,
[
'CLIProxy service may not be running',
'Start with: ccs config (opens dashboard, starts CLIProxy)',
'Or manually: ccs cliproxy start',
`Verify: curl ${endpoint}`,
'Check status: ccs doctor',
]
);
console.log(JSON.stringify(output));
process.exit(2);
}
/**
* Authentication error
*/
function outputAuthError(filePath, statusCode) {
const { provider } = getProviderContext();
const output = formatErrorOutput(
@@ -758,10 +400,11 @@ function shouldSkipHook() {
}
// Check if current provider has a vision model configured
const explicitModel = process.env.CCS_IMAGE_ANALYSIS_MODEL;
const currentProvider = process.env.CCS_CURRENT_PROVIDER || '';
const providerModels = parseProviderModels(process.env.CCS_IMAGE_ANALYSIS_PROVIDER_MODELS);
if (!providerModels[currentProvider]) {
if (!explicitModel?.trim() && !providerModels[currentProvider]) {
debugLog(`Skipping: provider "${currentProvider}" not in provider_models`, {
configured_providers: Object.keys(providerModels).join(', ') || 'none',
});
@@ -832,57 +475,15 @@ async function processHook() {
process.exit(0);
}
// Check if file exists
if (!fs.existsSync(filePath)) {
// Let native Read handle the error
process.exit(0);
}
// Check file size
const stats = fs.statSync(filePath);
if (stats.size >= MAX_FILE_SIZE_BYTES) {
outputFileTooLargeError(filePath, stats.size / 1024 / 1024, MAX_FILE_SIZE_MB);
return;
}
const debugContext = getDebugContext(filePath, null);
debugLog('Image analysis runtime prepared', debugContext);
// Check CLIProxy availability
const cliProxyAvailable = await isCliProxyAvailable();
if (!cliProxyAvailable) {
debugLog('Blocking: CLIProxy not available', {
endpoint: `http://${CLIPROXY_HOST}:${CLIPROXY_PORT}`,
action: 'blocking to prevent context overflow',
});
outputCliProxyUnavailableFallback(filePath);
return;
}
const model = getModelForProvider();
const timeout = parseInt(process.env.CCS_IMAGE_ANALYSIS_TIMEOUT || DEFAULT_TIMEOUT_SEC, 10);
const timeoutMs = Math.max(1, Math.min(600, timeout)) * 1000;
// Get debug context before analysis
const debugContext = getDebugContext(filePath, stats);
debugLog('Starting image analysis', debugContext);
// Encode file to base64
const base64Data = encodeFileToBase64(filePath);
const mediaType = getMediaType(filePath);
debugLog('File encoded', {
mediaType: mediaType,
base64Length: `${(base64Data.length / 1024).toFixed(1)}KB`,
});
// Analyze via CLIProxy with retry logic
const { description, model: usedModel } = await analyzeWithRetry(base64Data, mediaType, timeoutMs);
debugLog('Analysis complete', {
responseLength: `${description.length} chars`,
model: usedModel,
});
// Output success
outputSuccess(filePath, description, usedModel, stats.size);
const result = await analyzeFile(filePath);
outputSuccess(filePath, result.description, result.model, result.fileSize);
} catch (err) {
if (process.env.CCS_DEBUG) {
console.error('[CCS Hook] Error:', err.message);
@@ -893,7 +494,10 @@ async function processHook() {
// Categorize error by message pattern
const errMsg = err.message || '';
if (errMsg.startsWith('AUTH_ERROR:')) {
if (errMsg.startsWith('FILE_TOO_LARGE:')) {
const fileSizeMb = Number.parseInt(errMsg.split(':')[1], 10) / 1024 / 1024;
outputFileTooLargeError(filePath, fileSizeMb, 10);
} else if (errMsg.startsWith('AUTH_ERROR:')) {
const statusCode = parseInt(errMsg.split(':')[1], 10);
outputAuthError(filePath, statusCode);
} else if (errMsg.startsWith('RATE_LIMIT:')) {
@@ -907,8 +511,13 @@ async function processHook() {
} else if (errMsg === 'TIMEOUT' || errMsg.includes('timed out') || errMsg.includes('timeout')) {
const timeout = parseInt(process.env.CCS_IMAGE_ANALYSIS_TIMEOUT || DEFAULT_TIMEOUT_SEC, 10);
outputTimeoutError(filePath, timeout);
} else if (errMsg.includes('ECONNREFUSED') || errMsg.includes('ENOTFOUND')) {
outputCliProxyUnavailableError(filePath, `http://${CLIPROXY_HOST}:${CLIPROXY_PORT}`);
} else if (
errMsg.includes('ECONNREFUSED') ||
errMsg.includes('ENOTFOUND') ||
errMsg.includes('ENETUNREACH') ||
errMsg.includes('EAI_AGAIN')
) {
outputCliProxyUnavailableFallback(filePath);
} else if (errMsg.includes('EACCES') || errMsg.includes('EPERM')) {
outputFileAccessError(filePath, errMsg);
} else {