feat(image): add native-read controls and autosave settings

- add per-profile native image preferences and native-capability detection
- redesign Settings -> Image around compact sections with Web-style autosave
- expose richer backend/profile status and update Gemini Flash defaults
This commit is contained in:
Tam Nhu Tran committed 2026-04-01 15:32:37 -04:00
1 parent 9d1d281e34
commit 982ffc5895
30 files changed
+1893 -708

No files matched your search

+21
View File
@@ -9,6 +9,7 @@
* - WebSearch and ImageAnalysis hook integration
*/
import * as fs from 'fs';
import {
getEffectiveEnvVars,
getRemoteEnvVars,
@@ -29,6 +30,7 @@ import { ToolSanitizationProxy } from '../tool-sanitization-proxy';
import { HttpsTunnelProxy } from '../https-tunnel-proxy';
import { MODEL_ENV_VAR_KEYS, normalizeModelIdForProvider } from '../model-id-normalizer';
import type { ProxyTarget } from '../proxy-target-resolver';
import { isSettings, type Settings } from '../../types/config';
export interface RemoteProxyConfig {
host: string;
@@ -129,17 +131,36 @@ function normalizeCodexEnvForDirectUpstream(envVars: NodeJS.ProcessEnv): NodeJS.
return nextEnv ?? envVars;
}
function loadImageAnalysisSettings(settingsPath?: string): Settings | undefined {
if (!settingsPath) {
return undefined;
}
try {
if (!fs.existsSync(settingsPath)) {
return undefined;
}
const parsed = JSON.parse(fs.readFileSync(settingsPath, 'utf8')) as unknown;
return isSettings(parsed) ? parsed : undefined;
} catch {
return undefined;
}
}
export async function resolveCliproxyImageAnalysisEnv(
options: ResolveCliproxyImageAnalysisEnvOptions,
deps: Partial<CliproxyImageAnalysisDeps> = {}
): Promise<CliproxyImageAnalysisResolution> {
const resolvedDeps = { ...defaultCliproxyImageAnalysisDeps, ...deps };
const settings = loadImageAnalysisSettings(options.profileSettingsPath);
const context = {
profileName: options.profileName,
profileType: 'cliproxy' as const,
cliproxyProvider: options.provider,
isComposite: options.isComposite,
settingsPath: options.profileSettingsPath,
settings,
hookInstalled: resolvedDeps.hasImageAnalysisProfileHook(
options.profileName,
options.profileSettingsPath
+39
View File
@@ -59,6 +59,8 @@ export interface ModelEntry {
thinking?: ThinkingSupport;
/** Whether model supports 1M extended context window (appends [1m] suffix) */
extendedContext?: boolean;
/** Whether model can read image inputs natively without the Image transformer */
nativeImageInput?: boolean;
}
/**
@@ -86,6 +88,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-opus-4-6-thinking',
name: 'Claude Opus 4.6 Thinking',
description: 'Latest flagship, extended thinking',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -101,6 +104,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-sonnet-4-6',
name: 'Claude Sonnet 4.6',
description: 'Latest Sonnet with thinking budget support',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -113,6 +117,15 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'gemini-3.1-pro-preview',
name: 'Gemini 3.1 Pro',
description: 'Google latest Gemini Pro model via Antigravity',
nativeImageInput: true,
thinking: { type: 'levels', levels: ['low', 'high'], dynamicAllowed: true },
extendedContext: true,
},
{
id: 'gemini-3-1-flash-preview',
name: 'Gemini Flash',
description: 'Latest Gemini Flash model via Antigravity',
nativeImageInput: true,
thinking: { type: 'levels', levels: ['low', 'high'], dynamicAllowed: true },
extendedContext: true,
},
@@ -128,6 +141,16 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
name: 'Gemini 3.1 Pro',
tier: 'pro',
description: 'Latest Gemini Pro model, requires paid Google account',
nativeImageInput: true,
thinking: { type: 'levels', levels: ['low', 'high'], dynamicAllowed: true },
extendedContext: true,
},
{
id: 'gemini-3-flash-preview',
name: 'Gemini Flash',
tier: 'pro',
description: 'Latest Gemini Flash model, requires paid Google account',
nativeImageInput: true,
thinking: { type: 'levels', levels: ['low', 'high'], dynamicAllowed: true },
extendedContext: true,
},
@@ -135,6 +158,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'gemini-2.5-pro',
name: 'Gemini 2.5 Pro',
description: 'Stable, works with free Google account',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 128,
@@ -264,6 +288,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'kimi-k2.5',
name: 'Kimi K2.5',
description: 'Latest multimodal model (262K context)',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -300,6 +325,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-opus-4-6',
name: 'Claude Opus 4.6',
description: 'Latest flagship model',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -313,6 +339,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-sonnet-4-6',
name: 'Claude Sonnet 4.6',
description: 'Balanced performance and speed',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -326,6 +353,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-opus-4-5-20251101',
name: 'Claude Opus 4.5',
description: 'Most capable Claude model',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -339,6 +367,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-sonnet-4-5-20250929',
name: 'Claude Sonnet 4.5',
description: 'Balanced performance and speed',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -352,6 +381,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-sonnet-4-20250514',
name: 'Claude Sonnet 4',
description: 'Previous generation Sonnet',
nativeImageInput: true,
thinking: {
type: 'budget',
min: 1024,
@@ -365,6 +395,7 @@ export const MODEL_CATALOG: Partial<Record<CLIProxyProvider, ProviderCatalog>> =
id: 'claude-haiku-4-5-20251001',
name: 'Claude Haiku 4.5',
description: 'Fast and efficient',
nativeImageInput: true,
thinking: { type: 'none' },
},
],
@@ -514,6 +545,14 @@ export function supportsExtendedContext(provider: CLIProxyProvider, modelId: str
return model?.extendedContext === true;
}
/**
* Check if a model can read image inputs natively.
*/
export function supportsNativeImageInput(provider: CLIProxyProvider, modelId: string): boolean {
const model = findModel(provider, modelId);
return model?.nativeImageInput === true;
}
/**
* Check if model is a native Gemini model (not Claude via Antigravity).
* Native Gemini models get extended context auto-enabled.
@@ -140,7 +140,9 @@ function showHelp(): void {
if (IMAGE_ANALYSIS_PROVIDER_ALIASES.length > 0) {
console.log(` ${dim(`Aliases accepted: ${IMAGE_ANALYSIS_PROVIDER_ALIASES.join(', ')}`)}`);
}
console.log(` ${dim('Default model: gemini-2.5-flash (most providers)')}`);
console.log(
` ${dim('Defaults: agy -> gemini-3-1-flash-preview, gemini -> gemini-3-flash-preview')}`
);
console.log('');
console.log(subheader('Examples:'));
+2 -2
View File
@@ -773,8 +773,8 @@ export const DEFAULT_IMAGE_ANALYSIS_CONFIG: ImageAnalysisConfig = {
enabled: true,
timeout: 60,
provider_models: {
agy: 'gemini-2.5-flash',
gemini: 'gemini-2.5-flash',
agy: 'gemini-3-1-flash-preview',
gemini: 'gemini-3-flash-preview',
codex: 'gpt-5.1-codex-mini',
kiro: 'kiro-claude-haiku-4-5',
ghcp: 'claude-haiku-4.5',
@@ -42,7 +42,7 @@ export async function runImageAnalysisCheck(results: HealthCheck): Promise<void>
results.errors.push({
name: 'Image Analysis',
message: 'No provider models configured for image analysis',
fix: 'ccs config image-analysis --set-model agy gemini-2.5-flash',
fix: 'ccs config image-analysis --set-model agy gemini-3-1-flash-preview',
});
console.log(` ${warn('Providers:')} None configured`);
return;
+6
View File
@@ -75,6 +75,10 @@ export interface ModelPreset {
haiku: string;
}
export interface CcsImageSettings {
native_read?: boolean;
}
/**
* Claude CLI settings.json structure
* Located at: ~/.claude/settings.json or profile-specific
@@ -83,6 +87,8 @@ export interface Settings {
env?: EnvVars;
/** Saved model presets for this provider */
presets?: ModelPreset[];
/** CCS-only per-profile Image preferences */
ccs_image?: CcsImageSettings;
[key: string]: unknown; // Allow other settings
}
@@ -7,10 +7,12 @@ import {
isCLIProxyProvider,
mapExternalProviderName,
} from '../../cliproxy/provider-capabilities';
import { getProviderCatalog, supportsNativeImageInput } from '../../cliproxy/model-catalog';
import { extractProviderFromPathname } from '../../cliproxy/model-id-normalizer';
import type { CliproxyBridgeMetadata } from '../../api/services/profile-types';
import type { Settings } from '../../types/config';
import type { ProfileType } from '../../types/profile';
import { stripModelConfigurationSuffixes } from '../../shared/extended-context-utils';
export type ImageAnalysisResolutionSource =
| 'cliproxy-provider'
@@ -20,6 +22,7 @@ export type ImageAnalysisResolutionSource =
| 'cliproxy-bridge'
| 'profile-backend'
| 'fallback-backend'
| 'native-compatible'
| 'disabled'
| 'unsupported-profile'
| 'unresolved'
@@ -49,7 +52,7 @@ export interface ImageAnalysisResolutionContext {
settingsPath?: string | null;
cliproxyProvider?: string | null;
isComposite?: boolean;
settings?: Pick<Settings, 'env'> | null;
settings?: Pick<Settings, 'env' | 'ccs_image'> | null;
cliproxyBridge?: CliproxyBridgeMetadata | null;
hookInstalled?: boolean;
sharedHookInstalled?: boolean;
@@ -79,8 +82,26 @@ export interface ImageAnalysisStatus {
proxyReason: string | null;
effectiveRuntimeMode: ImageAnalysisEffectiveRuntimeMode;
effectiveRuntimeReason: string | null;
profileModel: string | null;
nativeReadPreference: boolean;
nativeImageCapable: boolean | null;
nativeImageReason: string | null;
}
interface NativeImageSupportResolution {
profileModel: string | null;
nativeReadPreference: boolean;
nativeImageCapable: boolean | null;
nativeImageReason: string | null;
}
const PROFILE_MODEL_ENV_KEYS = [
'ANTHROPIC_MODEL',
'ANTHROPIC_DEFAULT_OPUS_MODEL',
'ANTHROPIC_DEFAULT_SONNET_MODEL',
'ANTHROPIC_DEFAULT_HAIKU_MODEL',
] as const;
function resolveProviderFromBaseUrl(baseUrl: unknown): string | null {
if (typeof baseUrl !== 'string' || baseUrl.trim().length === 0) {
return null;
@@ -233,9 +254,102 @@ function getRuntimePath(backendId: string | null): string | null {
return `/api/provider/${backendId}`;
}
function resolveBackend(
function resolveNativeImageProvider(
context: ImageAnalysisResolutionContext,
knownBackends: string[]
): string | null {
return normalizeImageAnalysisBackendId(
context.cliproxyProvider ??
context.cliproxyBridge?.provider ??
resolveProviderFromBaseUrl(context.settings?.env?.ANTHROPIC_BASE_URL ?? undefined),
knownBackends
);
}
function resolveProfileModel(
context: ImageAnalysisResolutionContext,
provider: string | null
): string | null {
const env = context.settings?.env;
if (env && typeof env === 'object') {
for (const key of PROFILE_MODEL_ENV_KEYS) {
const value = env[key];
if (typeof value === 'string' && value.trim().length > 0) {
return value.trim();
}
}
}
if (provider && isCLIProxyProvider(provider)) {
return getProviderCatalog(provider)?.defaultModel ?? null;
}
return null;
}
function verifyNativeImageCapability(
provider: string | null,
modelId: string | null
): boolean | null {
if (!modelId) {
return null;
}
if (provider && isCLIProxyProvider(provider) && supportsNativeImageInput(provider, modelId)) {
return true;
}
const normalizedModel = stripModelConfigurationSuffixes(modelId).trim().toLowerCase();
if (!normalizedModel) {
return null;
}
if (
normalizedModel.startsWith('gemini-') ||
normalizedModel.startsWith('claude-') ||
normalizedModel.startsWith('gpt-4o') ||
normalizedModel.includes('vision') ||
normalizedModel.includes('multimodal') ||
/(^|[-_.])vl([-. _]|$)/.test(normalizedModel) ||
/^glm-[\d.]+v([-. _]|$)/.test(normalizedModel)
) {
return true;
}
return null;
}
function resolveNativeImageSupport(
context: ImageAnalysisResolutionContext,
config: ImageAnalysisConfig
): NativeImageSupportResolution {
const knownBackends = Object.keys(config.provider_models);
const provider = resolveNativeImageProvider(context, knownBackends);
const profileModel = resolveProfileModel(context, provider);
const nativeReadPreference = context.settings?.ccs_image?.native_read === true;
const nativeImageCapable = verifyNativeImageCapability(provider, profileModel);
let nativeImageReason: string | null = null;
if (!profileModel) {
nativeImageReason = 'No current model is configured for this profile yet.';
} else if (nativeImageCapable) {
nativeImageReason = `${profileModel} can read images natively.`;
} else {
nativeImageReason = `CCS cannot verify native image support for ${profileModel} yet.`;
}
return {
profileModel,
nativeReadPreference,
nativeImageCapable,
nativeImageReason,
};
}
function resolveBackend(
context: ImageAnalysisResolutionContext,
config: ImageAnalysisConfig,
nativeSupport: NativeImageSupportResolution
): Pick<ImageAnalysisStatus, 'backendId' | 'backendDisplayName' | 'resolutionSource' | 'reason'> {
const { profileName, profileType, cliproxyProvider, isComposite, cliproxyBridge, settings } =
context;
@@ -258,6 +372,30 @@ function resolveBackend(
};
}
if (nativeSupport.nativeReadPreference) {
return {
backendId: null,
backendDisplayName: null,
resolutionSource: 'native-compatible',
reason:
nativeSupport.nativeImageCapable === true
? 'This profile is set to use native image reading.'
: `${nativeSupport.nativeImageReason ?? 'Native image reading is enabled for this profile.'} CCS will bypass the transformer for this profile.`,
};
}
// Explicit profile mappings are the only user-authored override and must
// win before provider/bridge inference.
const mappedBackend = resolveConfiguredProfileBackend(profileName, config);
if (mappedBackend) {
return {
backendId: mappedBackend,
backendDisplayName: getBackendDisplayName(mappedBackend),
resolutionSource: 'profile-backend',
reason: null,
};
}
if (profileType === 'copilot' || profileName === 'copilot') {
const backendId = normalizeImageAnalysisBackendId('ghcp', Object.keys(config.provider_models));
return {
@@ -300,16 +438,6 @@ function resolveBackend(
};
}
const mappedBackend = resolveConfiguredProfileBackend(profileName, config);
if (mappedBackend) {
return {
backendId: mappedBackend,
backendDisplayName: getBackendDisplayName(mappedBackend),
resolutionSource: 'profile-backend',
reason: null,
};
}
const hasDirectAnthropicApiKey = Boolean(settings?.env?.ANTHROPIC_API_KEY?.trim());
const hasBaseUrl = Boolean(settings?.env?.ANTHROPIC_BASE_URL?.trim());
if (hasDirectAnthropicApiKey && !hasBaseUrl) {
@@ -347,7 +475,8 @@ export function resolveImageAnalysisStatus(
rawConfig: ImageAnalysisConfig = DEFAULT_IMAGE_ANALYSIS_CONFIG
): ImageAnalysisStatus {
const config = canonicalizeImageAnalysisConfig(rawConfig);
const resolution = resolveBackend(context, config);
const nativeSupport = resolveNativeImageSupport(context, config);
const resolution = resolveBackend(context, config, nativeSupport);
const model = resolution.backendId
? (config.provider_models[resolution.backendId] ?? null)
: null;
@@ -444,5 +573,9 @@ export function resolveImageAnalysisStatus(
status === 'hook-missing' || !config.enabled || !resolution.backendId || !model
? reason
: null,
profileModel: nativeSupport.profileModel,
nativeReadPreference: nativeSupport.nativeReadPreference,
nativeImageCapable: nativeSupport.nativeImageCapable,
nativeImageReason: nativeSupport.nativeImageReason,
};
}
+30 -7
View File
@@ -27,7 +27,14 @@ const IMAGE_ANALYSIS_LOCAL_ACCESS_ERROR =
type DashboardTarget = 'claude' | 'droid' | 'codex';
type DashboardSummaryState = 'ready' | 'partial' | 'needs_setup' | 'disabled';
type BackendState = 'ready' | 'starts_on_launch' | 'needs_auth' | 'needs_proxy' | 'review';
type CurrentTargetMode = 'active' | 'bypassed' | 'fallback' | 'setup' | 'disabled' | 'unresolved';
type CurrentTargetMode =
| 'active'
| 'bypassed'
| 'fallback'
| 'setup'
| 'disabled'
| 'native'
| 'unresolved';
interface ImageAnalysisRouteBody {
enabled?: boolean;
@@ -74,8 +81,9 @@ function resolveCurrentTargetMode(
status: Awaited<ReturnType<typeof resolveImageAnalysisRuntimeStatus>>
): CurrentTargetMode {
if (!status.enabled) return 'disabled';
if (!status.backendId) return 'unresolved';
if (target !== 'claude') return 'bypassed';
if (status.nativeReadPreference) return 'native';
if (!status.backendId) return 'unresolved';
if (status.status === 'hook-missing') return 'setup';
if (status.effectiveRuntimeMode === 'native-read') return 'fallback';
return 'active';
@@ -140,6 +148,10 @@ async function buildDashboardPayload() {
effectiveRuntimeMode: status.effectiveRuntimeMode,
effectiveRuntimeReason: status.effectiveRuntimeReason,
currentTargetMode: resolveCurrentTargetMode(resolveTarget(profile.target), status),
profileModel: status.profileModel,
nativeReadPreference: status.nativeReadPreference,
nativeImageCapable: status.nativeImageCapable,
nativeImageReason: status.nativeImageReason,
};
})
);
@@ -180,6 +192,10 @@ async function buildDashboardPayload() {
effectiveRuntimeMode: status.effectiveRuntimeMode,
effectiveRuntimeReason: status.effectiveRuntimeReason,
currentTargetMode: resolveCurrentTargetMode(resolveTarget(variant.target), status),
profileModel: status.profileModel,
nativeReadPreference: status.nativeReadPreference,
nativeImageCapable: status.nativeImageCapable,
nativeImageReason: status.nativeImageReason,
};
})
);
@@ -212,7 +228,9 @@ async function buildDashboardPayload() {
authReason: status.authReason,
proxyReadiness: status.proxyReadiness,
proxyReason: status.proxyReason,
profilesUsing: allProfileRows.filter((profile) => profile.backendId === backendId).length,
profilesUsing: allProfileRows.filter(
(profile) => profile.backendId === backendId && !profile.nativeReadPreference
).length,
};
})
);
@@ -226,23 +244,27 @@ async function buildDashboardPayload() {
const mappedProfileCount = allProfileRows.filter(
(row) => row.resolutionSource === 'profile-backend'
).length;
const nativeProfileCount = allProfileRows.filter((row) => row.nativeReadPreference).length;
const blockerCount = backendRows.filter(
(row) => row.state === 'needs_auth' || row.state === 'needs_proxy' || row.state === 'review'
).length;
let summaryState: DashboardSummaryState = 'ready';
let title = 'Ready';
let detail = `${activeProfileCount} profile${activeProfileCount === 1 ? '' : 's'} can use Image Analysis on the current Claude target path.`;
let detail = `${activeProfileCount} profile${activeProfileCount === 1 ? '' : 's'} route through Image on the current Claude target path.`;
if (nativeProfileCount > 0) {
detail += ` ${nativeProfileCount} prefer native image reading.`;
}
if (!config.enabled) {
summaryState = 'disabled';
title = 'Disabled';
detail =
'Image Analysis is turned off globally. Images and PDFs fall back to native file access.';
detail = 'Image is turned off globally. Images and PDFs fall back to native file access.';
} else if (backendRows.length === 0) {
summaryState = 'needs_setup';
title = 'Needs provider models';
detail = 'Add at least one provider model before turning Image Analysis on for profiles.';
detail = 'Add at least one provider model before turning Image on for profiles.';
} else if (blockerCount > 0) {
summaryState = activeProfileCount > 0 ? 'partial' : 'needs_setup';
title = activeProfileCount > 0 ? 'Partially ready' : 'Needs setup';
@@ -265,6 +287,7 @@ async function buildDashboardPayload() {
mappedProfileCount,
activeProfileCount,
bypassedProfileCount,
nativeProfileCount,
},
backends: backendRows,
profiles: allProfileRows,