Merge pull request #1699 from kaitranntt/kai/fix/cliproxy-safe-updates-dev

fix(cliproxy): keep proxy continuously recoverable
This commit is contained in:
Kai (Tam Nhu) Tran authored and GitHub committed 2026-08-10 22:20:13 -04:00
commit 2ef2612df8
21 files changed
+1289 -182

No files matched your search

+30 -2
View File
@@ -79,6 +79,32 @@ The `ccs docker` flow uses the integrated assets in this directory:
- `docker/supervisord.conf`
- `docker/entrypoint-integrated.sh`
### Optional Host Continuity Timers
For a persistent single-host stack, install the host reconciliation and safe-update scripts with
their systemd units:
```bash
sudo install -d /opt/cliproxy
sudo install -m 0755 docker/host/ccs-cliproxy-{reconcile,safe-update}.sh /opt/cliproxy/
sudo install -m 0644 docker/host/systemd/ccs-cliproxy-{reconcile,update}.{service,timer} /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now ccs-cliproxy-reconcile.timer ccs-cliproxy-update.timer
systemctl list-timers 'ccs-cliproxy-*'
journalctl -u ccs-cliproxy-reconcile.service -u ccs-cliproxy-update.service
```
By default, reconciliation expects `/root/.ccs/docker/docker-compose.integrated.yml` with Compose project
`docker`. Keep those defaults when adopting an existing stack so its `docker_ccs_home` and
`docker_ccs_logs` volumes remain attached. The scripts support environment overrides for the
container, Compose paths and project, lock file, and logs.
The updater downloads and verifies a replacement while the current proxy remains online, then
stops it only for the atomic swap and startup check. A failed update restores the previous binary
and version. Reconciliation probes the dashboard on port 3000 and CLIProxy on port 8317, then
escalates from supervised-process restart to container restart and finally Compose force-recreate.
### Network Binding and Dashboard Auth
The integrated Docker stack publishes the dashboard and CLIProxy ports on `127.0.0.1` by default. This keeps the services reachable from the Docker host and SSH tunnels without exposing them on every host interface.
@@ -575,9 +601,11 @@ docker exec ccs-cliproxy supervisorctl -c /etc/supervisord.conf restart cliproxy
docker exec ccs-cliproxy grep "client load complete" /var/log/ccs/cliproxy.log
```
### ETXTBSY Error on First Boot
### ETXTBSY During CLIProxy Update
On first container start, you may see `ETXTBSY: text file is busy` in dashboard logs. This is a known race condition where the dashboard tries to update the CLIProxy binary while it's already running. The dashboard recovers automatically on the next attempt. No action needed.
Current releases stage and verify CLIProxy updates before replacing the live binary atomically, so
`ETXTBSY: text file is busy` is not an expected first-boot condition. Upgrade older installations
that still report this error.
### Debug Mode
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env bash
set -Eeuo pipefail
container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker}"
compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.integrated.yml}"
compose_project="${CCS_CLIPROXY_COMPOSE_PROJECT:-docker}"
lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
log_file="${CCS_CLIPROXY_RECONCILE_LOG:-/var/log/ccs-cliproxy-reconcile.log}"
log() {
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$log_file"
}
exec 9>"$lock_file"
if ! flock -n 9; then
exit 0
fi
probe() {
docker exec "$container_name" sh -c \
'curl -fsS --max-time 2 http://127.0.0.1:3000/ >/dev/null && curl -fsS --max-time 2 http://127.0.0.1:8317/ >/dev/null' \
>/dev/null 2>&1
}
wait_for_health() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if probe; then
return 0
fi
attempts=$((attempts + 1))
sleep 2
done
return 1
}
compose_up() {
cd "$compose_dir"
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --no-build || \
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --build
}
compose_recreate() {
cd "$compose_dir"
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --force-recreate --no-build || \
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --force-recreate --build
}
running_state="$(docker inspect --format '{{.State.Running}}' "$container_name" 2>/dev/null || true)"
if [ -z "$running_state" ]; then
log "Container $container_name is missing; recreating from $compose_file"
compose_up
wait_for_health
log "Container $container_name was recreated and passed both health probes"
exit 0
fi
if [ "$running_state" != 'true' ]; then
log "Container $container_name is stopped; restoring the Compose service"
compose_up
wait_for_health
log "Container $container_name is running and passed both health probes"
exit 0
fi
if probe; then
exit 0
fi
sleep 10
if probe; then
exit 0
fi
log 'Dashboard or proxy failed two consecutive probes; restarting supervised processes'
if docker exec "$container_name" \
supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy; then
if wait_for_health; then
log 'Supervised processes recovered and passed both health probes'
exit 0
fi
fi
log 'Supervisor recovery failed; restarting the container'
if docker restart "$container_name" >/dev/null; then
if wait_for_health; then
log "Container $container_name recovered and passed both health probes"
exit 0
fi
fi
log "Container $container_name is still unhealthy; recreating it from $compose_file"
compose_recreate
wait_for_health
log "Container $container_name was recreated and passed both health probes"
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env bash
set -Eeuo pipefail
container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
log_file="${CCS_CLIPROXY_UPDATE_LOG:-/var/log/ccs-cliproxy-update.log}"
log() {
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$log_file"
}
exec 9>"$lock_file"
if ! flock -n 9; then
log 'Another CLIProxy maintenance operation is active; skipping update check'
exit 0
fi
if ! docker inspect "$container_name" >/dev/null 2>&1; then
log "Container $container_name is missing; reconciliation must restore it before updating"
exit 1
fi
if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'true' ]; then
log "Container $container_name is not running; reconciliation must restore it before updating"
exit 1
fi
status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus --verbose 2>&1)" || {
log "Unable to inspect the installed CLIProxy version: $status_output"
exit 1
}
if grep -qi 'Could not fetch' <<<"$status_output"; then
log "Unable to check the latest CLIProxy version: $(grep -im1 'Could not fetch' <<<"$status_output" | xargs)"
exit 1
fi
if ! grep -qi 'update available' <<<"$status_output"; then
exit 0
fi
log 'CLIProxy Plus update available; staging verified replacement while the current proxy stays online'
if ! docker exec -i "$container_name" sh -s <<'CONTAINER_UPDATE'
set -eu
live_dir='/root/.ccs/cliproxy/bin/plus'
live_binary="$live_dir/cli-proxy-api-plus"
live_version="$live_dir/.version"
stage_root="$(mktemp -d /root/.ccs/cliproxy/.host-update.XXXXXX)"
stage_ccs_dir="$stage_root/ccs"
stage_dir="$stage_ccs_dir/cliproxy/bin/plus"
stage_binary="$stage_dir/cli-proxy-api-plus"
stage_version="$stage_dir/.version"
backup_binary="$stage_root/previous-binary"
backup_version="$stage_root/previous-version"
install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'
install_lock_dir="$install_lock_target.lock"
install_lock_stale_seconds=600
install_lock_owned=0
maintenance_started=0
supervisorctl_cmd() {
supervisorctl -c /etc/supervisord.conf "$@"
}
cleanup() {
rm -rf -- "$stage_root"
}
remove_stale_install_lock() {
lock_mtime="$(
stat -c %Y "$install_lock_dir" 2>/dev/null || stat -f %m "$install_lock_dir" 2>/dev/null
)" || return 0
current_time="$(date +%s)"
lock_age=$((current_time - lock_mtime))
if [ "$lock_age" -gt "$install_lock_stale_seconds" ]; then
rmdir "$install_lock_dir" 2>/dev/null || true
fi
}
acquire_install_lock() {
mkdir -p "$install_lock_target"
attempts=0
while ! mkdir "$install_lock_dir" 2>/dev/null; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 240 ]; then
printf '[X] Timed out waiting for CLIProxy install lifecycle lock\n' >&2
return 1
fi
remove_stale_install_lock
sleep 0.25
done
install_lock_owned=1
touch "$install_lock_dir"
}
release_install_lock() {
if [ "$install_lock_owned" -eq 1 ]; then
rmdir "$install_lock_dir" 2>/dev/null || true
install_lock_owned=0
fi
}
wait_for_proxy() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if curl -fsS --max-time 2 http://127.0.0.1:8317/ >/dev/null; then
return 0
fi
attempts=$((attempts + 1))
sleep 2
done
return 1
}
rollback() {
rollback_ok=1
supervisorctl_cmd stop cliproxy >/dev/null 2>&1 || true
if [ -f "$backup_binary" ]; then
install -m 0755 "$backup_binary" "$live_binary.rollback" && \
mv -f "$live_binary.rollback" "$live_binary" || rollback_ok=0
else
rollback_ok=0
fi
if [ -f "$backup_version" ]; then
install -m 0644 "$backup_version" "$live_version.rollback" && \
mv -f "$live_version.rollback" "$live_version" || rollback_ok=0
else
rollback_ok=0
fi
supervisorctl_cmd start cliproxy >/dev/null || rollback_ok=0
wait_for_proxy || rollback_ok=0
[ "$rollback_ok" -eq 1 ]
}
on_exit() {
rc=$?
trap - EXIT INT TERM HUP
rollback_failed=0
if [ "$rc" -ne 0 ] && [ "$maintenance_started" -eq 1 ]; then
if ! rollback; then
rollback_failed=1
printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2
fi
fi
release_install_lock
if [ "$rollback_failed" -eq 0 ]; then
cleanup
else
rc=70
fi
exit "$rc"
}
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
trap 'exit 129' HUP
mkdir -p "$stage_ccs_dir"
CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest --backend plus
test -x "$stage_binary"
test -s "$stage_version"
"$stage_binary" --version >/dev/null
acquire_install_lock
cp -p "$live_binary" "$backup_binary"
cp -p "$live_version" "$backup_version"
maintenance_started=1
supervisorctl_cmd stop cliproxy >/dev/null
mv -f "$stage_binary" "$live_binary"
mv -f "$stage_version" "$live_version"
chmod 0755 "$live_binary"
supervisorctl_cmd start cliproxy >/dev/null
wait_for_proxy
maintenance_started=0
CONTAINER_UPDATE
then
log 'CLIProxy Plus update failed; rollback was attempted and reconciliation will verify service health'
exit 1
fi
installed_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)"
log "CLIProxy Plus update completed and passed health verification: $(grep -m1 'Version:' <<<"$installed_output" | xargs)"
@@ -0,0 +1,9 @@
[Unit]
Description=Reconcile the CCS CLIProxy Docker service
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh
TimeoutStartSec=10min
@@ -0,0 +1,11 @@
[Unit]
Description=Continuously reconcile the CCS CLIProxy Docker service
[Timer]
OnBootSec=30s
OnUnitActiveSec=30s
AccuracySec=5s
Persistent=true
[Install]
WantedBy=timers.target
@@ -0,0 +1,9 @@
[Unit]
Description=Safely update CLIProxy Plus in ccs-cliproxy
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
ExecStart=/opt/cliproxy/ccs-cliproxy-safe-update.sh
TimeoutStartSec=10min
@@ -0,0 +1,10 @@
[Unit]
Description=Poll for CLIProxy Plus releases without interrupting the live binary
[Timer]
OnBootSec=3min
OnUnitActiveSec=15min
Persistent=true
[Install]
WantedBy=timers.target
+30 -30
View File
@@ -1,10 +1,10 @@
{
"scope": "src/**/*.{ts,tsx,js,jsx,mjs,cjs}",
"syncFs": {
"totalOccurrences": 2471,
"filesAffected": 261,
"hotpathOccurrences": 1186,
"hotpathFilesAffected": 155,
"totalOccurrences": 2519,
"filesAffected": 263,
"hotpathOccurrences": 1193,
"hotpathFilesAffected": 156,
"topHotpathFiles": [
{
"file": "src/management/shared-manager/diverged-file-adopter.ts",
@@ -228,6 +228,22 @@
],
"markers": []
},
{
"file": "src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts",
"count": 41,
"calls": [
"existsSync",
"mkdirSync",
"mkdtempSync",
"readdirSync",
"readFileSync",
"renameSync",
"rmdirSync",
"rmSync",
"writeFileSync"
],
"markers": []
},
{
"file": "src/management/shared-manager/diverged-file-adopter.ts",
"count": 39,
@@ -286,22 +302,6 @@
"writeFileSync"
],
"markers": []
},
{
"file": "src/utils/browser/mcp-installer.ts",
"count": 32,
"calls": [
"chmodSync",
"copyFileSync",
"existsSync",
"mkdirSync",
"readFileSync",
"renameSync",
"statSync",
"unlinkSync",
"writeFileSync"
],
"markers": []
}
]
},
@@ -497,16 +497,16 @@
},
"maintainability": {
"typedErrors": {
"totalThrows": 454,
"typedThrows": 83,
"plainThrows": 311,
"totalThrows": 455,
"typedThrows": 89,
"plainThrows": 306,
"otherThrows": 60,
"adoptionRatio": 0.1828,
"adoptionRatio": 0.1956,
"topSubdomainsByThrows": [
{
"subdomain": "web-server",
"count": 103,
"typed": 16,
"count": 104,
"typed": 17,
"plain": 45
},
{
@@ -579,8 +579,8 @@
},
"loggerCoverage": {
"filesWithCreateLogger": 65,
"totalSourceFiles": 764,
"coverageRatio": 0.0851,
"totalSourceFiles": 765,
"coverageRatio": 0.085,
"subdomainsWithZeroCreateLogger": [
"api",
"bin",
@@ -652,9 +652,9 @@
]
},
"hotpathConsoleErrors": {
"totalOccurrences": 592,
"totalOccurrences": 590,
"exemptOccurrences": 326,
"hotpathOccurrences": 266,
"hotpathOccurrences": 264,
"filesAffected": 81,
"topFiles": [
{
+7 -7
View File
@@ -6,10 +6,10 @@ Scope: `src/**/*.{ts,tsx,js,jsx,mjs,cjs}`
| Metric | Value |
|---|---:|
| Sync fs occurrences (all) | 2471 |
| Sync fs files affected (all) | 261 |
| Sync fs occurrences (runtime hotpaths) | 1186 |
| Sync fs files affected (runtime hotpaths) | 155 |
| Sync fs occurrences (all) | 2519 |
| Sync fs files affected (all) | 263 |
| Sync fs occurrences (runtime hotpaths) | 1193 |
| Sync fs files affected (runtime hotpaths) | 156 |
| Legacy shim markers | 465 |
| Legacy shim files affected | 176 |
@@ -56,11 +56,11 @@ Scope: `src/**/*.{ts,tsx,js,jsx,mjs,cjs}`
| Metric | Value |
|---|---:|
| typed-error adoption (typed/total throws) | 18.3% (83/454) |
| typed-error adoption (typed/total throws) | 19.6% (89/455) |
| typed-error adoption (P4 locked subdomains) | 93.3% (28/30), target 40% |
| hotpath console.error/warn occurrences | 266 (592 total, 326 CLI-UX exempt) |
| hotpath console.error/warn occurrences | 264 (590 total, 326 CLI-UX exempt) |
| hotpath console.error/warn files | 81 |
| files with createLogger | 65/764 |
| files with createLogger | 65/765 |
| subdomains with zero createLogger | 15 (api, bin, channels, cliproxy, cliproxy/accounts, cliproxy/ai-providers, cliproxy/binary, cliproxy/config, cliproxy/management, cliproxy/sync, cliproxy/types, config, dispatcher, shared, types) |
| files > 400 LOC | 91 |
| files > 600 LOC | 42 |
+38 -28
View File
@@ -41,6 +41,7 @@ import {
import type { CLIProxyBackend } from './types';
import { getVersionListCachePath } from './binary/version-cache';
import { loadOrCreateUnifiedConfig } from '../config/config-loader-facade';
import { withInstallLifecycleLock } from './binary/install-lifecycle-lock';
export const CLIPROXY_DELETED_PLUS_REPO = 'router-for-me/CLIProxyAPIPlus';
export const CLIPROXY_PLUS_FALLBACK_TRACKING_URL = 'https://github.com/kaitranntt/ccs/issues/1062';
@@ -176,6 +177,14 @@ function getBackendBinDir(backend: CLIProxyBackend = DEFAULT_BACKEND): string {
return `${baseDir}/${backend}`;
}
export async function withCliproxyInstallLifecycleLock<T>(
backend: CLIProxyBackend,
operation: () => Promise<T>
): Promise<T> {
const lockTarget = path.join(getBinDir(), `.install-lifecycle-${backend}`);
return withInstallLifecycleLock(lockTarget, operation);
}
/** Default configuration (uses backend from config.yaml or defaults to `DEFAULT_BACKEND`) */
function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): BinaryManagerConfig {
const backendConfig = BACKEND_CONFIG[backend];
@@ -186,6 +195,7 @@ function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): Binary
maxRetries: 3,
verbose: false,
forceVersion: false,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: true,
backend, // Pass backend for installer to use correct download URL
@@ -317,6 +327,7 @@ interface InstallCliproxyVersionDeps {
formatInfo?: typeof info;
formatWarn?: typeof warn;
getInstalledVersion?: typeof getInstalledCliproxyVersion;
withInstallLifecycleLockFn?: typeof withCliproxyInstallLifecycleLock;
}
/** Install a specific version of CLIProxyAPI */
@@ -329,42 +340,41 @@ export async function installCliproxyVersion(
const configuredBackend = backend ?? getConfiguredOrDefaultBackend();
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
const manager =
deps.createManager?.({ version, verbose, forceVersion: true }, effectiveBackend) ??
new BinaryManager({ version, verbose, forceVersion: true }, effectiveBackend);
deps.createManager?.(
{ version, verbose, forceVersion: true, replaceExisting: true },
effectiveBackend
) ??
new BinaryManager(
{ version, verbose, forceVersion: true, replaceExisting: true },
effectiveBackend
);
const stopProxyFn = deps.stopProxyFn ?? stopProxy;
const waitForPortFreeFn = deps.waitForPortFreeFn ?? waitForPortFree;
const formatInfo = deps.formatInfo ?? info;
const formatWarn = deps.formatWarn ?? warn;
const getInstalledVersion = deps.getInstalledVersion ?? getInstalledCliproxyVersion;
const withLifecycleLock = deps.withInstallLifecycleLockFn ?? withCliproxyInstallLifecycleLock;
// Always attempt a best-effort stop first so we also catch untracked proxies
// that are running without a session lock.
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
const result = await stopProxyFn();
if (result.stopped) {
const stoppedPort = result.port ?? resolveLifecyclePort();
// Wait for port to be fully released
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
if (!portFree && verbose) {
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
await withLifecycleLock(effectiveBackend, async () => {
// Always attempt a best-effort stop first so we also catch untracked proxies
// that are running without a session lock.
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
const result = await stopProxyFn();
if (result.stopped) {
const stoppedPort = result.port ?? resolveLifecyclePort();
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
if (!portFree && verbose) {
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
}
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
}
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
}
if (manager.isBinaryInstalled()) {
const label = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
if (verbose)
console.log(
formatInfo(`Removing existing ${label} v${getInstalledVersion(effectiveBackend)}`)
);
manager.deleteBinary();
}
await manager.ensureBinary();
await manager.ensureBinary();
if (verbose) {
console.log(formatInfo('New version will be active on next CLIProxy command'));
}
if (verbose) {
console.log(formatInfo('New version will be active on next CLIProxy command'));
}
});
}
/** Fetch the latest CLIProxyAPI version from GitHub API */
@@ -0,0 +1,359 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { getExecutableName } from '../platform-detector';
import { downloadAndInstall } from '../installer';
import { ensureBinary } from '../lifecycle';
import { withInstallLifecycleLock } from '../install-lifecycle-lock';
describe('atomic binary installation', () => {
let binPath: string;
beforeEach(() => {
binPath = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-atomic-installer-'));
});
afterEach(() => {
fs.rmSync(binPath, { recursive: true, force: true });
});
function stagingDirectories(): string[] {
return fs.readdirSync(binPath).filter((entry) => entry.startsWith('.cliproxy-install-'));
}
it('preserves the installed binary and version when verification fails', async () => {
const binaryPath = path.join(binPath, getExecutableName('original'));
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await expect(
downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: false,
expected: 'expected-checksum',
actual: 'actual-checksum',
}),
extractArchiveFn: async () => {
throw new Error('extract should not run');
},
}
)
).rejects.toThrow('Checksum mismatch');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
expect(stagingDirectories()).toEqual([]);
});
it('atomically swaps the verified staged binary and then records its version', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
}
);
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('new-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.7.1');
expect(stagingDirectories()).toEqual([]);
});
it('preserves the installed binary and version when the atomic swap fails', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await expect(
downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
renameSyncFn: () => {
throw Object.assign(new Error('replacement blocked'), { code: 'EPERM' });
},
}
)
).rejects.toThrow('replacement blocked');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
expect(stagingDirectories()).toEqual([]);
});
it('installs a forced version even when an older binary already exists', async () => {
const binaryPath = path.join(binPath, getExecutableName('original'));
fs.writeFileSync(binaryPath, 'old-binary');
let installs = 0;
const resolvedPath = await ensureBinary(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
{
downloadAndInstallFn: async () => {
installs += 1;
},
}
);
expect(resolvedPath).toBe(binaryPath);
expect(installs).toBe(1);
});
it('reuses an existing pinned binary during runtime bootstrap', async () => {
const binaryPath = path.join(binPath, getExecutableName('original'));
fs.writeFileSync(binaryPath, 'pinned-binary');
let installs = 0;
const resolvedPath = await ensureBinary(
{
version: '6.6.80',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: false,
backend: 'original',
},
{
downloadAndInstallFn: async () => {
installs += 1;
},
}
);
expect(resolvedPath).toBe(binaryPath);
expect(installs).toBe(0);
});
it('waits for an externally held compatible install lifecycle lock', async () => {
const lockTarget = path.join(binPath, '.install-lifecycle-plus');
fs.mkdirSync(lockTarget, { recursive: true });
fs.mkdirSync(`${lockTarget}.lock`);
let entered = false;
const operation = withInstallLifecycleLock(lockTarget, async () => {
entered = true;
});
await Bun.sleep(50);
expect(entered).toBe(false);
fs.rmdirSync(`${lockTarget}.lock`);
await operation;
expect(entered).toBe(true);
});
it('restores the previous binary when publishing the version marker fails', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
let renames = 0;
await expect(
downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
renameSyncFn: (source, destination) => {
renames += 1;
if (renames === 2) throw new Error('version marker blocked');
fs.renameSync(source, destination);
},
}
)
).rejects.toThrow('version marker blocked');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
expect(stagingDirectories()).toEqual([]);
});
it('serializes concurrent installs so the binary and version stay paired', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
const install = (version: string) =>
downloadAndInstall(
{
version,
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), `binary-${version}`);
},
}
);
await Promise.all([install('6.7.1'), install('6.7.2')]);
const installedVersion = fs.readFileSync(versionPath, 'utf8');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe(`binary-${installedVersion}`);
expect(stagingDirectories()).toEqual([]);
});
it('removes staging residue left by an interrupted prior install', async () => {
const stalePath = path.join(binPath, '.cliproxy-install-stale');
fs.mkdirSync(stalePath);
fs.writeFileSync(path.join(stalePath, 'partial-archive'), 'partial');
const binaryName = getExecutableName('original');
await downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
}
);
expect(fs.existsSync(stalePath)).toBe(false);
expect(stagingDirectories()).toEqual([]);
});
});
@@ -33,6 +33,7 @@ describe('installCliproxyVersion', () => {
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: (_config: unknown, backend: string) => {
seenBackend = backend;
return {
@@ -147,6 +148,7 @@ describe('installCliproxyVersion', () => {
const binaryManager = await import(`../../binary-manager?binary-manager-install=${Date.now()}`);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => false,
deleteBinary: () => {
@@ -176,6 +178,39 @@ describe('installCliproxyVersion', () => {
expect(calls.ensureBinary).toBe(1);
});
it('keeps the installed binary in place while the replacement is prepared', async () => {
const calls = {
deleteBinary: 0,
ensureBinary: 0,
};
const binaryManager = await import(
`../../binary-manager?binary-manager-atomic-install=${Date.now()}`
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => true,
deleteBinary: () => {
calls.deleteBinary += 1;
},
ensureBinary: async () => {
calls.ensureBinary += 1;
return '/tmp/ccs-bin/plus/cli-proxy-api-plus';
},
}),
stopProxyFn: async () => ({ stopped: false, error: 'No active CLIProxy session found' }),
waitForPortFreeFn: async () => true,
formatInfo: (message: string) => message,
formatWarn: (message: string) => message,
getInstalledVersion: () => '6.6.80',
});
expect(calls.deleteBinary).toBe(0);
expect(calls.ensureBinary).toBe(1);
});
it('waits for the port that was actually stopped before continuing install', async () => {
let waitedPort: number | undefined;
@@ -184,6 +219,7 @@ describe('installCliproxyVersion', () => {
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => false,
deleteBinary: () => undefined,
@@ -0,0 +1,37 @@
import { AsyncLocalStorage } from 'async_hooks';
import * as fs from 'fs';
import * as path from 'path';
import * as lockfile from 'proper-lockfile';
const heldInstallLocks = new AsyncLocalStorage<Set<string>>();
export async function withInstallLifecycleLock<T>(
lockTarget: string,
operation: () => Promise<T>
): Promise<T> {
const resolvedTarget = path.resolve(lockTarget);
const held = heldInstallLocks.getStore();
if (held?.has(resolvedTarget)) return operation();
fs.mkdirSync(resolvedTarget, { recursive: true });
const release = await lockfile.lock(resolvedTarget, {
stale: 10 * 60 * 1000,
retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 },
});
const nextHeld = new Set(held);
nextHeld.add(resolvedTarget);
let operationError: unknown;
try {
return await heldInstallLocks.run(nextHeld, operation);
} catch (error) {
operationError = error;
throw error;
} finally {
try {
await release();
} catch (error) {
if (!operationError) throw error;
}
}
}
+90 -46
View File
@@ -5,6 +5,7 @@
import * as fs from 'fs';
import * as path from 'path';
import * as lockfile from 'proper-lockfile';
import { BinaryManagerConfig } from '../types';
import {
detectPlatform,
@@ -16,66 +17,71 @@ import {
import { downloadWithRetry } from './downloader';
import { verifyChecksum, computeChecksum } from './verifier';
import { extractArchive } from './extractor';
import { writeInstalledVersion } from './version-cache';
import { ProgressIndicator } from '../../utils/progress-indicator';
import { ok } from '../../utils/ui';
import { BinaryError, NetworkError } from '../../errors/error-types';
interface DownloadAndInstallDeps {
downloadWithRetryFn?: typeof downloadWithRetry;
verifyChecksumFn?: typeof verifyChecksum;
extractArchiveFn?: typeof extractArchive;
renameSyncFn?: typeof fs.renameSync;
}
/**
* Download and install the binary
*/
export async function downloadAndInstall(
config: BinaryManagerConfig,
verbose = false
verbose = false,
deps: DownloadAndInstallDeps = {}
): Promise<void> {
const backend = config.backend ?? DEFAULT_BACKEND;
const platform = detectPlatform(config.version, backend);
const downloadUrl = getDownloadUrl(config.version, backend);
const checksumsUrl = getChecksumsUrl(config.version, backend);
const backendLabel = backend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
const downloadWithRetryFn = deps.downloadWithRetryFn ?? downloadWithRetry;
const verifyChecksumFn = deps.verifyChecksumFn ?? verifyChecksum;
const extractArchiveFn = deps.extractArchiveFn ?? extractArchive;
const renameSyncFn = deps.renameSyncFn ?? fs.renameSync;
fs.mkdirSync(config.binPath, { recursive: true });
// Delete existing binary before install to prevent mismatched binaries.
// Abort if binary is currently running (ETXTBSY) — cannot replace in-use binary.
// Happens in Docker when dashboard tries to update while bootstrap's instance is active.
const existingBinary = path.join(config.binPath, getExecutableName(backend));
if (fs.existsSync(existingBinary)) {
try {
fs.unlinkSync(existingBinary);
if (verbose) console.error(`[cliproxy] Removed existing binary: ${existingBinary}`);
} catch (error: unknown) {
const code =
error instanceof Error && 'code' in error ? (error as { code: string }).code : '';
// ETXTBSY: Linux-specific error when unlinking a running executable.
// EBUSY on Windows may mean something different (mount point, etc.),
// so only treat ETXTBSY as "binary in use" to avoid misleading messages.
if (code === 'ETXTBSY') {
if (verbose)
console.error(`[cliproxy] Binary is running, cannot replace: ${existingBinary}`);
throw new Error(
'CLIProxy binary is currently running and cannot be replaced. Restart the container to apply the update.'
);
}
throw error;
}
}
const archivePath = path.join(config.binPath, `cliproxy-archive.${platform.extension}`);
const releaseLock = await lockfile.lock(config.binPath, {
stale: 10 * 60 * 1000,
retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 },
});
let stagingPath: string | undefined;
const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`);
spinner.start();
let installError: unknown;
try {
const result = await downloadWithRetry(downloadUrl, archivePath, {
for (const entry of fs.readdirSync(config.binPath)) {
if (entry.startsWith('.cliproxy-install-')) {
fs.rmSync(path.join(config.binPath, entry), { recursive: true, force: true });
}
}
stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-'));
const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`);
const stagedBinary = path.join(stagingPath, getExecutableName(backend));
const stagedVersion = path.join(stagingPath, '.version');
const installedBinary = path.join(config.binPath, getExecutableName(backend));
const installedVersion = path.join(config.binPath, '.version');
const backupBinary = path.join(stagingPath, '.previous-binary');
const hadInstalledBinary = fs.existsSync(installedBinary);
spinner.start();
const result = await downloadWithRetryFn(downloadUrl, archivePath, {
maxRetries: config.maxRetries,
verbose,
});
if (!result.success) {
spinner.fail('Download failed');
throw new Error(result.error || 'Download failed after retries');
throw new NetworkError(result.error || 'Download failed after retries', downloadUrl);
}
spinner.update('Verifying checksum');
const checksumResult = await verifyChecksum(
const checksumResult = await verifyChecksumFn(
archivePath,
platform.binaryName,
checksumsUrl,
@@ -84,29 +90,64 @@ export async function downloadAndInstall(
if (!checksumResult.valid) {
spinner.fail('Checksum mismatch');
fs.unlinkSync(archivePath);
throw new Error(
throw new BinaryError(
`Checksum mismatch for ${platform.binaryName}\nExpected: ${checksumResult.expected}\n` +
`Actual: ${checksumResult.actual}\n\nManual download: ${downloadUrl}`
`Actual: ${checksumResult.actual}\n\nManual download: ${downloadUrl}`,
stagedBinary
);
}
spinner.update('Extracting binary');
await extractArchive(archivePath, config.binPath, platform.extension, verbose, backend);
spinner.succeed(`${backendLabel} ready`);
fs.unlinkSync(archivePath);
const binaryPath = path.join(config.binPath, getExecutableName(backend));
if (platform.os !== 'windows' && fs.existsSync(binaryPath)) {
fs.chmodSync(binaryPath, 0o755);
if (verbose) console.error(`[cliproxy] Set executable permissions: ${binaryPath}`);
await extractArchiveFn(archivePath, stagingPath, platform.extension, verbose, backend);
if (!fs.existsSync(stagedBinary) || !fs.statSync(stagedBinary).isFile()) {
throw new BinaryError(
`Extracted archive did not contain ${getExecutableName(backend)}`,
stagedBinary
);
}
writeInstalledVersion(config.binPath, config.version);
if (platform.os !== 'windows') {
fs.chmodSync(stagedBinary, 0o755);
if (verbose) console.error(`[cliproxy] Set executable permissions: ${stagedBinary}`);
}
fs.writeFileSync(stagedVersion, config.version, 'utf8');
if (hadInstalledBinary) {
fs.copyFileSync(installedBinary, backupBinary);
if (platform.os !== 'windows') fs.chmodSync(backupBinary, 0o755);
}
renameSyncFn(stagedBinary, installedBinary);
try {
renameSyncFn(stagedVersion, installedVersion);
} catch (error) {
if (hadInstalledBinary) {
renameSyncFn(backupBinary, installedBinary);
} else {
fs.unlinkSync(installedBinary);
}
throw error;
}
spinner.succeed(`${backendLabel} ready`);
console.log(ok(`${backendLabel} v${config.version} installed successfully`));
} catch (error) {
installError = error;
spinner.fail('Installation failed');
throw error;
} finally {
let cleanupError: unknown;
try {
if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
} catch (error) {
cleanupError = error;
} finally {
try {
await releaseLock();
} catch (error) {
cleanupError ??= error;
}
}
if (!installError && cleanupError) throw cleanupError;
}
}
@@ -124,7 +165,10 @@ export function deleteBinary(binPath: string, verbose = false, backend?: CLIProx
const code =
error instanceof Error && 'code' in error ? (error as { code: string }).code : '';
if (code === 'ETXTBSY') {
throw new Error('CLIProxy binary is currently running and cannot be deleted.');
throw new BinaryError(
'CLIProxy binary is currently running and cannot be deleted.',
binaryPath
);
}
throw error;
}
+23 -7
View File
@@ -11,10 +11,11 @@ import {
isNewerVersion,
isVersionFaulty,
} from './version-checker';
import { downloadAndInstall, deleteBinary, getBinaryPath } from './installer';
import { downloadAndInstall, getBinaryPath } from './installer';
import { info, warn } from '../../utils/ui';
import { isCliproxyRunning } from '../services/stats-fetcher';
import { resolveLifecyclePort } from '../config/port-manager';
import { BinaryError } from '../../errors/error-types';
import {
CLIPROXY_MAX_STABLE_VERSION,
CLIPROXY_FAULTY_RANGE,
@@ -93,7 +94,7 @@ async function handleAutoUpdate(config: BinaryManagerConfig, verbose: boolean):
} else {
console.log(info(updateMsg));
console.log(info(`Updating ${backendLabel}...`));
deleteBinary(config.binPath, verbose, backend);
// The installer stages and validates the replacement before atomically swapping it.
config.version = targetVersion;
await downloadAndInstall(config, verbose);
}
@@ -103,17 +104,31 @@ async function handleAutoUpdate(config: BinaryManagerConfig, verbose: boolean):
* Ensure binary is available (download if missing, update if outdated)
* @returns Path to executable binary
*/
export async function ensureBinary(config: BinaryManagerConfig): Promise<string> {
interface EnsureBinaryDeps {
downloadAndInstallFn?: typeof downloadAndInstall;
}
export async function ensureBinary(
config: BinaryManagerConfig,
deps: EnsureBinaryDeps = {}
): Promise<string> {
const verbose = config.verbose;
const backend: CLIProxyBackend = config.backend ?? DEFAULT_BACKEND;
const binaryPath = getBinaryPath(config.binPath, backend);
const downloadAndInstallFn = deps.downloadAndInstallFn ?? downloadAndInstall;
// Binary exists - check for updates unless forceVersion
if (fs.existsSync(binaryPath)) {
log(`Binary exists: ${binaryPath}`, verbose);
if (config.replaceExisting) {
log(`Force version mode: installing specified version ${config.version}`, verbose);
await downloadAndInstallFn(config, verbose);
return binaryPath;
}
if (config.forceVersion) {
log('Force version mode: skipping auto-update', verbose);
log(`Pinned version mode: using installed version ${config.version}`, verbose);
return binaryPath;
}
@@ -134,9 +149,10 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise<string>
// Binary missing
if (!config.allowInstall) {
throw new Error(
throw new BinaryError(
`${getBackendLabel(backend)} binary is not installed locally. ` +
'Run "ccs cliproxy install" when you have network access.'
'Run "ccs cliproxy install" when you have network access.',
binaryPath
);
}
@@ -161,6 +177,6 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise<string>
log(`Force version mode: using specified version ${config.version}`, verbose);
}
await downloadAndInstall(config, verbose);
await downloadAndInstallFn(config, verbose);
return binaryPath;
}
+27 -22
View File
@@ -19,6 +19,7 @@ import {
clearPinnedVersion,
isVersionPinned,
resolveLocalBackend,
withCliproxyInstallLifecycleLock,
} from '../binary-manager';
import { BACKEND_CONFIG, DEFAULT_BACKEND } from '../binary/platform-detector';
import { CLIProxyBackend } from '../types';
@@ -127,14 +128,16 @@ export async function installVersion(
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
try {
await installCliproxyVersion(version, verbose, effectiveBackend);
savePinnedVersion(version, effectiveBackend);
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
await installCliproxyVersion(version, verbose, effectiveBackend);
savePinnedVersion(version, effectiveBackend);
return {
success: true,
version,
wasPinned: true,
};
return {
success: true,
version,
wasPinned: true,
};
});
} catch (error) {
return {
success: false,
@@ -156,26 +159,28 @@ export async function installLatest(
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
try {
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
const wasPinned = isVersionPinned(effectiveBackend);
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
const wasPinned = isVersionPinned(effectiveBackend);
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
return {
success: true,
version: latestVersion,
error: `Already running latest version: v${latestVersion}`,
};
}
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
clearPinnedVersion(effectiveBackend);
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
return {
success: true,
version: latestVersion,
error: `Already running latest version: v${latestVersion}`,
wasPinned,
};
}
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
clearPinnedVersion(effectiveBackend);
return {
success: true,
version: latestVersion,
wasPinned,
};
});
} catch (error) {
return {
success: false,
@@ -56,6 +56,7 @@ describe('types.ts backward compatibility', () => {
maxRetries: 3,
verbose: false,
forceVersion: false,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: true,
};
+1
View File
@@ -14,6 +14,7 @@ export interface BinaryManagerConfig {
maxRetries: number;
verbose: boolean;
forceVersion: boolean;
replaceExisting?: boolean;
skipAutoUpdate: boolean;
allowInstall: boolean;
backend?: CLIProxyBackend;
@@ -1,9 +1,14 @@
import { installCliproxyVersion, resolveLocalBackend } from '../../cliproxy/binary-manager';
import {
installCliproxyVersion,
resolveLocalBackend,
withCliproxyInstallLifecycleLock,
} from '../../cliproxy/binary-manager';
import { resolveLifecyclePort } from '../../cliproxy/config/port-manager';
import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/service-manager';
import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker';
import { isCliproxyRunning } from '../../cliproxy/services/stats-fetcher';
import type { CLIProxyBackend } from '../../cliproxy/types';
import { ProxyError } from '../../errors/error-types';
import {
isRunningUnderSupervisord,
restartCliproxyViaSupervisord,
@@ -22,6 +27,9 @@ interface InstallDashboardCliproxyVersionDeps {
backend?: CLIProxyBackend
) => Promise<void>;
ensureCliproxyService: () => Promise<ServiceStartResult>;
isRunningUnderSupervisord?: () => boolean;
restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord;
withInstallLifecycleLock?: typeof withCliproxyInstallLifecycleLock;
}
const defaultDeps: InstallDashboardCliproxyVersionDeps = {
@@ -48,6 +56,23 @@ async function wasProxyRunning(deps: InstallDashboardCliproxyVersionDeps): Promi
return deps.isCliproxyRunning();
}
async function restoreProxyService(
deps: InstallDashboardCliproxyVersionDeps
): Promise<ServiceStartResult> {
const underSupervisord = deps.isRunningUnderSupervisord?.() ?? isRunningUnderSupervisord();
if (underSupervisord) {
const restart = deps.restartCliproxyViaSupervisord?.() ?? restartCliproxyViaSupervisord();
return {
started: restart.success,
alreadyRunning: false,
port: restart.port ?? resolveLifecyclePort(),
error: restart.error,
};
}
return deps.ensureCliproxyService();
}
export async function installDashboardCliproxyVersion(
version: string,
backend: CLIProxyBackend,
@@ -55,48 +80,53 @@ export async function installDashboardCliproxyVersion(
): Promise<DashboardCliproxyInstallResult> {
const effectiveBackend = resolveLocalBackend(backend, { notifyOnPlus: true });
const backendLabel = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
const shouldRestoreService = await wasProxyRunning(deps);
const withLifecycleLock = deps.withInstallLifecycleLock ?? withCliproxyInstallLifecycleLock;
// The installer owns the stop-and-replace lifecycle, including best-effort
// shutdown for tracked and untracked proxies before swapping the binary.
await deps.installCliproxyVersion(version, true, effectiveBackend);
return withLifecycleLock(effectiveBackend, async () => {
const shouldRestoreService = await wasProxyRunning(deps);
// The installer owns the stop-and-replace lifecycle, including best-effort
// shutdown for tracked and untracked proxies before swapping the binary.
try {
await deps.installCliproxyVersion(version, true, effectiveBackend);
} catch (error) {
if (shouldRestoreService) {
const restoreResult = await restoreProxyService(deps);
if (!restoreResult.started && !restoreResult.alreadyRunning) {
const installMessage = error instanceof Error ? error.message : String(error);
throw new ProxyError(
`${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
restoreResult.port
);
}
}
throw error;
}
if (!shouldRestoreService) {
return {
success: true,
restarted: false,
message: `Successfully installed ${backendLabel} v${version}`,
};
}
// In Docker, supervisord owns process lifecycle — delegate restart to it
const startResult = await restoreProxyService(deps);
if (!startResult.started && !startResult.alreadyRunning) {
return {
success: false,
restarted: false,
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
};
}
if (!shouldRestoreService) {
return {
success: true,
restarted: false,
message: `Successfully installed ${backendLabel} v${version}`,
restarted: true,
port: startResult.port,
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
};
}
// In Docker, supervisord owns process lifecycle — delegate restart to it
if (isRunningUnderSupervisord()) {
const result = restartCliproxyViaSupervisord();
return {
success: result.success,
restarted: result.success,
port: result.port,
error: result.error,
message: result.success
? `Successfully installed ${backendLabel} v${version} and restarted it on port ${result.port}`
: `Installed ${backendLabel} v${version}, but restart failed`,
};
}
const startResult = await deps.ensureCliproxyService();
if (!startResult.started && !startResult.alreadyRunning) {
return {
success: false,
restarted: false,
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
};
}
return {
success: true,
restarted: true,
port: startResult.port,
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
};
});
}
@@ -0,0 +1,143 @@
import { mkdtempSync, readFileSync, rmSync } from 'fs';
import { spawnSync } from 'child_process';
import { describe, expect, it } from 'bun:test';
import { tmpdir } from 'os';
import { join } from 'path';
const updateScript = readFileSync('docker/host/ccs-cliproxy-safe-update.sh', 'utf8');
const reconcileScript = readFileSync('docker/host/ccs-cliproxy-reconcile.sh', 'utf8');
const updateService = readFileSync('docker/host/systemd/ccs-cliproxy-update.service', 'utf8');
const updateTimer = readFileSync('docker/host/systemd/ccs-cliproxy-update.timer', 'utf8');
const reconcileService = readFileSync('docker/host/systemd/ccs-cliproxy-reconcile.service', 'utf8');
const reconcileTimer = readFileSync('docker/host/systemd/ccs-cliproxy-reconcile.timer', 'utf8');
describe('CLIProxy Docker host continuity assets', () => {
it('stages and validates the replacement before stopping the live proxy', () => {
const stageIndex = updateScript.indexOf('CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest');
const validateIndex = updateScript.indexOf('"$stage_binary" --version');
const stopIndex = updateScript.indexOf('supervisorctl_cmd stop cliproxy', validateIndex);
expect(stageIndex).toBeGreaterThan(-1);
expect(validateIndex).toBeGreaterThan(stageIndex);
expect(stopIndex).toBeGreaterThan(validateIndex);
});
it('serializes updates and reconciliation with the same lock', () => {
expect(updateScript).toContain('/run/lock/ccs-cliproxy-maintenance.lock');
expect(reconcileScript).toContain('/run/lock/ccs-cliproxy-maintenance.lock');
expect(updateScript).toContain('flock -n 9');
expect(reconcileScript).toContain('flock -n 9');
});
it('shares the in-container install lifecycle lock with CLI and dashboard installs', () => {
expect(updateScript).toContain(
"install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'"
);
expect(updateScript).toContain('install_lock_dir="$install_lock_target.lock"');
expect(updateScript).toContain('while ! mkdir "$install_lock_dir"');
expect(updateScript).toContain('touch "$install_lock_dir"');
expect(updateScript).toContain('install_lock_stale_seconds=600');
expect(updateScript).toContain('remove_stale_install_lock');
expect(updateScript).toContain('rmdir "$install_lock_dir"');
expect(updateScript.indexOf('acquire_install_lock')).toBeLessThan(
updateScript.indexOf('supervisorctl_cmd stop cliproxy')
);
});
it('reclaims an orphaned stale lifecycle lock', () => {
const testRoot = mkdtempSync(join(tmpdir(), 'ccs-cliproxy-host-lock-'));
const helpersStart = updateScript.indexOf('remove_stale_install_lock() {');
const helpersEnd = updateScript.indexOf('\nwait_for_proxy() {');
const helpers = updateScript.slice(helpersStart, helpersEnd);
try {
const result = spawnSync(
'bash',
[
'-c',
`set -Eeuo pipefail
install_lock_target="$1/target"
install_lock_dir="$install_lock_target.lock"
install_lock_stale_seconds=600
install_lock_owned=0
mkdir -p "$install_lock_target" "$install_lock_dir"
touch -t 200001010000 "$install_lock_dir"
${helpers}
acquire_install_lock
test "$install_lock_owned" -eq 1
release_install_lock`,
'test-shell',
testRoot,
],
{ encoding: 'utf8' }
);
expect(result.status).toBe(0);
} finally {
rmSync(testRoot, { recursive: true, force: true });
}
});
it('rolls back failed swaps and health-checks both services', () => {
expect(updateScript).toContain('rollback()');
expect(updateScript).toContain('previous-binary');
expect(updateScript).toContain('wait_for_proxy');
expect(reconcileScript).toContain('http://127.0.0.1:3000/');
expect(reconcileScript).toContain('http://127.0.0.1:8317/');
});
it('recreates missing containers and escalates unhealthy recovery', () => {
expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker');
expect(reconcileScript).toContain('$compose_dir/docker-compose.integrated.yml');
expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_PROJECT:-docker');
expect(reconcileScript).toContain('--project-name "$compose_project"');
expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build');
expect(reconcileScript).toContain("2>/dev/null || true");
expect(reconcileScript).toContain('if [ -z "$running_state" ]');
expect(reconcileScript).toContain('restart ccs-dashboard cliproxy');
expect(reconcileScript).toContain('docker restart "$container_name"');
expect(reconcileScript).toContain('up -d --force-recreate --no-build');
expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan(
reconcileScript.indexOf('restart ccs-dashboard cliproxy')
);
});
it('installs executable services on bounded timers', () => {
expect(updateService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-safe-update.sh');
expect(updateService).toContain('TimeoutStartSec=10min');
expect(updateTimer).toContain('OnUnitActiveSec=15min');
expect(reconcileService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh');
expect(reconcileTimer).toContain('OnUnitActiveSec=30s');
});
it('forces nonzero signal exits so the EXIT trap rolls back maintenance', () => {
expect(updateScript).toContain("trap 'exit 130' INT");
expect(updateScript).toContain("trap 'exit 143' TERM");
expect(updateScript).toContain("trap 'exit 129' HUP");
const handlerStart = updateScript.indexOf('on_exit() {');
const handlerEnd = updateScript.indexOf('\ntrap on_exit EXIT');
const handlers = updateScript.slice(handlerStart, handlerEnd);
const result = spawnSync(
'bash',
[
'-c',
`set -Eeuo pipefail
maintenance_started=1
stage_root=/tmp/unused
rollback() { printf 'rollback\\n'; }
cleanup() { printf 'cleanup\\n'; }
release_install_lock() { :; }
${handlers}
trap on_exit EXIT
trap 'exit 143' TERM
true
kill -TERM $$`,
],
{ encoding: 'utf8' }
);
expect(result.status).toBe(143);
expect(result.stdout).toContain('rollback');
expect(result.stdout).toContain('cleanup');
});
});
@@ -10,6 +10,7 @@ function createDeps(
sessionRunning?: boolean;
remoteRunning?: boolean;
startResult?: { started: boolean; alreadyRunning: boolean; port: number; error?: string };
installError?: Error;
} = {}
) {
const calls = {
@@ -30,6 +31,7 @@ function createDeps(
_backend?: CLIProxyBackend
) => {
calls.installCliproxyVersion += 1;
if (overrides.installError) throw overrides.installError;
},
ensureCliproxyService: async () => {
calls.ensureCliproxyService += 1;
@@ -41,6 +43,10 @@ function createDeps(
}
);
},
withInstallLifecycleLock: async (
_backend: CLIProxyBackend,
operation: () => Promise<unknown>
) => operation(),
};
return { deps, calls };
@@ -122,4 +128,67 @@ describe('installDashboardCliproxyVersion', () => {
message: 'Installed CLIProxy Plus v6.7.1, but failed to restart it',
});
});
it('restores a previously running proxy when installation fails', async () => {
const { deps, calls } = createDeps({
sessionRunning: true,
installError: new Error('checksum mismatch'),
});
await expect(installDashboardCliproxyVersion('6.7.1', 'plus', deps)).rejects.toThrow(
'checksum mismatch'
);
expect(calls.ensureCliproxyService).toBe(1);
});
it('serializes concurrent dashboard stop-install-restore transactions', async () => {
let running = true;
let queue = Promise.resolve();
let activeTransactions = 0;
let maxActiveTransactions = 0;
let restores = 0;
const deps = {
getProxyStatus: () => ({ running }),
isCliproxyRunning: async () => running,
installCliproxyVersion: async () => {
running = false;
await new Promise((resolve) => setTimeout(resolve, 5));
},
ensureCliproxyService: async () => {
running = true;
restores += 1;
return { started: true, alreadyRunning: false, port: 8317 };
},
withInstallLifecycleLock: <T>(
_backend: CLIProxyBackend,
operation: () => Promise<T>
): Promise<T> => {
const result = queue.then(async () => {
activeTransactions += 1;
maxActiveTransactions = Math.max(maxActiveTransactions, activeTransactions);
try {
return await operation();
} finally {
activeTransactions -= 1;
}
});
queue = result.then(
() => undefined,
() => undefined
);
return result;
},
};
const results = await Promise.all([
installDashboardCliproxyVersion('6.7.1', 'plus', deps),
installDashboardCliproxyVersion('6.7.2', 'plus', deps),
]);
expect(maxActiveTransactions).toBe(1);
expect(restores).toBe(2);
expect(results.every((result) => result.restarted)).toBe(true);
expect(running).toBe(true);
});
});