fix(cliproxy): harden update recovery paths

docker/host/ccs-cliproxy-reconcile.sh

@@ -3,8 +3,9 @@
 set -Eeuo pipefail
 
 container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
-compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker}"
-compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.integrated.yml}"
+compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/opt/cliproxy}"
+compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.yml}"
+compose_project="${CCS_CLIPROXY_COMPOSE_PROJECT:-docker}"
 lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
 log_file="${CCS_CLIPROXY_RECONCILE_LOG:-/var/log/ccs-cliproxy-reconcile.log}"
 
@@ -37,8 +38,10 @@ wait_for_health() {
 
 compose_up() {
   cd "$compose_dir"
-  docker compose -f "$compose_file" up -d --no-build || \
-    docker compose -f "$compose_file" up -d --build
+  docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
+    -f "$compose_file" up -d --no-build || \
+    docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
+      -f "$compose_file" up -d --build
 }
 
 if ! docker inspect "$container_name" >/dev/null 2>&1; then
@@ -67,10 +70,12 @@ if probe; then
 fi
 
 log 'Dashboard or proxy failed two consecutive probes; restarting supervised processes'
-docker exec "$container_name" supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy
-if wait_for_health; then
-  log 'Supervised processes recovered and passed both health probes'
-  exit 0
+if docker exec "$container_name" \
+  supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy; then
+  if wait_for_health; then
+    log 'Supervised processes recovered and passed both health probes'
+    exit 0
+  fi
 fi
 
 log 'Supervisor recovery failed; restarting the container'

docker/host/ccs-cliproxy-safe-update.sh

@@ -26,11 +26,16 @@ if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'tru
   exit 1
 fi
 
-status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)" || {
+status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus --verbose 2>&1)" || {
   log "Unable to inspect the installed CLIProxy version: $status_output"
   exit 1
 }
 
+if grep -qi 'Could not fetch' <<<"$status_output"; then
+  log "Unable to check the latest CLIProxy version: $(grep -im1 'Could not fetch' <<<"$status_output" | xargs)"
+  exit 1
+fi
+
 if ! grep -qi 'update available' <<<"$status_output"; then
   exit 0
 fi
@@ -50,7 +55,7 @@ stage_binary="$stage_dir/cli-proxy-api-plus"
 stage_version="$stage_dir/.version"
 backup_binary="$stage_root/previous-binary"
 backup_version="$stage_root/previous-version"
-swap_started=0
+maintenance_started=0
 
 supervisorctl_cmd() {
   supervisorctl -c /etc/supervisord.conf "$@"
@@ -73,26 +78,40 @@ wait_for_proxy() {
 }
 
 rollback() {
+  rollback_ok=1
   supervisorctl_cmd stop cliproxy >/dev/null 2>&1 || true
   if [ -f "$backup_binary" ]; then
-    install -m 0755 "$backup_binary" "$live_binary.rollback"
-    mv -f "$live_binary.rollback" "$live_binary"
+    install -m 0755 "$backup_binary" "$live_binary.rollback" && \
+      mv -f "$live_binary.rollback" "$live_binary" || rollback_ok=0
+  else
+    rollback_ok=0
   fi
   if [ -f "$backup_version" ]; then
-    install -m 0644 "$backup_version" "$live_version.rollback"
-    mv -f "$live_version.rollback" "$live_version"
+    install -m 0644 "$backup_version" "$live_version.rollback" && \
+      mv -f "$live_version.rollback" "$live_version" || rollback_ok=0
+  else
+    rollback_ok=0
   fi
-  supervisorctl_cmd start cliproxy >/dev/null
-  wait_for_proxy
+  supervisorctl_cmd start cliproxy >/dev/null || rollback_ok=0
+  wait_for_proxy || rollback_ok=0
+  [ "$rollback_ok" -eq 1 ]
 }
 
 on_exit() {
   rc=$?
   trap - EXIT INT TERM HUP
-  if [ "$rc" -ne 0 ] && [ "$swap_started" -eq 1 ]; then
-    rollback || true
+  rollback_failed=0
+  if [ "$rc" -ne 0 ] && [ "$maintenance_started" -eq 1 ]; then
+    if ! rollback; then
+      rollback_failed=1
+      printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2
+    fi
+  fi
+  if [ "$rollback_failed" -eq 0 ]; then
+    cleanup
+  else
+    rc=70
   fi
-  cleanup
   exit "$rc"
 }
 
@@ -107,17 +126,17 @@ test -s "$stage_version"
 cp -p "$live_binary" "$backup_binary"
 cp -p "$live_version" "$backup_version"
 
+maintenance_started=1
 supervisorctl_cmd stop cliproxy >/dev/null
-swap_started=1
 mv -f "$stage_binary" "$live_binary"
 mv -f "$stage_version" "$live_version"
 chmod 0755 "$live_binary"
 supervisorctl_cmd start cliproxy >/dev/null
 wait_for_proxy
-swap_started=0
+maintenance_started=0
 CONTAINER_UPDATE
 then
-  log 'CLIProxy Plus update failed; previous binary was restored and restarted'
+  log 'CLIProxy Plus update failed; rollback was attempted and reconciliation will verify service health'
   exit 1
 fi
 

docker/host/systemd/ccs-cliproxy-reconcile.service

@@ -6,4 +6,4 @@ Requires=docker.service
 [Service]
 Type=oneshot
 ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh
-TimeoutStartSec=5min
+TimeoutStartSec=10min