fix(deploy): detach gateway upgrades from service (#45)

.github/workflows/dev-beta-release.yaml

@@ -377,12 +377,21 @@ jobs:
         run: |
           base_url="${GOCLAW_DEPLOY_URL%/}"
           for attempt in {1..90}; do
+            status_err="$(mktemp)"
             status_json="$(curl -fsS --retry 3 --retry-delay 2 \
               -H "Authorization: Bearer ${GOCLAW_GATEWAY_TOKEN}" \
               -H "X-GoClaw-Upgrade-Token: ${GOCLAW_UPGRADE_TOKEN}" \
               -H "X-GoClaw-User-Id: ${GOCLAW_DEPLOY_USER_ID}" \
-              "${base_url}/v1/system/gateway/upgrade/status")"
-            state="$(python3 -c 'import json,sys; print(json.load(sys.stdin).get("state", ""))' <<< "$status_json")"
+              "${base_url}/v1/system/gateway/upgrade/status" 2>"$status_err" || true)"
+            if [[ -z "$status_json" ]]; then
+              echo "upgrade status unavailable; attempt ${attempt}/90"
+              cat "$status_err"
+              rm -f "$status_err"
+              sleep 10
+              continue
+            fi
+            rm -f "$status_err"
+            state="$(python3 -c 'import json,sys; print(json.load(sys.stdin).get("state", ""))' <<< "$status_json" 2>/dev/null || true)"
             if [[ "$state" == "succeeded" ]]; then
               echo "$status_json"
               exit 0

docs/deployment-guide.md

@@ -162,7 +162,7 @@ sudo /usr/local/bin/goclaw-upgrade-release latest
 sudo /usr/local/bin/goclaw-upgrade-release v3.12.0
 ```
 
-The script downloads the Linux amd64 GitHub Release tarball from `digitopvn/goclaw`, follows GitHub release redirects, verifies `CHECKSUMS.sha256` when present, falls back to the GitHub release asset SHA256 digest for beta assets without checksum files, extracts to `/opt/goclaw/releases/<tag>`, and calls `goclaw-deploy`.
+The script downloads the Linux amd64 GitHub Release tarball from `digitopvn/goclaw`, follows GitHub release redirects, verifies `CHECKSUMS.sha256` when present, falls back to the GitHub release asset SHA256 digest for beta assets without checksum files, extracts to `/opt/goclaw/releases/<tag>`, and calls `goclaw-deploy`. When invoked from the running gateway service, it first re-launches itself as a transient `systemd-run` unit so `goclaw-deploy` can stop/restart `goclaw` without killing the upgrade job.
 
 The HTTP API still accepts only `tag`; it does not accept repo names or custom download URLs.
 

docs/project-changelog.md

@@ -30,6 +30,12 @@ Significant changes, features, and fixes in reverse chronological order.
 
 - Updated the host release-upgrade script to support beta asset filenames with a leading `v`.
 - Added checksum fallback to GitHub release asset SHA256 digests when beta releases do not publish `CHECKSUMS.sha256`.
+- Detached gateway-triggered upgrades into a transient `systemd-run` unit so stopping `goclaw` during deploy no longer kills the upgrade job.
+- Allowed stale upgrade `running` status records to be superseded after timeout and made the zuey deploy wait loop tolerate transient 502s during restart.
+
+**Tests**
+
+- Added regression coverage for stale gateway upgrade status recovery.
 
 ---
 

internal/http/gateway_upgrade.go

@@ -22,6 +22,7 @@ const (
 	defaultGatewayUpgradeScript = "/usr/local/bin/goclaw-upgrade-release"
 	defaultGatewayUpgradeStatus = "/var/lib/goclaw/update-jobs/current.json"
 	gatewayUpgradeTokenHeader   = "X-GoClaw-Upgrade-Token"
+	gatewayUpgradeRunningMaxAge = 30 * time.Minute
 )
 
 var gatewayUpgradeTagRE = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-(beta|rc)\.[0-9]+)?$`)
@@ -130,7 +131,7 @@ func (h *GatewayUpgradeHandler) handleStart(w http.ResponseWriter, r *http.Reque
 		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to read gateway upgrade status"})
 		return
 	}
-	if status["state"] == "running" {
+	if gatewayUpgradeStatusRunning(status, time.Now().UTC()) {
 		writeJSON(w, http.StatusConflict, map[string]string{"error": "gateway upgrade already running"})
 		return
 	}
@@ -195,6 +196,26 @@ func (h *GatewayUpgradeHandler) readStatus() (map[string]any, error) {
 	return status, nil
 }
 
+func gatewayUpgradeStatusRunning(status map[string]any, now time.Time) bool {
+	if status["state"] != "running" {
+		return false
+	}
+	startedRaw, ok := status["startedAt"].(string)
+	if !ok || strings.TrimSpace(startedRaw) == "" {
+		return true
+	}
+	startedAt, err := time.Parse(time.RFC3339, startedRaw)
+	if err != nil {
+		return true
+	}
+	age := now.Sub(startedAt)
+	if age < 0 || age <= gatewayUpgradeRunningMaxAge {
+		return true
+	}
+	slog.Warn("gateway upgrade stale running status ignored", "started_at", startedRaw, "age", age.String())
+	return false
+}
+
 func (h *GatewayUpgradeHandler) writeRunningStatus(tag string) error {
 	return h.writeStatus(map[string]any{
 		"jobId":        time.Now().UTC().Format("20060102T150405Z") + "-" + tag,

internal/http/gateway_upgrade_test.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 )
 
 type fakeGatewayUpgradeRunner struct {
@@ -135,6 +136,30 @@ func TestGatewayUpgradeStartRejectsRunningJob(t *testing.T) {
 	}
 }
 
+func TestGatewayUpgradeStartAllowsStaleRunningJob(t *testing.T) {
+	dir := t.TempDir()
+	statusPath := filepath.Join(dir, "status.json")
+	staleStartedAt := time.Now().UTC().Add(-gatewayUpgradeRunningMaxAge - time.Minute).Format(time.RFC3339)
+	if err := os.WriteFile(statusPath, []byte(`{"state":"running","startedAt":"`+staleStartedAt+`"}`), 0o600); err != nil {
+		t.Fatalf("write status: %v", err)
+	}
+	runner := &fakeGatewayUpgradeRunner{}
+	h := &GatewayUpgradeHandler{StatusPath: statusPath, TriggerToken: "secret-token", Runner: runner}
+	req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
+	req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
+	req = req.WithContext(ownerCtx(req.Context(), "gateway-stale-running-owner"))
+	w := httptest.NewRecorder()
+
+	h.handleStart(w, req)
+
+	if w.Code != http.StatusAccepted {
+		t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String())
+	}
+	if len(runner.tags) != 1 || runner.tags[0] != "latest" {
+		t.Fatalf("runner tags = %#v, want [latest]", runner.tags)
+	}
+}
+
 func TestGatewayUpgradeTriggerTokenGuard(t *testing.T) {
 	runner := &fakeGatewayUpgradeRunner{}
 	h := &GatewayUpgradeHandler{

scripts/goclaw-upgrade-release.sh

@@ -9,6 +9,7 @@ STATUS_DIR="/var/lib/goclaw/update-jobs"
 STATUS_FILE="${STATUS_DIR}/current.json"
 STATUS_OWNER="${GOCLAW_STATUS_OWNER:-goclaw:goclaw}"
 DRY_RUN=0
+DETACHED="${GOCLAW_UPGRADE_DETACHED:-0}"
 
 log() { printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
 json_escape() { python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$1"; }
@@ -81,12 +82,42 @@ fi
 
 require_bin() { command -v "$1" >/dev/null 2>&1 || fail "missing dependency: $1"; }
 require_bin curl
-require_bin flock
 require_bin tar
 require_bin sha256sum
 require_bin python3
 
+detach_into_systemd() {
+  if [ "$DRY_RUN" = "1" ] || [ "$DETACHED" = "1" ]; then
+    return
+  fi
+  if ! command -v systemd-run >/dev/null 2>&1 || [ ! -d /run/systemd/system ]; then
+    return
+  fi
+  if ! grep -q 'goclaw.service' "/proc/$$/cgroup" 2>/dev/null; then
+    return
+  fi
+
+  local script_path unit_tag unit_name
+  script_path="$(readlink -f "$0" 2>/dev/null || true)"
+  if [ -z "$script_path" ]; then
+    script_path="$0"
+  fi
+  unit_tag="$(printf '%s' "$REQUESTED_TAG" | tr -c 'A-Za-z0-9_.-' '-')"
+  unit_name="goclaw-upgrade-${unit_tag}-$(date -u +%Y%m%dT%H%M%SZ)"
+  log "starting detached systemd upgrade unit=${unit_name}"
+  systemd-run \
+    --unit="$unit_name" \
+    --collect \
+    --property=Type=oneshot \
+    --setenv=GOCLAW_UPGRADE_DETACHED=1 \
+    "$script_path" "$REQUESTED_TAG"
+  exit 0
+}
+
+detach_into_systemd
+
 if [ "$DRY_RUN" != "1" ]; then
+  require_bin flock
   mkdir -p "$STATUS_DIR"
   exec 9>"${STATUS_DIR}/upgrade.lock"
   flock -n 9 || fail "gateway upgrade already running"