diff --git a/.github/workflows/dev-beta-release.yaml b/.github/workflows/dev-beta-release.yaml index 928f35ac..5cbd83c2 100644 --- a/.github/workflows/dev-beta-release.yaml +++ b/.github/workflows/dev-beta-release.yaml @@ -377,12 +377,21 @@ jobs: run: | base_url="${GOCLAW_DEPLOY_URL%/}" for attempt in {1..90}; do + status_err="$(mktemp)" status_json="$(curl -fsS --retry 3 --retry-delay 2 \ -H "Authorization: Bearer ${GOCLAW_GATEWAY_TOKEN}" \ -H "X-GoClaw-Upgrade-Token: ${GOCLAW_UPGRADE_TOKEN}" \ -H "X-GoClaw-User-Id: ${GOCLAW_DEPLOY_USER_ID}" \ - "${base_url}/v1/system/gateway/upgrade/status")" - state="$(python3 -c 'import json,sys; print(json.load(sys.stdin).get("state", ""))' <<< "$status_json")" + "${base_url}/v1/system/gateway/upgrade/status" 2>"$status_err" || true)" + if [[ -z "$status_json" ]]; then + echo "upgrade status unavailable; attempt ${attempt}/90" + cat "$status_err" + rm -f "$status_err" + sleep 10 + continue + fi + rm -f "$status_err" + state="$(python3 -c 'import json,sys; print(json.load(sys.stdin).get("state", ""))' <<< "$status_json" 2>/dev/null || true)" if [[ "$state" == "succeeded" ]]; then echo "$status_json" exit 0 diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 88c37c21..29bbd4f9 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -162,7 +162,7 @@ sudo /usr/local/bin/goclaw-upgrade-release latest sudo /usr/local/bin/goclaw-upgrade-release v3.12.0 ``` -The script downloads the Linux amd64 GitHub Release tarball from `digitopvn/goclaw`, follows GitHub release redirects, verifies `CHECKSUMS.sha256` when present, falls back to the GitHub release asset SHA256 digest for beta assets without checksum files, extracts to `/opt/goclaw/releases/`, and calls `goclaw-deploy`. +The script downloads the Linux amd64 GitHub Release tarball from `digitopvn/goclaw`, follows GitHub release redirects, verifies `CHECKSUMS.sha256` when present, falls back to the GitHub release asset SHA256 digest for beta assets without checksum files, extracts to `/opt/goclaw/releases/`, and calls `goclaw-deploy`. When invoked from the running gateway service, it first re-launches itself as a transient `systemd-run` unit so `goclaw-deploy` can stop/restart `goclaw` without killing the upgrade job. The HTTP API still accepts only `tag`; it does not accept repo names or custom download URLs. diff --git a/docs/project-changelog.md b/docs/project-changelog.md index bd9954e2..7707f953 100644 --- a/docs/project-changelog.md +++ b/docs/project-changelog.md @@ -30,6 +30,12 @@ Significant changes, features, and fixes in reverse chronological order. - Updated the host release-upgrade script to support beta asset filenames with a leading `v`. - Added checksum fallback to GitHub release asset SHA256 digests when beta releases do not publish `CHECKSUMS.sha256`. +- Detached gateway-triggered upgrades into a transient `systemd-run` unit so stopping `goclaw` during deploy no longer kills the upgrade job. +- Allowed stale upgrade `running` status records to be superseded after timeout and made the zuey deploy wait loop tolerate transient 502s during restart. + +**Tests** + +- Added regression coverage for stale gateway upgrade status recovery. --- diff --git a/internal/http/gateway_upgrade.go b/internal/http/gateway_upgrade.go index 73f17449..af04db14 100644 --- a/internal/http/gateway_upgrade.go +++ b/internal/http/gateway_upgrade.go @@ -22,6 +22,7 @@ const ( defaultGatewayUpgradeScript = "/usr/local/bin/goclaw-upgrade-release" defaultGatewayUpgradeStatus = "/var/lib/goclaw/update-jobs/current.json" gatewayUpgradeTokenHeader = "X-GoClaw-Upgrade-Token" + gatewayUpgradeRunningMaxAge = 30 * time.Minute ) var gatewayUpgradeTagRE = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-(beta|rc)\.[0-9]+)?$`) @@ -130,7 +131,7 @@ func (h *GatewayUpgradeHandler) handleStart(w http.ResponseWriter, r *http.Reque writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to read gateway upgrade status"}) return } - if status["state"] == "running" { + if gatewayUpgradeStatusRunning(status, time.Now().UTC()) { writeJSON(w, http.StatusConflict, map[string]string{"error": "gateway upgrade already running"}) return } @@ -195,6 +196,26 @@ func (h *GatewayUpgradeHandler) readStatus() (map[string]any, error) { return status, nil } +func gatewayUpgradeStatusRunning(status map[string]any, now time.Time) bool { + if status["state"] != "running" { + return false + } + startedRaw, ok := status["startedAt"].(string) + if !ok || strings.TrimSpace(startedRaw) == "" { + return true + } + startedAt, err := time.Parse(time.RFC3339, startedRaw) + if err != nil { + return true + } + age := now.Sub(startedAt) + if age < 0 || age <= gatewayUpgradeRunningMaxAge { + return true + } + slog.Warn("gateway upgrade stale running status ignored", "started_at", startedRaw, "age", age.String()) + return false +} + func (h *GatewayUpgradeHandler) writeRunningStatus(tag string) error { return h.writeStatus(map[string]any{ "jobId": time.Now().UTC().Format("20060102T150405Z") + "-" + tag, diff --git a/internal/http/gateway_upgrade_test.go b/internal/http/gateway_upgrade_test.go index 4737fb6e..9b1289ae 100644 --- a/internal/http/gateway_upgrade_test.go +++ b/internal/http/gateway_upgrade_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) type fakeGatewayUpgradeRunner struct { @@ -135,6 +136,30 @@ func TestGatewayUpgradeStartRejectsRunningJob(t *testing.T) { } } +func TestGatewayUpgradeStartAllowsStaleRunningJob(t *testing.T) { + dir := t.TempDir() + statusPath := filepath.Join(dir, "status.json") + staleStartedAt := time.Now().UTC().Add(-gatewayUpgradeRunningMaxAge - time.Minute).Format(time.RFC3339) + if err := os.WriteFile(statusPath, []byte(`{"state":"running","startedAt":"`+staleStartedAt+`"}`), 0o600); err != nil { + t.Fatalf("write status: %v", err) + } + runner := &fakeGatewayUpgradeRunner{} + h := &GatewayUpgradeHandler{StatusPath: statusPath, TriggerToken: "secret-token", Runner: runner} + req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`)) + req.Header.Set(gatewayUpgradeTokenHeader, "secret-token") + req = req.WithContext(ownerCtx(req.Context(), "gateway-stale-running-owner")) + w := httptest.NewRecorder() + + h.handleStart(w, req) + + if w.Code != http.StatusAccepted { + t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String()) + } + if len(runner.tags) != 1 || runner.tags[0] != "latest" { + t.Fatalf("runner tags = %#v, want [latest]", runner.tags) + } +} + func TestGatewayUpgradeTriggerTokenGuard(t *testing.T) { runner := &fakeGatewayUpgradeRunner{} h := &GatewayUpgradeHandler{ diff --git a/scripts/goclaw-upgrade-release.sh b/scripts/goclaw-upgrade-release.sh index f2056964..6c7c670e 100644 --- a/scripts/goclaw-upgrade-release.sh +++ b/scripts/goclaw-upgrade-release.sh @@ -9,6 +9,7 @@ STATUS_DIR="/var/lib/goclaw/update-jobs" STATUS_FILE="${STATUS_DIR}/current.json" STATUS_OWNER="${GOCLAW_STATUS_OWNER:-goclaw:goclaw}" DRY_RUN=0 +DETACHED="${GOCLAW_UPGRADE_DETACHED:-0}" log() { printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } json_escape() { python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$1"; } @@ -81,12 +82,42 @@ fi require_bin() { command -v "$1" >/dev/null 2>&1 || fail "missing dependency: $1"; } require_bin curl -require_bin flock require_bin tar require_bin sha256sum require_bin python3 +detach_into_systemd() { + if [ "$DRY_RUN" = "1" ] || [ "$DETACHED" = "1" ]; then + return + fi + if ! command -v systemd-run >/dev/null 2>&1 || [ ! -d /run/systemd/system ]; then + return + fi + if ! grep -q 'goclaw.service' "/proc/$$/cgroup" 2>/dev/null; then + return + fi + + local script_path unit_tag unit_name + script_path="$(readlink -f "$0" 2>/dev/null || true)" + if [ -z "$script_path" ]; then + script_path="$0" + fi + unit_tag="$(printf '%s' "$REQUESTED_TAG" | tr -c 'A-Za-z0-9_.-' '-')" + unit_name="goclaw-upgrade-${unit_tag}-$(date -u +%Y%m%dT%H%M%SZ)" + log "starting detached systemd upgrade unit=${unit_name}" + systemd-run \ + --unit="$unit_name" \ + --collect \ + --property=Type=oneshot \ + --setenv=GOCLAW_UPGRADE_DETACHED=1 \ + "$script_path" "$REQUESTED_TAG" + exit 0 +} + +detach_into_systemd + if [ "$DRY_RUN" != "1" ]; then + require_bin flock mkdir -p "$STATUS_DIR" exec 9>"${STATUS_DIR}/upgrade.lock" flock -n 9 || fail "gateway upgrade already running"