refactor(server): dispatch event for reachability notifications, drop retry loop

Move reachability notification triggering out of isReachableChanged into
a dedicated ServerReachabilityChanged event dispatched by
ServerConnectionCheckJob. Remove the blocking 3-attempt sleep loop from
isReachableChanged — unreachable_count threshold alone now gates the
Unreachable notification. Add feature and unit tests covering all
notification dispatch paths.

app/Jobs/ServerConnectionCheckJob.php

@@ -2,6 +2,7 @@
 
 namespace App\Jobs;
 
+use App\Events\ServerReachabilityChanged;
 use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
 use App\Services\ConfigurationRepository;
@@ -43,6 +44,9 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
 
     public function handle()
     {
+        $wasReachable = (bool) $this->server->settings->is_reachable;
+        $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
         try {
             // Check if server is disabled
             if ($this->server->settings->force_disabled) {
@@ -84,6 +88,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                     'server_ip' => $this->server->ip,
                 ]);
 
+                $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
                 return;
             }
 
@@ -99,6 +105,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
                 $this->server->update(['unreachable_count' => 0]);
             }
 
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, true);
+
         } catch (\Throwable $e) {
 
             Log::error('ServerConnectionCheckJob failed', [
@@ -111,6 +119,8 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
             ]);
             $this->server->increment('unreachable_count');
 
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
             return;
         }
     }
@@ -118,17 +128,41 @@ class ServerConnectionCheckJob implements ShouldBeEncrypted, ShouldQueue
     public function failed(?\Throwable $exception): void
     {
         if ($exception instanceof TimeoutExceededException) {
+            $wasReachable = (bool) $this->server->settings->is_reachable;
+            $wasNotified = (bool) $this->server->unreachable_notification_sent;
+
             $this->server->settings->update([
                 'is_reachable' => false,
                 'is_usable' => false,
             ]);
             $this->server->increment('unreachable_count');
 
+            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
+
             // Delete the queue job so it doesn't appear in Horizon's failed list.
             $this->job?->delete();
         }
     }
 
+    /**
+     * Fire ServerReachabilityChanged when state crosses the unreachable threshold (count >= 2)
+     * or when a previously-notified server recovers. Skips noise from single transient flaps.
+     */
+    private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $wasNotified, bool $isReachable): void
+    {
+        if ($isReachable) {
+            if (! $wasReachable || $wasNotified) {
+                ServerReachabilityChanged::dispatch($this->server);
+            }
+
+            return;
+        }
+
+        if ($this->server->unreachable_count >= 2 && ! $wasNotified) {
+            ServerReachabilityChanged::dispatch($this->server);
+        }
+    }
+
     private function checkHetznerStatus(): void
     {
         $status = null;

app/Models/Server.php

@@ -1236,10 +1236,8 @@ $schema://$host {
         $this->refresh();
         $unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
         $isReachable = (bool) $this->settings->is_reachable;
-        if ($isReachable === true) {
-            $this->unreachable_count = 0;
-            $this->save();
 
+        if ($isReachable === true) {
             if ($unreachableNotificationSent === true) {
                 $this->sendReachableNotification();
             }
@@ -1247,28 +1245,8 @@ $schema://$host {
             return;
         }
 
-        $this->increment('unreachable_count');
-
-        if ($this->unreachable_count === 1) {
-            $this->settings->is_reachable = true;
-            $this->settings->save();
-
-            return;
-        }
-
         if ($this->unreachable_count >= 2 && ! $unreachableNotificationSent) {
-            $failedChecks = 0;
-            for ($i = 0; $i < 3; $i++) {
-                $status = $this->serverStatus();
-                sleep(5);
-                if (! $status) {
-                    $failedChecks++;
-                }
-            }
-
-            if ($failedChecks === 3 && ! $unreachableNotificationSent) {
-                $this->sendUnreachableNotification();
-            }
+            $this->sendUnreachableNotification();
         }
     }