mirror of
https://github.com/tiennm99/keepalive.git
synced 2026-10-14 18:19:01 +00:00
fix(runner): retry failed services after one minute and fail fast on bad config
A failed connect or tick now closes the connection and reconnects after one minute, re-running initialization so a dropped table or row is recreated. Before, a failing tick logged every interval forever and connect retries logged every 10 seconds. Connection URLs that fail to parse no longer appear in logs, since their error repeats the password. An unknown adapter or a missing required config key now stops keepalive at startup instead of silently idling one service.
This commit is contained in:
1 parent
30fd62a46c
commit
42c8a871cf
5 files changed
+168
-32
No files matched your search
+88
-3
@@ -3,6 +3,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -34,9 +37,9 @@ func (a *retryConnectAdapter) Close(context.Context) error {
|
||||
}
|
||||
|
||||
func TestRunServiceRetriesConnectFailure(t *testing.T) {
|
||||
oldReconnectDelay := reconnectDelay
|
||||
reconnectDelay = 10 * time.Millisecond
|
||||
defer func() { reconnectDelay = oldReconnectDelay }()
|
||||
oldRetryDelay := retryDelay
|
||||
retryDelay = 10 * time.Millisecond
|
||||
defer func() { retryDelay = oldRetryDelay }()
|
||||
|
||||
var connects atomic.Int32
|
||||
connected := make(chan struct{})
|
||||
@@ -79,3 +82,85 @@ func TestRunServiceRetriesConnectFailure(t *testing.T) {
|
||||
t.Fatalf("connect attempts = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
type failingIncrementAdapter struct {
|
||||
connects *atomic.Int32
|
||||
closes *atomic.Int32
|
||||
}
|
||||
|
||||
func (a *failingIncrementAdapter) Connect(context.Context) error {
|
||||
a.connects.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *failingIncrementAdapter) Increment(context.Context) (int64, error) {
|
||||
return 0, errors.New("relation \"keepalive\" does not exist")
|
||||
}
|
||||
|
||||
func (a *failingIncrementAdapter) Close(context.Context) error {
|
||||
a.closes.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRunServiceReconnectsAfterIncrementFailure(t *testing.T) {
|
||||
oldRetryDelay := retryDelay
|
||||
retryDelay = 10 * time.Millisecond
|
||||
defer func() { retryDelay = oldRetryDelay }()
|
||||
|
||||
var connects, closes atomic.Int32
|
||||
adapter.Registry["fail-tick-test"] = func(adapter.Config) (adapter.Adapter, error) {
|
||||
return &failingIncrementAdapter{connects: &connects, closes: &closes}, nil
|
||||
}
|
||||
defer delete(adapter.Registry, "fail-tick-test")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var wg sync.WaitGroup
|
||||
runService(ctx, &wg, serviceConfig{
|
||||
Name: "fail-tick-test",
|
||||
AdapterType: "fail-tick-test",
|
||||
Interval: 5 * time.Millisecond,
|
||||
Config: adapter.Config{},
|
||||
})
|
||||
|
||||
deadline := time.After(time.Second)
|
||||
for connects.Load() < 3 {
|
||||
select {
|
||||
case <-deadline:
|
||||
cancel()
|
||||
t.Fatalf("connect attempts = %d, want at least 3", connects.Load())
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
wg.Wait()
|
||||
|
||||
if closes.Load() < connects.Load()-1 {
|
||||
t.Fatalf("closes = %d for %d connects; each failed session must be closed", closes.Load(), connects.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactURLErrorHidesPassword(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"postgres://user:S3CR%ETpw@db.example.com/keepalive",
|
||||
"redis://user:S3CRETpw@cache.example.com:port/0",
|
||||
} {
|
||||
_, err := url.Parse(raw)
|
||||
if err == nil {
|
||||
t.Fatalf("%s: want parse error", raw)
|
||||
}
|
||||
got := redactURLError(fmt.Errorf("connect: %w", err)).Error()
|
||||
if strings.Contains(got, "S3CR") {
|
||||
t.Fatalf("redacted error leaks password: %s", got)
|
||||
}
|
||||
if !strings.Contains(got, "invalid connection URL") {
|
||||
t.Fatalf("redacted error = %q, want a URL hint", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactURLErrorKeepsOtherErrors(t *testing.T) {
|
||||
err := errors.New("dial tcp: connection refused")
|
||||
if got := redactURLError(err); got != err {
|
||||
t.Fatalf("redactURLError changed a non-URL error: %v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user