mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 00:14:02 +00:00
Skip scheduled sync of a remote source paused for reconnect
S3 and GitHub sources synced from a connection kept being retried on every schedule after their connection needed reconnecting, failing and notifying the owner each time. They now wait, like Drive and Confluence sources, and resume when the connection is reconnected.
This commit is contained in:
1 parent
84230409c6
commit
bbf83f02b2
2 files changed
+39
-1
No files matched your search
+18
-1
@@ -1519,7 +1519,7 @@ def sync_worker(self, frequency):
|
||||
with db_readonly() as conn:
|
||||
result = conn.execute(
|
||||
sql_text(
|
||||
"SELECT id, name, user_id, type, remote_data, retriever, connection_id "
|
||||
"SELECT id, name, user_id, type, remote_data, retriever, connection_id, metadata "
|
||||
"FROM sources WHERE sync_frequency = :freq"
|
||||
),
|
||||
{"freq": frequency},
|
||||
@@ -1549,6 +1549,23 @@ def sync_worker(self, frequency):
|
||||
sync_counts["sync_skipped"] += 1
|
||||
continue
|
||||
|
||||
metadata = doc.get("metadata")
|
||||
if isinstance(metadata, str):
|
||||
try:
|
||||
metadata = json.loads(metadata)
|
||||
except ValueError:
|
||||
metadata = {}
|
||||
if (
|
||||
doc.get("connection_id")
|
||||
and isinstance(metadata, dict)
|
||||
and metadata.get("sync_state") == "paused_reconnect"
|
||||
):
|
||||
# An S3 or GitHub source whose connection needs reconnecting:
|
||||
# it resumes when the owner reconnects, rather than failing (and
|
||||
# notifying) on every schedule until then.
|
||||
sync_counts["sync_skipped"] += 1
|
||||
continue
|
||||
|
||||
source_data = normalize_remote_data(source_type, doc.get("remote_data"))
|
||||
if not source_data:
|
||||
# No syncable URL/config — skip instead of dispatching a sync
|
||||
|
||||
@@ -239,6 +239,27 @@ class TestScheduledSync:
|
||||
assert counts["sync_dispatched"] == 1
|
||||
assert counts["sync_skipped"] == 1
|
||||
|
||||
def test_paused_repository_is_skipped_until_reconnected(self, pg_conn):
|
||||
"""A GitHub or S3 source paused for reconnect is not retried (and failed) on every schedule."""
|
||||
from docsgpt import worker
|
||||
|
||||
cid = _connection(pg_conn, provider="github", status="reconnect_needed",
|
||||
secrets={"credentials": {"access_token": "revoked"}})
|
||||
pg_conn.execute(text(
|
||||
"INSERT INTO sources (user_id, name, type, sync_frequency, connection_id, remote_data, metadata) "
|
||||
"VALUES ('alice', 'acme/api', 'github', 'daily', CAST(:c AS uuid), '{\"repo_url\": \"acme/api\"}', "
|
||||
"'{\"sync_state\": \"paused_reconnect\"}')"
|
||||
), {"c": cid})
|
||||
|
||||
@contextmanager
|
||||
def _yield():
|
||||
yield pg_conn
|
||||
|
||||
with patch.object(worker, "db_readonly", _yield), patch.object(worker, "sync") as sync:
|
||||
counts = worker.sync_worker(MagicMock(), "daily")
|
||||
sync.assert_not_called()
|
||||
assert counts["sync_skipped"] == 1
|
||||
|
||||
def test_paused_connection_is_not_synced(self, pg_conn):
|
||||
from docsgpt import worker
|
||||
|
||||
|
||||
Reference in new issue
Block a user