diff --git a/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/cleanup-failed-mirrors.ps1 b/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/cleanup-failed-mirrors.ps1 deleted file mode 100644 index de0c274..0000000 --- a/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/cleanup-failed-mirrors.ps1 +++ /dev/null @@ -1,131 +0,0 @@ -<# -.SYNOPSIS - Acts on the plan produced by detect-failed-mirrors.ps1. - Dry-run by default: prints the exact commands and changes nothing. - -.DESCRIPTION - case A delete repo in Gitea, then reset its mirror-DB row to 'imported' - so the next scheduled run re-migrates it - case B delete repo in Gitea only (upstream is gone; re-mirror would fail) - case C never touched - reported for retry - case D reset the stale mirror-DB row only, no deletion - -.EXAMPLE - ./cleanup-failed-mirrors.ps1 # dry run - ./cleanup-failed-mirrors.ps1 -Apply # execute - ./cleanup-failed-mirrors.ps1 -Apply -Case A -#> -[CmdletBinding()] -param( - [switch] $Apply, - [string] $Login = 'localhost', - [string[]]$Case = @('A', 'B', 'D'), - [string] $ComposeDir = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..\..')).Path, - [string] $MirrorSvc = 'gitea-mirror', - [string] $DbPath = '//app/data/gitea-mirror.db', - [string] $PlanFile = (Join-Path $env:TEMP 'gitea-mirror-failed-plan.json'), - [int] $TimeoutSec = 120 -) - -$ErrorActionPreference = 'Stop' - -if (-not (Test-Path $PlanFile)) { throw "plan not found: $PlanFile - run detect-failed-mirrors.ps1 first" } -$plan = Get-Content $PlanFile -Raw | ConvertFrom-Json -if (-not $plan) { 'Plan is empty - nothing to do.'; return } - -# Cases C and E are advisory only and can never be selected for action: -# C still holds content, E is a clone in progress. -$targets = @($plan | Where-Object { $_.case -in $Case -and $_.case -notin 'C', 'E' }) -$skipped = @($plan | Where-Object { $_.case -in 'C', 'E' }) - -if ($skipped.Count -gt 0) { - "`n=== NOT TOUCHED (cases C and E) ===" - $skipped | Sort-Object case, full_name | - Format-Table case, full_name, reason -AutoSize -Wrap | Out-String -Width 200 -} - -# A plan goes stale quickly while the scheduler is running: a repo that was a -# broken shell can be re-queued, and a queued one can complete. -$planAge = (Get-Date) - (Get-Item $PlanFile).LastWriteTime -if ($planAge.TotalMinutes -gt 15) { - Write-Warning ("plan is {0:N0} min old - re-run detect-failed-mirrors.ps1 before applying" -f $planAge.TotalMinutes) -} - -if ($targets.Count -eq 0) { "No repos match case(s): $($Case -join ',')"; return } - -# tea reads stdin on some paths; run it as a job so it can never hang the script. -# The job must not start inside a git work tree: tea infers its target from the -# local remote, and when that remote matches no configured login it discards -# --login and fails with "remote repository required". Run from a neutral dir. -function Invoke-Tea([string[]]$TeaArgs, [int]$Timeout) { - $j = Start-Job -ArgumentList $TeaArgs, $env:TEMP { - param($a, $neutral) Set-Location $neutral; & tea @a 2>&1; "EXIT:$LASTEXITCODE" - } - if (Wait-Job $j -Timeout $Timeout) { - $out = Receive-Job $j; Remove-Job $j -Force - $code = ($out | Where-Object { $_ -like 'EXIT:*' }) -replace 'EXIT:', '' - $msg = ($out | Where-Object { $_ -notlike 'EXIT:*' }) -join ' ' - return [pscustomobject]@{ ok = ($code -eq '0'); code = $code; msg = $msg } - } - Stop-Job $j; Remove-Job $j -Force - return [pscustomobject]@{ ok = $false; code = 'timeout'; msg = "no response in ${Timeout}s" } -} - -function Reset-MirrorRow([string]$FullName) { - # Single-quote escaping for SQLite string literals. - $safe = $FullName.Replace("'", "''") - $sql = "UPDATE repositories SET status='imported', last_mirrored=NULL, error_message=NULL WHERE full_name='$safe';" - Push-Location $ComposeDir - try { - $out = docker compose exec -T $MirrorSvc sqlite3 $DbPath $sql 2>&1 - return [pscustomobject]@{ ok = ($LASTEXITCODE -eq 0); msg = ($out -join ' ') } - } finally { Pop-Location } -} - -if (-not $Apply) { - "`n=== DRY RUN - no changes made ===`n" - foreach ($t in ($targets | Sort-Object case, full_name)) { - "# $($t.full_name) (case $($t.case): $($t.reason))" - if ($t.action -like 'delete*') { - "tea repos delete --login $Login --owner $($t.owner) --name $($t.name) --force" - } - if ($t.action -like '*reset*') { - "docker compose exec -T $MirrorSvc sqlite3 $DbPath ""UPDATE repositories SET status='imported', last_mirrored=NULL, error_message=NULL WHERE full_name='$($t.full_name)';""" - } - '' - } - "{0} repo(s) would be actioned. Re-run with -Apply to execute." -f $targets.Count - return -} - -"`n=== APPLYING to $($targets.Count) repo(s) ===`n" -$results = foreach ($t in ($targets | Sort-Object case, full_name)) { - $delOk = $null; $resetOk = $null; $note = '' - - if ($t.action -like 'delete*') { - $r = Invoke-Tea @('repos', 'delete', '--login', $Login, '--owner', $t.owner, '--name', $t.name, '--force') $TimeoutSec - $delOk = $r.ok - if (-not $r.ok) { $note = "delete failed ($($r.code)) $($r.msg)" } - Write-Host (" {0} delete {1}" -f $(if ($r.ok) { 'OK ' } else { 'FAIL' }), $t.full_name) ` - -ForegroundColor $(if ($r.ok) { 'Green' } else { 'Red' }) - } - - # Only reset after a successful delete, so a live repo is never marked pending. - if ($t.action -like '*reset*' -and ($delOk -ne $false)) { - $r = Reset-MirrorRow $t.full_name - $resetOk = $r.ok - if (-not $r.ok) { $note = ($note + " reset failed: $($r.msg)").Trim() } - Write-Host (" {0} reset {1}" -f $(if ($r.ok) { 'OK ' } else { 'FAIL' }), $t.full_name) ` - -ForegroundColor $(if ($r.ok) { 'Green' } else { 'Red' }) - } - - [pscustomobject]@{ full_name = $t.full_name; case = $t.case; deleted = $delOk; reset = $resetOk; note = $note } -} - -"`n=== RESULT ===" -$results | Format-Table full_name, case, deleted, reset, note -AutoSize -Wrap | Out-String -Width 200 -$failed = @($results | Where-Object { $_.deleted -eq $false -or $_.reset -eq $false }) -"{0} succeeded, {1} failed." -f ($results.Count - $failed.Count), $failed.Count -if ($results | Where-Object { $_.reset }) { - 'Reset repos will be re-migrated on the next scheduled run. Verify with detect-failed-mirrors.ps1 afterwards.' -} diff --git a/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/detect-failed-mirrors.ps1 b/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/detect-failed-mirrors.ps1 deleted file mode 100644 index 03c3ed0..0000000 --- a/gitea-mirror/.claude/skills/gitea-mirror-maintenance/scripts/detect-failed-mirrors.ps1 +++ /dev/null @@ -1,196 +0,0 @@ -<# -.SYNOPSIS - Read-only audit of the local Gitea mirror stack. Classifies every failing - mirror into cases A-D and writes a JSON plan for cleanup-failed-mirrors.ps1. - -.DESCRIPTION - Gathers four independent failure signals: - 1. Gitea API - mirrors with empty:true and no completed initial pull - 2. Upstream - HEAD probe of original_url, to tell "retry" from "gone" - 3. mirror DB - repositories rows with status='failed' - 4. gitea log - mirror_pull.go [E] SyncMirrors errors (periodic sync) - - Makes no changes. Safe to run any time. -#> -[CmdletBinding()] -param( - [string]$Login = 'localhost', - [string]$ApiBase = 'http://localhost:3000/api/v1', - [string]$ComposeDir = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..\..')).Path, - [string]$MirrorSvc = 'gitea-mirror', - [string]$GiteaSvc = 'gitea', - [string]$DbPath = '//app/data/gitea-mirror.db', - [string]$OutFile = (Join-Path $env:TEMP 'gitea-mirror-failed-plan.json'), - [int] $MaxPages = 60 -) - -$ErrorActionPreference = 'Stop' - -# --- token for the requested tea login ------------------------------------- -function Get-TeaToken([string]$LoginName) { - $cfgPath = Join-Path $env:LOCALAPPDATA 'tea\config.yml' - if (-not (Test-Path $cfgPath)) { throw "tea config not found at $cfgPath" } - $inLogin = $false - foreach ($line in Get-Content $cfgPath) { - if ($line -match "^\s*-?\s*name:\s*$([regex]::Escape($LoginName))\s*$") { $inLogin = $true; continue } - if ($inLogin -and $line -match '^\s*-\s*name:') { break } - if ($inLogin -and $line -match '^\s*token:\s*(\S+)') { return $Matches[1] } - } - throw "no token found for tea login '$LoginName' - run: tea logins add" -} - -$token = Get-TeaToken $Login -$headers = @{ Authorization = "token $token" } - -# --- signal 1: every repo the token can see -------------------------------- -Write-Host 'Scanning Gitea repositories...' -ForegroundColor Cyan -$all = @() -for ($page = 1; $page -le $MaxPages; $page++) { - $r = Invoke-RestMethod -Headers $headers -Uri "$ApiBase/repos/search?limit=50&page=$page" - if (-not $r.data -or $r.data.Count -eq 0) { break } - $all += $r.data -} -$mirrors = $all | Where-Object { $_.mirror } -$empty = $all | Where-Object { $_.empty } -Write-Host (" {0} repos, {1} mirrors, {2} empty" -f $all.Count, $mirrors.Count, $empty.Count) - -# --- signal 4: periodic-sync errors from the gitea container log ----------- -Write-Host 'Reading gitea container log...' -ForegroundColor Cyan -$syncErrorRepos = @{} -Push-Location $ComposeDir -try { - $log = docker compose logs $GiteaSvc --no-log-prefix 2>&1 - foreach ($m in [regex]::Matches(($log -join "`n"), 'repo: ]+)>')) { - $syncErrorRepos[$m.Groups[1].Value] = $true - } -} finally { Pop-Location } -Write-Host (" {0} repos with sync errors in log" -f $syncErrorRepos.Count) - -# --- signal 3: mirror-app DB state for every tracked repo ------------------ -# The per-repo status is what distinguishes a broken shell from a clone that is -# still in progress, so fetch all of them, not only the failed ones. -Write-Host 'Querying gitea-mirror database...' -ForegroundColor Cyan -$dbStatus = @{} -$dbFailed = @{} -Push-Location $ComposeDir -try { - $rows = docker compose exec -T $MirrorSvc sqlite3 -separator '|' $DbPath ` - "SELECT full_name, status, substr(replace(coalesce(error_message,''),'|',' '),1,160) FROM repositories;" 2>&1 - foreach ($row in $rows) { - if ($row -match '^([^|]+)\|([^|]*)\|(.*)$') { - $fn = $Matches[1].Trim(); $st = $Matches[2].Trim() - $dbStatus[$fn] = $st - if ($st -eq 'failed') { $dbFailed[$fn] = $Matches[3].Trim() } - } - } -} finally { Pop-Location } -Write-Host (" {0} tracked rows, {1} with status='failed'" -f $dbStatus.Count, $dbFailed.Count) - -# --- signal 2 + classification --------------------------------------------- -Write-Host 'Probing upstreams and classifying...' -ForegroundColor Cyan -$plan = @() - -foreach ($repo in $empty) { - # An unset mirror_updated means the initial migration never finished. - $neverPulled = (-not $repo.mirror_updated) -or ([datetime]$repo.mirror_updated).Year -le 1 - $st = $dbStatus[$repo.full_name] - - # A clone still in progress looks exactly like a broken shell: empty, size 0, - # mirror_updated unset. Only the mirror app's own status tells them apart, so - # never action a repo it is still working on or has not attempted yet. - if ($st -in 'mirroring', 'imported') { - $owner, $name = $repo.full_name -split '/', 2 - $plan += [pscustomobject]@{ - full_name = $repo.full_name; owner = $owner; name = $name - case = 'E'; action = 'report-only' - size_MB = [math]::Round($repo.size / 1024, 1) - reason = "empty but mirror DB status='$st' - in flight or queued, leave alone" - db_status = $st - } - continue - } - - $upAlive = $true - $upNote = 'no original_url - assumed alive' - if ($repo.original_url) { - try { - $resp = Invoke-WebRequest -Uri $repo.original_url -Method Head -TimeoutSec 20 -MaximumRedirection 5 - $upNote = "HTTP $($resp.StatusCode)" - } catch { - $code = $null - if ($_.Exception.Response) { $code = $_.Exception.Response.StatusCode.value__ } - # Only a definite 404/410 proves the upstream is gone; treat network - # trouble as "alive" so a flaky connection never deletes permanently. - if ($code -in 404, 410) { $upAlive = $false; $upNote = "HTTP $code gone" } - else { $upNote = "probe failed ($(if($code){"HTTP $code"}else{'unreachable'})) - assumed alive" } - } - } - - $owner, $name = $repo.full_name -split '/', 2 - $plan += [pscustomobject]@{ - full_name = $repo.full_name - owner = $owner - name = $name - case = if ($upAlive) { 'A' } else { 'B' } - action = if ($upAlive) { 'delete+reset' } else { 'delete' } - size_MB = [math]::Round($repo.size / 1024, 1) - reason = "empty:true$(if($neverPulled){', initial pull never completed'}); mirror DB status='$(if($st){$st}else{'(untracked)'})'; upstream $upNote" - db_status = $st - } -} - -# Case C: has content but the periodic sync is erroring. Never delete these. -$emptyNames = @($empty | ForEach-Object { $_.full_name }) -foreach ($fn in $syncErrorRepos.Keys) { - if ($emptyNames -contains $fn) { continue } - $owner, $name = $fn -split '/', 2 - $plan += [pscustomobject]@{ - full_name = $fn; owner = $owner; name = $name - case = 'C'; action = 'report-only'; size_MB = $null - reason = 'sync error in gitea log but repo has content - retry, do not delete' - db_status = $null - } -} - -# Case D: mirror DB says failed, but the repo itself looks healthy. -foreach ($fn in $dbFailed.Keys) { - if ($emptyNames -contains $fn) { continue } - $owner, $name = $fn -split '/', 2 - $plan += [pscustomobject]@{ - full_name = $fn; owner = $owner; name = $name - case = 'D'; action = 'reset-only'; size_MB = $null - reason = "repo healthy in Gitea but mirror DB status='failed'" - db_status = $dbFailed[$fn] - } -} - -# Annotate any empty repo that the DB also flagged. -foreach ($p in $plan) { - if ($dbFailed.ContainsKey($p.full_name) -and -not $p.db_status) { $p.db_status = $dbFailed[$p.full_name] } -} - -# --- report ---------------------------------------------------------------- -"`n=== FAILED MIRROR REPORT ===`n" -if ($plan.Count -eq 0) { - 'No failing mirrors detected. Nothing to clean up.' -} else { - $plan | Sort-Object case, full_name | - Format-Table case, action, full_name, size_MB, reason -AutoSize -Wrap | Out-String -Width 200 - - $plan | Group-Object case | Sort-Object Name | ForEach-Object { - $desc = switch ($_.Name) { - 'A' { 'broken shell, upstream alive -> delete + reset for re-mirror' } - 'B' { 'broken shell, upstream gone -> delete only' } - 'C' { 'has content, sync erroring -> RETRY, never delete' } - 'D' { 'healthy repo, stale DB status -> reset only' } - 'E' { 'clone in flight or queued -> LEAVE ALONE' } - } - " case {0}: {1,3} repo(s) {2}" -f $_.Name, $_.Count, $desc - } - $reclaim = ($plan | Where-Object { $_.case -in 'A','B' } | Measure-Object size_MB -Sum).Sum - "`n reclaimable: {0} MB" -f [math]::Round($reclaim, 1) -} - -$plan | ConvertTo-Json -Depth 4 | Set-Content -Encoding UTF8 $OutFile -"`nPlan written to: $OutFile" -'Nothing was changed. To act on it, run cleanup-failed-mirrors.ps1 (add -Apply to execute).' diff --git a/gitea-mirror/.env.example b/gitea-mirror/.env.example index b68109b..f182af6 100644 --- a/gitea-mirror/.env.example +++ b/gitea-mirror/.env.example @@ -1,5 +1,6 @@ POSTGRES_PASSWORD=gitea GITEA_ROOT_URL=https://gitea.example.com/ +GITEA_CLONE_TIMEOUT=3600 BETTER_AUTH_SECRET= ENCRYPTION_SECRET= GITEA_MIRROR_URL=https://gitea-mirror.example.com diff --git a/gitea-mirror/README.md b/gitea-mirror/README.md index d2e89f2..56184a0 100644 --- a/gitea-mirror/README.md +++ b/gitea-mirror/README.md @@ -24,6 +24,7 @@ port, matching the two URL variables. | --- | --- | --- | | `POSTGRES_PASSWORD` | `db`, `gitea` | Defaults to `gitea`. | | `GITEA_ROOT_URL` | Gitea `server.ROOT_URL` | Public URL, with trailing slash. Gitea builds clone URLs and redirects from it. | +| `GITEA_CLONE_TIMEOUT` | Gitea `git.timeout` `MIGRATE` and `MIRROR` | Seconds a mirror's first clone or a later fetch may run. Defaults to `3600`. | | `BETTER_AUTH_SECRET` | gitea-mirror | Signs sessions and encrypts its login keys. Generate with `openssl rand -base64 32`. | | `ENCRYPTION_SECRET` | gitea-mirror | Encrypts the stored GitHub and Gitea tokens. Generate with `openssl rand -base64 48`. | | `GITEA_MIRROR_URL` | `BETTER_AUTH_URL`, `PUBLIC_BETTER_AUTH_URL`, `BETTER_AUTH_TRUSTED_ORIGINS` | Public URL of the mirror UI, no trailing slash. | @@ -50,6 +51,10 @@ install. - **HTTPS only.** The proxy routes HTTP, not SSH, so Gitea's SSH server is disabled and the UI offers HTTPS clone URLs only. +- **One-hour clone timeout.** Gitea's defaults (600 s to migrate, 300 s to + fetch) cut off multi-gigabyte repositories mid-clone, leaving empty mirrors + that still hold gigabytes of unreachable packfiles. gitea-mirror sets no + timeout of its own on the migrate request, so Gitea's is the one that counts. - **`gitea/gitea:28`.** Gitea publishes major tags; the major pin takes updates without a surprise major upgrade. - **`gitea-mirror:latest`** with `pull_policy: always`: upstream publishes no diff --git a/gitea-mirror/compose.yml b/gitea-mirror/compose.yml index 08085ea..60638af 100644 --- a/gitea-mirror/compose.yml +++ b/gitea-mirror/compose.yml @@ -28,6 +28,8 @@ services: GITEA__database__USER: gitea GITEA__server__ROOT_URL: ${GITEA_ROOT_URL:?required} GITEA__server__DISABLE_SSH: "true" + GITEA__git_0X2E_timeout__MIGRATE: ${GITEA_CLONE_TIMEOUT:-3600} + GITEA__git_0X2E_timeout__MIRROR: ${GITEA_CLONE_TIMEOUT:-3600} volumes: - gitea-data:/data healthcheck: