mirror of
https://github.com/tiennm99/composes.git
synced 2026-10-11 12:09:25 +00:00
feat(gitea-mirror): raise the Gitea clone timeout to one hour
Gitea's default migrate and fetch timeouts cut multi-gigabyte mirrors off mid-clone. GITEA_CLONE_TIMEOUT sets both, defaulting to 3600 seconds.
This commit is contained in:
tiennm99 committed
1 parent
d4882c6f3e
commit
a84e158738
5 files changed
+8
-327
No files matched your search
-131
@@ -1,131 +0,0 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Acts on the plan produced by detect-failed-mirrors.ps1.
|
||||
Dry-run by default: prints the exact commands and changes nothing.
|
||||
|
||||
.DESCRIPTION
|
||||
case A delete repo in Gitea, then reset its mirror-DB row to 'imported'
|
||||
so the next scheduled run re-migrates it
|
||||
case B delete repo in Gitea only (upstream is gone; re-mirror would fail)
|
||||
case C never touched - reported for retry
|
||||
case D reset the stale mirror-DB row only, no deletion
|
||||
|
||||
.EXAMPLE
|
||||
./cleanup-failed-mirrors.ps1 # dry run
|
||||
./cleanup-failed-mirrors.ps1 -Apply # execute
|
||||
./cleanup-failed-mirrors.ps1 -Apply -Case A
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[switch] $Apply,
|
||||
[string] $Login = 'localhost',
|
||||
[string[]]$Case = @('A', 'B', 'D'),
|
||||
[string] $ComposeDir = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..\..')).Path,
|
||||
[string] $MirrorSvc = 'gitea-mirror',
|
||||
[string] $DbPath = '//app/data/gitea-mirror.db',
|
||||
[string] $PlanFile = (Join-Path $env:TEMP 'gitea-mirror-failed-plan.json'),
|
||||
[int] $TimeoutSec = 120
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not (Test-Path $PlanFile)) { throw "plan not found: $PlanFile - run detect-failed-mirrors.ps1 first" }
|
||||
$plan = Get-Content $PlanFile -Raw | ConvertFrom-Json
|
||||
if (-not $plan) { 'Plan is empty - nothing to do.'; return }
|
||||
|
||||
# Cases C and E are advisory only and can never be selected for action:
|
||||
# C still holds content, E is a clone in progress.
|
||||
$targets = @($plan | Where-Object { $_.case -in $Case -and $_.case -notin 'C', 'E' })
|
||||
$skipped = @($plan | Where-Object { $_.case -in 'C', 'E' })
|
||||
|
||||
if ($skipped.Count -gt 0) {
|
||||
"`n=== NOT TOUCHED (cases C and E) ==="
|
||||
$skipped | Sort-Object case, full_name |
|
||||
Format-Table case, full_name, reason -AutoSize -Wrap | Out-String -Width 200
|
||||
}
|
||||
|
||||
# A plan goes stale quickly while the scheduler is running: a repo that was a
|
||||
# broken shell can be re-queued, and a queued one can complete.
|
||||
$planAge = (Get-Date) - (Get-Item $PlanFile).LastWriteTime
|
||||
if ($planAge.TotalMinutes -gt 15) {
|
||||
Write-Warning ("plan is {0:N0} min old - re-run detect-failed-mirrors.ps1 before applying" -f $planAge.TotalMinutes)
|
||||
}
|
||||
|
||||
if ($targets.Count -eq 0) { "No repos match case(s): $($Case -join ',')"; return }
|
||||
|
||||
# tea reads stdin on some paths; run it as a job so it can never hang the script.
|
||||
# The job must not start inside a git work tree: tea infers its target from the
|
||||
# local remote, and when that remote matches no configured login it discards
|
||||
# --login and fails with "remote repository required". Run from a neutral dir.
|
||||
function Invoke-Tea([string[]]$TeaArgs, [int]$Timeout) {
|
||||
$j = Start-Job -ArgumentList $TeaArgs, $env:TEMP {
|
||||
param($a, $neutral) Set-Location $neutral; & tea @a 2>&1; "EXIT:$LASTEXITCODE"
|
||||
}
|
||||
if (Wait-Job $j -Timeout $Timeout) {
|
||||
$out = Receive-Job $j; Remove-Job $j -Force
|
||||
$code = ($out | Where-Object { $_ -like 'EXIT:*' }) -replace 'EXIT:', ''
|
||||
$msg = ($out | Where-Object { $_ -notlike 'EXIT:*' }) -join ' '
|
||||
return [pscustomobject]@{ ok = ($code -eq '0'); code = $code; msg = $msg }
|
||||
}
|
||||
Stop-Job $j; Remove-Job $j -Force
|
||||
return [pscustomobject]@{ ok = $false; code = 'timeout'; msg = "no response in ${Timeout}s" }
|
||||
}
|
||||
|
||||
function Reset-MirrorRow([string]$FullName) {
|
||||
# Single-quote escaping for SQLite string literals.
|
||||
$safe = $FullName.Replace("'", "''")
|
||||
$sql = "UPDATE repositories SET status='imported', last_mirrored=NULL, error_message=NULL WHERE full_name='$safe';"
|
||||
Push-Location $ComposeDir
|
||||
try {
|
||||
$out = docker compose exec -T $MirrorSvc sqlite3 $DbPath $sql 2>&1
|
||||
return [pscustomobject]@{ ok = ($LASTEXITCODE -eq 0); msg = ($out -join ' ') }
|
||||
} finally { Pop-Location }
|
||||
}
|
||||
|
||||
if (-not $Apply) {
|
||||
"`n=== DRY RUN - no changes made ===`n"
|
||||
foreach ($t in ($targets | Sort-Object case, full_name)) {
|
||||
"# $($t.full_name) (case $($t.case): $($t.reason))"
|
||||
if ($t.action -like 'delete*') {
|
||||
"tea repos delete --login $Login --owner $($t.owner) --name $($t.name) --force"
|
||||
}
|
||||
if ($t.action -like '*reset*') {
|
||||
"docker compose exec -T $MirrorSvc sqlite3 $DbPath ""UPDATE repositories SET status='imported', last_mirrored=NULL, error_message=NULL WHERE full_name='$($t.full_name)';"""
|
||||
}
|
||||
''
|
||||
}
|
||||
"{0} repo(s) would be actioned. Re-run with -Apply to execute." -f $targets.Count
|
||||
return
|
||||
}
|
||||
|
||||
"`n=== APPLYING to $($targets.Count) repo(s) ===`n"
|
||||
$results = foreach ($t in ($targets | Sort-Object case, full_name)) {
|
||||
$delOk = $null; $resetOk = $null; $note = ''
|
||||
|
||||
if ($t.action -like 'delete*') {
|
||||
$r = Invoke-Tea @('repos', 'delete', '--login', $Login, '--owner', $t.owner, '--name', $t.name, '--force') $TimeoutSec
|
||||
$delOk = $r.ok
|
||||
if (-not $r.ok) { $note = "delete failed ($($r.code)) $($r.msg)" }
|
||||
Write-Host (" {0} delete {1}" -f $(if ($r.ok) { 'OK ' } else { 'FAIL' }), $t.full_name) `
|
||||
-ForegroundColor $(if ($r.ok) { 'Green' } else { 'Red' })
|
||||
}
|
||||
|
||||
# Only reset after a successful delete, so a live repo is never marked pending.
|
||||
if ($t.action -like '*reset*' -and ($delOk -ne $false)) {
|
||||
$r = Reset-MirrorRow $t.full_name
|
||||
$resetOk = $r.ok
|
||||
if (-not $r.ok) { $note = ($note + " reset failed: $($r.msg)").Trim() }
|
||||
Write-Host (" {0} reset {1}" -f $(if ($r.ok) { 'OK ' } else { 'FAIL' }), $t.full_name) `
|
||||
-ForegroundColor $(if ($r.ok) { 'Green' } else { 'Red' })
|
||||
}
|
||||
|
||||
[pscustomobject]@{ full_name = $t.full_name; case = $t.case; deleted = $delOk; reset = $resetOk; note = $note }
|
||||
}
|
||||
|
||||
"`n=== RESULT ==="
|
||||
$results | Format-Table full_name, case, deleted, reset, note -AutoSize -Wrap | Out-String -Width 200
|
||||
$failed = @($results | Where-Object { $_.deleted -eq $false -or $_.reset -eq $false })
|
||||
"{0} succeeded, {1} failed." -f ($results.Count - $failed.Count), $failed.Count
|
||||
if ($results | Where-Object { $_.reset }) {
|
||||
'Reset repos will be re-migrated on the next scheduled run. Verify with detect-failed-mirrors.ps1 afterwards.'
|
||||
}
|
||||
-196
@@ -1,196 +0,0 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Read-only audit of the local Gitea mirror stack. Classifies every failing
|
||||
mirror into cases A-D and writes a JSON plan for cleanup-failed-mirrors.ps1.
|
||||
|
||||
.DESCRIPTION
|
||||
Gathers four independent failure signals:
|
||||
1. Gitea API - mirrors with empty:true and no completed initial pull
|
||||
2. Upstream - HEAD probe of original_url, to tell "retry" from "gone"
|
||||
3. mirror DB - repositories rows with status='failed'
|
||||
4. gitea log - mirror_pull.go [E] SyncMirrors errors (periodic sync)
|
||||
|
||||
Makes no changes. Safe to run any time.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Login = 'localhost',
|
||||
[string]$ApiBase = 'http://localhost:3000/api/v1',
|
||||
[string]$ComposeDir = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..\..')).Path,
|
||||
[string]$MirrorSvc = 'gitea-mirror',
|
||||
[string]$GiteaSvc = 'gitea',
|
||||
[string]$DbPath = '//app/data/gitea-mirror.db',
|
||||
[string]$OutFile = (Join-Path $env:TEMP 'gitea-mirror-failed-plan.json'),
|
||||
[int] $MaxPages = 60
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# --- token for the requested tea login -------------------------------------
|
||||
function Get-TeaToken([string]$LoginName) {
|
||||
$cfgPath = Join-Path $env:LOCALAPPDATA 'tea\config.yml'
|
||||
if (-not (Test-Path $cfgPath)) { throw "tea config not found at $cfgPath" }
|
||||
$inLogin = $false
|
||||
foreach ($line in Get-Content $cfgPath) {
|
||||
if ($line -match "^\s*-?\s*name:\s*$([regex]::Escape($LoginName))\s*$") { $inLogin = $true; continue }
|
||||
if ($inLogin -and $line -match '^\s*-\s*name:') { break }
|
||||
if ($inLogin -and $line -match '^\s*token:\s*(\S+)') { return $Matches[1] }
|
||||
}
|
||||
throw "no token found for tea login '$LoginName' - run: tea logins add"
|
||||
}
|
||||
|
||||
$token = Get-TeaToken $Login
|
||||
$headers = @{ Authorization = "token $token" }
|
||||
|
||||
# --- signal 1: every repo the token can see --------------------------------
|
||||
Write-Host 'Scanning Gitea repositories...' -ForegroundColor Cyan
|
||||
$all = @()
|
||||
for ($page = 1; $page -le $MaxPages; $page++) {
|
||||
$r = Invoke-RestMethod -Headers $headers -Uri "$ApiBase/repos/search?limit=50&page=$page"
|
||||
if (-not $r.data -or $r.data.Count -eq 0) { break }
|
||||
$all += $r.data
|
||||
}
|
||||
$mirrors = $all | Where-Object { $_.mirror }
|
||||
$empty = $all | Where-Object { $_.empty }
|
||||
Write-Host (" {0} repos, {1} mirrors, {2} empty" -f $all.Count, $mirrors.Count, $empty.Count)
|
||||
|
||||
# --- signal 4: periodic-sync errors from the gitea container log -----------
|
||||
Write-Host 'Reading gitea container log...' -ForegroundColor Cyan
|
||||
$syncErrorRepos = @{}
|
||||
Push-Location $ComposeDir
|
||||
try {
|
||||
$log = docker compose logs $GiteaSvc --no-log-prefix 2>&1
|
||||
foreach ($m in [regex]::Matches(($log -join "`n"), 'repo: <Repository \d+:([^>]+)>')) {
|
||||
$syncErrorRepos[$m.Groups[1].Value] = $true
|
||||
}
|
||||
} finally { Pop-Location }
|
||||
Write-Host (" {0} repos with sync errors in log" -f $syncErrorRepos.Count)
|
||||
|
||||
# --- signal 3: mirror-app DB state for every tracked repo ------------------
|
||||
# The per-repo status is what distinguishes a broken shell from a clone that is
|
||||
# still in progress, so fetch all of them, not only the failed ones.
|
||||
Write-Host 'Querying gitea-mirror database...' -ForegroundColor Cyan
|
||||
$dbStatus = @{}
|
||||
$dbFailed = @{}
|
||||
Push-Location $ComposeDir
|
||||
try {
|
||||
$rows = docker compose exec -T $MirrorSvc sqlite3 -separator '|' $DbPath `
|
||||
"SELECT full_name, status, substr(replace(coalesce(error_message,''),'|',' '),1,160) FROM repositories;" 2>&1
|
||||
foreach ($row in $rows) {
|
||||
if ($row -match '^([^|]+)\|([^|]*)\|(.*)$') {
|
||||
$fn = $Matches[1].Trim(); $st = $Matches[2].Trim()
|
||||
$dbStatus[$fn] = $st
|
||||
if ($st -eq 'failed') { $dbFailed[$fn] = $Matches[3].Trim() }
|
||||
}
|
||||
}
|
||||
} finally { Pop-Location }
|
||||
Write-Host (" {0} tracked rows, {1} with status='failed'" -f $dbStatus.Count, $dbFailed.Count)
|
||||
|
||||
# --- signal 2 + classification ---------------------------------------------
|
||||
Write-Host 'Probing upstreams and classifying...' -ForegroundColor Cyan
|
||||
$plan = @()
|
||||
|
||||
foreach ($repo in $empty) {
|
||||
# An unset mirror_updated means the initial migration never finished.
|
||||
$neverPulled = (-not $repo.mirror_updated) -or ([datetime]$repo.mirror_updated).Year -le 1
|
||||
$st = $dbStatus[$repo.full_name]
|
||||
|
||||
# A clone still in progress looks exactly like a broken shell: empty, size 0,
|
||||
# mirror_updated unset. Only the mirror app's own status tells them apart, so
|
||||
# never action a repo it is still working on or has not attempted yet.
|
||||
if ($st -in 'mirroring', 'imported') {
|
||||
$owner, $name = $repo.full_name -split '/', 2
|
||||
$plan += [pscustomobject]@{
|
||||
full_name = $repo.full_name; owner = $owner; name = $name
|
||||
case = 'E'; action = 'report-only'
|
||||
size_MB = [math]::Round($repo.size / 1024, 1)
|
||||
reason = "empty but mirror DB status='$st' - in flight or queued, leave alone"
|
||||
db_status = $st
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
$upAlive = $true
|
||||
$upNote = 'no original_url - assumed alive'
|
||||
if ($repo.original_url) {
|
||||
try {
|
||||
$resp = Invoke-WebRequest -Uri $repo.original_url -Method Head -TimeoutSec 20 -MaximumRedirection 5
|
||||
$upNote = "HTTP $($resp.StatusCode)"
|
||||
} catch {
|
||||
$code = $null
|
||||
if ($_.Exception.Response) { $code = $_.Exception.Response.StatusCode.value__ }
|
||||
# Only a definite 404/410 proves the upstream is gone; treat network
|
||||
# trouble as "alive" so a flaky connection never deletes permanently.
|
||||
if ($code -in 404, 410) { $upAlive = $false; $upNote = "HTTP $code gone" }
|
||||
else { $upNote = "probe failed ($(if($code){"HTTP $code"}else{'unreachable'})) - assumed alive" }
|
||||
}
|
||||
}
|
||||
|
||||
$owner, $name = $repo.full_name -split '/', 2
|
||||
$plan += [pscustomobject]@{
|
||||
full_name = $repo.full_name
|
||||
owner = $owner
|
||||
name = $name
|
||||
case = if ($upAlive) { 'A' } else { 'B' }
|
||||
action = if ($upAlive) { 'delete+reset' } else { 'delete' }
|
||||
size_MB = [math]::Round($repo.size / 1024, 1)
|
||||
reason = "empty:true$(if($neverPulled){', initial pull never completed'}); mirror DB status='$(if($st){$st}else{'(untracked)'})'; upstream $upNote"
|
||||
db_status = $st
|
||||
}
|
||||
}
|
||||
|
||||
# Case C: has content but the periodic sync is erroring. Never delete these.
|
||||
$emptyNames = @($empty | ForEach-Object { $_.full_name })
|
||||
foreach ($fn in $syncErrorRepos.Keys) {
|
||||
if ($emptyNames -contains $fn) { continue }
|
||||
$owner, $name = $fn -split '/', 2
|
||||
$plan += [pscustomobject]@{
|
||||
full_name = $fn; owner = $owner; name = $name
|
||||
case = 'C'; action = 'report-only'; size_MB = $null
|
||||
reason = 'sync error in gitea log but repo has content - retry, do not delete'
|
||||
db_status = $null
|
||||
}
|
||||
}
|
||||
|
||||
# Case D: mirror DB says failed, but the repo itself looks healthy.
|
||||
foreach ($fn in $dbFailed.Keys) {
|
||||
if ($emptyNames -contains $fn) { continue }
|
||||
$owner, $name = $fn -split '/', 2
|
||||
$plan += [pscustomobject]@{
|
||||
full_name = $fn; owner = $owner; name = $name
|
||||
case = 'D'; action = 'reset-only'; size_MB = $null
|
||||
reason = "repo healthy in Gitea but mirror DB status='failed'"
|
||||
db_status = $dbFailed[$fn]
|
||||
}
|
||||
}
|
||||
|
||||
# Annotate any empty repo that the DB also flagged.
|
||||
foreach ($p in $plan) {
|
||||
if ($dbFailed.ContainsKey($p.full_name) -and -not $p.db_status) { $p.db_status = $dbFailed[$p.full_name] }
|
||||
}
|
||||
|
||||
# --- report ----------------------------------------------------------------
|
||||
"`n=== FAILED MIRROR REPORT ===`n"
|
||||
if ($plan.Count -eq 0) {
|
||||
'No failing mirrors detected. Nothing to clean up.'
|
||||
} else {
|
||||
$plan | Sort-Object case, full_name |
|
||||
Format-Table case, action, full_name, size_MB, reason -AutoSize -Wrap | Out-String -Width 200
|
||||
|
||||
$plan | Group-Object case | Sort-Object Name | ForEach-Object {
|
||||
$desc = switch ($_.Name) {
|
||||
'A' { 'broken shell, upstream alive -> delete + reset for re-mirror' }
|
||||
'B' { 'broken shell, upstream gone -> delete only' }
|
||||
'C' { 'has content, sync erroring -> RETRY, never delete' }
|
||||
'D' { 'healthy repo, stale DB status -> reset only' }
|
||||
'E' { 'clone in flight or queued -> LEAVE ALONE' }
|
||||
}
|
||||
" case {0}: {1,3} repo(s) {2}" -f $_.Name, $_.Count, $desc
|
||||
}
|
||||
$reclaim = ($plan | Where-Object { $_.case -in 'A','B' } | Measure-Object size_MB -Sum).Sum
|
||||
"`n reclaimable: {0} MB" -f [math]::Round($reclaim, 1)
|
||||
}
|
||||
|
||||
$plan | ConvertTo-Json -Depth 4 | Set-Content -Encoding UTF8 $OutFile
|
||||
"`nPlan written to: $OutFile"
|
||||
'Nothing was changed. To act on it, run cleanup-failed-mirrors.ps1 (add -Apply to execute).'
|
||||
@@ -1,5 +1,6 @@
|
||||
POSTGRES_PASSWORD=gitea
|
||||
GITEA_ROOT_URL=https://gitea.example.com/
|
||||
GITEA_CLONE_TIMEOUT=3600
|
||||
BETTER_AUTH_SECRET=
|
||||
ENCRYPTION_SECRET=
|
||||
GITEA_MIRROR_URL=https://gitea-mirror.example.com
|
||||
@@ -24,6 +24,7 @@ port, matching the two URL variables.
|
||||
| --- | --- | --- |
|
||||
| `POSTGRES_PASSWORD` | `db`, `gitea` | Defaults to `gitea`. |
|
||||
| `GITEA_ROOT_URL` | Gitea `server.ROOT_URL` | Public URL, with trailing slash. Gitea builds clone URLs and redirects from it. |
|
||||
| `GITEA_CLONE_TIMEOUT` | Gitea `git.timeout` `MIGRATE` and `MIRROR` | Seconds a mirror's first clone or a later fetch may run. Defaults to `3600`. |
|
||||
| `BETTER_AUTH_SECRET` | gitea-mirror | Signs sessions and encrypts its login keys. Generate with `openssl rand -base64 32`. |
|
||||
| `ENCRYPTION_SECRET` | gitea-mirror | Encrypts the stored GitHub and Gitea tokens. Generate with `openssl rand -base64 48`. |
|
||||
| `GITEA_MIRROR_URL` | `BETTER_AUTH_URL`, `PUBLIC_BETTER_AUTH_URL`, `BETTER_AUTH_TRUSTED_ORIGINS` | Public URL of the mirror UI, no trailing slash. |
|
||||
@@ -50,6 +51,10 @@ install.
|
||||
|
||||
- **HTTPS only.** The proxy routes HTTP, not SSH, so Gitea's SSH server is
|
||||
disabled and the UI offers HTTPS clone URLs only.
|
||||
- **One-hour clone timeout.** Gitea's defaults (600 s to migrate, 300 s to
|
||||
fetch) cut off multi-gigabyte repositories mid-clone, leaving empty mirrors
|
||||
that still hold gigabytes of unreachable packfiles. gitea-mirror sets no
|
||||
timeout of its own on the migrate request, so Gitea's is the one that counts.
|
||||
- **`gitea/gitea:28`.** Gitea publishes major tags; the major pin takes
|
||||
updates without a surprise major upgrade.
|
||||
- **`gitea-mirror:latest`** with `pull_policy: always`: upstream publishes no
|
||||
|
||||
@@ -28,6 +28,8 @@ services:
|
||||
GITEA__database__USER: gitea
|
||||
GITEA__server__ROOT_URL: ${GITEA_ROOT_URL:?required}
|
||||
GITEA__server__DISABLE_SSH: "true"
|
||||
GITEA__git_0X2E_timeout__MIGRATE: ${GITEA_CLONE_TIMEOUT:-3600}
|
||||
GITEA__git_0X2E_timeout__MIRROR: ${GITEA_CLONE_TIMEOUT:-3600}
|
||||
volumes:
|
||||
- gitea-data:/data
|
||||
healthcheck:
|
||||
|
||||
Reference in new issue
Block a user