mirror of
https://github.com/tiennm99/telegram-exporter.git
synced 2026-10-11 03:13:49 +00:00
tiennm99
4624c506e5
A run reported one aggregate line, which could say how much was done but never what was happening: which files were moving, whether a stall was a slow download or a slow upload, or how long the rest would take. On a transfer measured in hours those are the only questions worth answering. The pipeline now emits a per-item lifecycle rather than only Stats, and a terminal renders it as an overall bar plus one bar per file in flight. Uploads get a spinner rather than a bar because rclone's MoveFile is a single blocking call with no byte callbacks; naming the file is still the point, since a run that looks stalled is usually waiting on one large object. Before the transfer, the run states what it found and what it will do. The survey breaks the outstanding set down by reason — never fetched, zero-byte, wrong size, unarchivable — because that is the difference between a run that will converge and one that cannot, and the old "N to fetch" hid it. The plan states the cap, staging path and concurrency, so a wrong setting is visible before hours of transfer rather than after. Redirected output keeps plain lines and gains one per archived file. Bars are continuous cursor movement, and a captured log of them is what tdl's progress bar did to the shell pipeline's logs. cmd/uidemo renders the whole thing against fake data. It is how the layout was checked without a session, and it earned its place immediately: the overall bar sat at zero because Stats only tracked the file count and never advanced it.
106 lines
2.6 KiB
Go
106 lines
2.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/iyear/tdl/core/downloader"
|
|
|
|
"github.com/tiennm99dev/telegram-exporter/internal/tgsource"
|
|
)
|
|
|
|
// Outcome is what happened to one item.
|
|
type Outcome struct {
|
|
Item tgsource.Item
|
|
Err error // nil when the file downloaded and was renamed into place
|
|
}
|
|
|
|
// Stats is a snapshot of a run's progress.
|
|
type Stats struct {
|
|
Started int
|
|
Done int
|
|
Failed int
|
|
BytesDone int64
|
|
BytesTotal int64
|
|
}
|
|
|
|
// progress collects per-item results and feeds an optional live reporter.
|
|
//
|
|
// The downloader calls these from its worker goroutines, so everything here is
|
|
// mutex-guarded. OnDone fires once per item whether it succeeded or not, which
|
|
// is what makes it the right place to finish the file — the downloader itself
|
|
// never closes or renames what To() handed it.
|
|
type progress struct {
|
|
mu sync.Mutex
|
|
stats Stats
|
|
outcomes []Outcome
|
|
inFlight map[int]int64 // message id -> bytes written so far
|
|
|
|
finish func(*elem, error) error
|
|
events Events
|
|
}
|
|
|
|
func newProgress(finish func(*elem, error) error, events Events) *progress {
|
|
if events == nil {
|
|
events = nopEvents{}
|
|
}
|
|
return &progress{
|
|
inFlight: make(map[int]int64),
|
|
finish: finish,
|
|
events: events,
|
|
}
|
|
}
|
|
|
|
func (p *progress) OnAdd(e downloader.Elem) {
|
|
el := e.(*elem)
|
|
p.mu.Lock()
|
|
p.stats.Started++
|
|
p.stats.BytesTotal += el.item.Size()
|
|
stats := p.stats
|
|
p.mu.Unlock()
|
|
p.events.DownloadStart(el.item)
|
|
p.events.Stats(stats)
|
|
}
|
|
|
|
func (p *progress) OnDownload(e downloader.Elem, state downloader.ProgressState) {
|
|
el := e.(*elem)
|
|
p.mu.Lock()
|
|
// State carries the running total for this item, not a delta, so the
|
|
// aggregate is adjusted by the difference since the last callback.
|
|
prev := p.inFlight[el.item.MessageID]
|
|
p.inFlight[el.item.MessageID] = state.Downloaded
|
|
p.stats.BytesDone += state.Downloaded - prev
|
|
stats := p.stats
|
|
p.mu.Unlock()
|
|
p.events.DownloadBytes(el.item, state.Downloaded)
|
|
p.events.Stats(stats)
|
|
}
|
|
|
|
func (p *progress) OnDone(e downloader.Elem, err error) {
|
|
el := e.(*elem)
|
|
|
|
// Closing and renaming happens here because this is the only callback that
|
|
// runs exactly once per item and knows whether it succeeded.
|
|
if ferr := p.finish(el, err); ferr != nil && err == nil {
|
|
err = ferr
|
|
}
|
|
|
|
p.mu.Lock()
|
|
delete(p.inFlight, el.item.MessageID)
|
|
if err != nil {
|
|
p.stats.Failed++
|
|
} else {
|
|
p.stats.Done++
|
|
}
|
|
p.outcomes = append(p.outcomes, Outcome{Item: el.item, Err: err})
|
|
stats := p.stats
|
|
p.mu.Unlock()
|
|
p.events.DownloadDone(el.item, err)
|
|
p.events.Stats(stats)
|
|
}
|
|
|
|
func (p *progress) results() ([]Outcome, Stats) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
return p.outcomes, p.stats
|
|
}
|