feat: show per-file progress and state the plan before a run starts

A run reported one aggregate line, which could say how much was done but never
what was happening: which files were moving, whether a stall was a slow
download or a slow upload, or how long the rest would take. On a transfer
measured in hours those are the only questions worth answering.

The pipeline now emits a per-item lifecycle rather than only Stats, and a
terminal renders it as an overall bar plus one bar per file in flight. Uploads
get a spinner rather than a bar because rclone's MoveFile is a single blocking
call with no byte callbacks; naming the file is still the point, since a run
that looks stalled is usually waiting on one large object.

Before the transfer, the run states what it found and what it will do. The
survey breaks the outstanding set down by reason — never fetched, zero-byte,
wrong size, unarchivable — because that is the difference between a run that
will converge and one that cannot, and the old "N to fetch" hid it. The plan
states the cap, staging path and concurrency, so a wrong setting is visible
before hours of transfer rather than after.

Redirected output keeps plain lines and gains one per archived file. Bars are
continuous cursor movement, and a captured log of them is what tdl's progress
bar did to the shell pipeline's logs.

cmd/uidemo renders the whole thing against fake data. It is how the layout was
checked without a session, and it earned its place immediately: the overall bar
sat at zero because Stats only tracked the file count and never advanced it.
This commit is contained in:
tiennm99 committed 2026-09-06 21:21:07 +07:00
1 parent fafeab4750
commit 4624c506e5
13 files changed
+644 -51

No files matched your search

+16 -6
View File
@@ -142,8 +142,7 @@ func syncCmd(ctx context.Context, args []string) error {
}
warnCollisions(idx)
before := verify.Check(items, idx)
fmt.Fprintf(os.Stderr, "%d media messages, %d already archived, %d to fetch\n",
before.Expected, before.Present, len(before.Todo()))
report.Survey(os.Stderr, before)
todo := selectTodo(items, before, *limitItems)
if len(todo) == 0 {
@@ -155,13 +154,24 @@ func syncCmd(ctx context.Context, args []string) error {
return err
}
var todoBytes int64
var todoBytes, largest int64
for _, it := range todo {
todoBytes += it.Size()
largest = max(largest, it.Size())
}
fmt.Fprintf(os.Stderr, "fetching %d file(s), %.1f GiB\n", len(todo), float64(todoBytes)/(1<<30))
report.Plan(os.Stderr, report.PlanInfo{
Files: len(todo),
Bytes: todoBytes,
Largest: largest,
Budget: budget,
Staging: *staging,
Threads: *threads,
Downloads: *limit,
Uploads: *uploads,
Destination: dst.String(),
})
rep := report.New(os.Stderr, len(todo), todoBytes)
rep := report.Events(os.Stderr, len(todo), todoBytes)
var res pipeline.Result
res, runErr = pipeline.Run(ctx, sliceSeq(todo), pipeline.Options{
Pool: pool,
@@ -173,7 +183,7 @@ func syncCmd(ctx context.Context, args []string) error {
Budget: budget,
Confirm: *confirm,
Takeout: *takeout,
Report: rep.Update,
Events: rep,
// Re-checked during the run, not only before it: an archive of this
// size runs for hours, and the destination can fill in the middle.
FreeBytes: func(ctx context.Context) (int64, bool) {
+75
View File
@@ -0,0 +1,75 @@
// Command uidemo renders the sync progress UI with fake data, so the layout can
// be checked without a Telegram session or a multi-hour transfer.
package main
import (
"fmt"
"os"
"time"
"github.com/iyear/tdl/core/tmedia"
"github.com/tiennm99dev/telegram-exporter/internal/pipeline"
"github.com/tiennm99dev/telegram-exporter/internal/report"
"github.com/tiennm99dev/telegram-exporter/internal/tgsource"
"github.com/tiennm99dev/telegram-exporter/internal/verify"
)
func item(id int, name string, size int64) tgsource.Item {
return tgsource.Item{MessageID: id, Name: name, Media: &tmedia.Media{Size: size}}
}
func main() {
files := []tgsource.Item{
item(4242, "1234567890_4242_1000000000000000001.mp4", 96<<20),
item(4243, "1234567890_4243_1000000000000000002.mp4", 1900<<20),
item(4244, "1234567890_4244_short.jpg", 2<<20),
}
var total int64
for _, f := range files {
total += f.Size()
}
fmt.Fprintln(os.Stderr, "PikPak root 'mychannel' has 1438 GiB free")
fmt.Fprintln(os.Stderr, "reading mychannel")
fmt.Fprintln(os.Stderr, " 12,000 messages read in 4m31s")
fmt.Fprintln(os.Stderr, "indexing PikPak root 'mychannel'")
fmt.Fprintln(os.Stderr, " 11,406 objects listed in 2m10s")
survey := verify.Report{Expected: 12000, Present: 11400, Bytes: 518 << 30}
for i := range 2607 {
survey.Absent = append(survey.Absent, 4242+i)
}
for i := range 6 {
survey.Mismatched = append(survey.Mismatched,
verify.Mismatch{MessageID: 14290 + i, Want: 2 << 30, Got: 221 << 20})
}
report.Survey(os.Stderr, survey)
report.Plan(os.Stderr, report.PlanInfo{
Files: 2613, Bytes: 79 << 30, Largest: 2 << 30, Budget: 40 << 30,
Staging: "./staging", Threads: 4, Downloads: 2, Uploads: 2,
Destination: "PikPak root 'mychannel'",
})
ev := report.Events(os.Stderr, 2613, 79<<30)
st := pipeline.Stats{}
for _, f := range files {
ev.DownloadStart(f)
}
for step := range 30 {
for _, f := range files {
ev.DownloadBytes(f, f.Size()*int64(step+1)/30)
}
st.BytesDone += total / 30
ev.Stats(st)
time.Sleep(60 * time.Millisecond)
}
for i, f := range files {
ev.DownloadDone(f, nil)
ev.UploadStart(f)
st.Done = i + 1
ev.Stats(st)
time.Sleep(400 * time.Millisecond)
ev.UploadDone(f, nil)
}
ev.Finish(st)
}