mirror of
https://github.com/tiennm99/telegram-exporter.git
synced 2026-10-17 04:15:17 +00:00
tiennm99
efb29c9c39
core's downloader logs a failed transfer and returns nil, and nothing here installed a logger, so logctx handed out a nop and the reason was destroyed. The size check in finish was all that survived, which reported a failed two-gigabyte fetch as "short download: got 0 bytes" — a symptom with no cause an operator can act on. The logger it reaches for now feeds a core that keeps error entries and stores them on the element they belong to, so finish reports the reason ahead of the byte count. A failed download is also retried inside the run, up to three passes over whatever is still missing and spaced minutes apart, because the failure this repairs is transient: a dead connection takes every transfer in flight with it and all of them are fetchable again afterwards, while leaving them to the next run costs a full re-walk of the chat and a re-index of the remote first. An item is counted once however many attempts it took, and a pass that recovers reports nothing. The download leg gains the breaker the upload leg already had. A source that refuses everything refuses the rest of the list in milliseconds, so the pass gives up after five failures in a row and the run exits 3 rather than spending the entire outstanding list finding that out.
81 lines
2.5 KiB
Go
81 lines
2.5 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zapcore"
|
|
|
|
"github.com/iyear/tdl/core/logctx"
|
|
)
|
|
|
|
// captureCauses points core's downloader logging back into this package.
|
|
//
|
|
// core's Download never returns why a transfer failed. It logs the error and
|
|
// returns nil (downloader.go:47-60), so the size check in finish is the only
|
|
// evidence anything went wrong — and a size check can say how many bytes
|
|
// arrived, never why the rest did not. Nothing here installed a logger either,
|
|
// which means logctx handed out zap.NewNop() and the reason was destroyed: a
|
|
// failed two-gigabyte fetch was reported as "short download: got 0 bytes" with
|
|
// the flood wait, expired file reference, or dead connection behind it gone.
|
|
//
|
|
// So the error is taken from the one place it exists. The entry carries the
|
|
// element it belongs to, which is this package's own *elem, so the error is
|
|
// stored on it and finish reports it beside the size.
|
|
func captureCauses(ctx context.Context) context.Context {
|
|
return logctx.With(ctx, zap.New(&causeCore{}))
|
|
}
|
|
|
|
// causeCore is a zapcore.Core that keeps error entries and drops everything
|
|
// else. Fields are matched on their type rather than their key so a rename
|
|
// upstream cannot silently switch the capture off.
|
|
type causeCore struct{ fields []zapcore.Field }
|
|
|
|
func (c *causeCore) Enabled(l zapcore.Level) bool { return l >= zapcore.ErrorLevel }
|
|
|
|
func (c *causeCore) With(fields []zapcore.Field) zapcore.Core {
|
|
joined := make([]zapcore.Field, 0, len(c.fields)+len(fields))
|
|
joined = append(joined, c.fields...)
|
|
joined = append(joined, fields...)
|
|
return &causeCore{fields: joined}
|
|
}
|
|
|
|
func (c *causeCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry {
|
|
if c.Enabled(e.Level) {
|
|
return ce.AddCore(e, c)
|
|
}
|
|
return ce
|
|
}
|
|
|
|
// Write pairs a logged error with the element it was logged for.
|
|
//
|
|
// The write happens on the download worker's own goroutine, immediately before
|
|
// the deferred OnDone that reads it, so storing the error on the element needs
|
|
// no synchronisation.
|
|
func (c *causeCore) Write(_ zapcore.Entry, fields []zapcore.Field) error {
|
|
var (
|
|
el *elem
|
|
err error
|
|
)
|
|
for _, set := range [][]zapcore.Field{c.fields, fields} {
|
|
for _, f := range set {
|
|
switch f.Type {
|
|
case zapcore.ErrorType:
|
|
if e, ok := f.Interface.(error); ok {
|
|
err = e
|
|
}
|
|
case zapcore.ReflectType:
|
|
if e, ok := f.Interface.(*elem); ok {
|
|
el = e
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if el != nil && err != nil {
|
|
el.cause = err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *causeCore) Sync() error { return nil }
|