From bd16c305ebb5f941815a07a4428a872c84d404f9 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Wed, 27 May 2026 17:49:02 +0700 Subject: [PATCH] feat(scribe): add runnable docker-compose example with Go test sender Introduces a complete runnable example for the Scribe logging image, featuring: - docker-compose.yml orchestrating scribe daemon and Go test sender - Go test sender application (main.go, go.mod, Dockerfile) - Example README with setup and usage instructions - Updated scribe/README.md with Runnable Example section --- scribe/README.md | 15 ++ scribe/example/README.md | 89 ++++++++++++ scribe/example/docker-compose.yml | 26 ++++ scribe/example/logs/.gitkeep | 0 scribe/example/sender/Dockerfile | 10 ++ scribe/example/sender/go.mod | 3 + scribe/example/sender/main.go | 233 ++++++++++++++++++++++++++++++ 7 files changed, 376 insertions(+) create mode 100644 scribe/example/README.md create mode 100644 scribe/example/docker-compose.yml create mode 100644 scribe/example/logs/.gitkeep create mode 100644 scribe/example/sender/Dockerfile create mode 100644 scribe/example/sender/go.mod create mode 100644 scribe/example/sender/main.go diff --git a/scribe/README.md b/scribe/README.md index 854028c..c19526e 100644 --- a/scribe/README.md +++ b/scribe/README.md @@ -23,6 +23,21 @@ Scribe will listen on port **1463** (Thrift binary protocol) and write incoming --- +## Runnable Example + +Want to see logs actually written end-to-end? The [`example/`](example/) directory has a +`docker compose` setup that boots Scribe plus a small Go sender: + +```bash +cd scribe/example +docker compose up --build +``` + +The sender pushes a few messages and exits; written log files appear on your host under +`example/logs/`. See [`example/README.md`](example/README.md) for details. + +--- + ## Custom Configuration Mount your own `scribe.conf` over the default one: diff --git a/scribe/example/README.md b/scribe/example/README.md new file mode 100644 index 0000000..fd6fff6 --- /dev/null +++ b/scribe/example/README.md @@ -0,0 +1,89 @@ +# Scribe Runnable Example + +A one-command demo that proves the Scribe config actually writes logs. + +`docker compose up --build` starts the Scribe daemon (using the repo's default +[`config/scribe.conf`](../config/scribe.conf)) plus a tiny **Go sender** that connects over +Thrift, pushes a few messages, and exits. The resulting log files appear on your host under +[`./logs`](./logs). + +> Scribe is receive-only — it writes **nothing** until a client sends it messages. The Go +> sender is what makes this example produce log files. + +--- + +## Run + +```bash +cd scribe/example +docker compose up --build +``` + +You'll see the sender connect and report: + +``` +sender-1 | sent 5 messages to category "example", result=OK +``` + +The sender then exits; the Scribe daemon keeps running. Press `Ctrl+C`, then `docker compose down` to stop it. + +--- + +## Verify + +Log files are written on the host under `./logs//`: + +```bash +ls logs/example/ +cat logs/example/example-*_00000 +``` + +You should see the five `Hello from Go sender — message N/5` lines. + +--- + +## Customize + +Adjust the sender via environment variables in [`docker-compose.yml`](./docker-compose.yml): + +| Env | Default | Purpose | +|-----|---------|---------| +| `SCRIBE_CATEGORY` | `example` | Message category → log subdirectory name | +| `SCRIBE_COUNT` | `5` | Number of messages to send | +| `SCRIBE_HOST` | `scribe` | Daemon hostname (compose service name) | +| `SCRIBE_PORT` | `1463` | Thrift port | + +To change *where/how* Scribe stores logs (rotation, store type, per-category routing), edit +[`config/scribe.conf`](../config/scribe.conf) — see the +[Configuration Reference](../README.md#configuration-reference). + +The sender ([`sender/main.go`](./sender/main.go)) hand-encodes the Thrift framed binary +`Log()` call using only the Go standard library — no Apache Thrift dependency, no generated +bindings. For the higher-level client API, see the Python snippet in the +[main README](../README.md#sending-log-messages). + +--- + +## Troubleshoot + +- **`logs/` stays empty** — no messages were written yet. Check the `sender` output in the + compose logs; it exits non-zero and logs the cause if the send failed. +- **Permission denied writing logs (native Linux hosts)** — Scribe runs as a non-root user + inside the container, which may not be able to write to the bind-mounted host directory. + Fix by loosening the local dir before running: + + ```bash + chmod 777 logs + ``` + + On Docker Desktop (Windows/macOS) this is normally not needed. +- **Sender can't connect** — it retries for ~10s on startup. If Scribe takes longer to boot, + re-run `docker compose up` or increase the retry window in `sender/main.go`. + +--- + +## Stop + +```bash +docker compose down +``` diff --git a/scribe/example/docker-compose.yml b/scribe/example/docker-compose.yml new file mode 100644 index 0000000..c2089f2 --- /dev/null +++ b/scribe/example/docker-compose.yml @@ -0,0 +1,26 @@ +# Runnable Scribe example: the daemon plus a one-shot Go sender. +# Run from this directory: docker compose up --build +# Written logs appear on the host under ./logs//. +services: + scribe: + image: ghcr.io/tiennm99/scribe:2.2 + ports: + - "1463:1463" + volumes: + # Reuse the repo's default config (read-only). + - ../config/scribe.conf:/etc/scribe/scribe.conf:ro + # Host-visible log output — files land here after the sender runs. + - ./logs:/var/log/scribe + restart: unless-stopped + + sender: + build: ./sender + depends_on: + - scribe + # Reaches scribe over the compose network by service name (internal port), + # independent of the host 1463 mapping. The sender retries until scribe is up. + environment: + - SCRIBE_HOST=scribe + - SCRIBE_PORT=1463 + - SCRIBE_CATEGORY=example + - SCRIBE_COUNT=5 diff --git a/scribe/example/logs/.gitkeep b/scribe/example/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scribe/example/sender/Dockerfile b/scribe/example/sender/Dockerfile new file mode 100644 index 0000000..9a91a51 --- /dev/null +++ b/scribe/example/sender/Dockerfile @@ -0,0 +1,10 @@ +# Multi-stage build: compile a static binary, ship it on a minimal runtime. +FROM golang:1.22-alpine AS build +WORKDIR /src +COPY go.mod . +COPY main.go . +RUN CGO_ENABLED=0 go build -o /sender . + +FROM alpine:3.20 +COPY --from=build /sender /sender +ENTRYPOINT ["/sender"] diff --git a/scribe/example/sender/go.mod b/scribe/example/sender/go.mod new file mode 100644 index 0000000..05c64d5 --- /dev/null +++ b/scribe/example/sender/go.mod @@ -0,0 +1,3 @@ +module scribe-example-sender + +go 1.22 diff --git a/scribe/example/sender/main.go b/scribe/example/sender/main.go new file mode 100644 index 0000000..a2593ca --- /dev/null +++ b/scribe/example/sender/main.go @@ -0,0 +1,233 @@ +// Command scribe-example-sender pushes a few LogEntry messages to a Scribe +// daemon to demonstrate that the example writes real log files. +// +// It speaks Scribe's Thrift RPC by hand-encoding the framed binary `Log()` call +// using only the Go standard library — no Apache Thrift dependency and no +// generated bindings. The Scribe Thrift contract: +// +// enum ResultCode { OK = 0, TRY_LATER = 1 } +// struct LogEntry { 1: string category; 2: string message; } +// service scribe { ResultCode Log(1: list messages); } +package main + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "log" + "net" + "os" + "strconv" + "time" +) + +// Thrift binary protocol type IDs and message types. +const ( + tSTOP = 0 + tI32 = 8 + tSTRING = 11 + tSTRUCT = 12 + tLIST = 15 + + msgCALL = 1 +) + +func env(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +// --- TBinaryProtocol writers (all integers big-endian / network order) --- + +func writeI16(b *bytes.Buffer, v int16) { + var tmp [2]byte + binary.BigEndian.PutUint16(tmp[:], uint16(v)) + b.Write(tmp[:]) +} + +func writeI32(b *bytes.Buffer, v int32) { + var tmp [4]byte + binary.BigEndian.PutUint32(tmp[:], uint32(v)) + b.Write(tmp[:]) +} + +func writeString(b *bytes.Buffer, s string) { + writeI32(b, int32(len(s))) + b.WriteString(s) +} + +func writeFieldBegin(b *bytes.Buffer, typeID byte, id int16) { + b.WriteByte(typeID) + writeI16(b, id) +} + +// buildLogFrame builds a framed Thrift `Log()` CALL carrying one LogEntry per +// message, all under the same category. +// +// Uses the non-strict TBinaryProtocol message header (string name, byte type, +// i32 seqid) to match the README Python client (strictWrite=False); the Scribe +// C++ server reads with strict_read=false and accepts it. +func buildLogFrame(category string, messages []string) []byte { + var p bytes.Buffer + + // messageBegin (non-strict): name, type, seqid + writeString(&p, "Log") + p.WriteByte(msgCALL) + writeI32(&p, 0) + + // Log_args struct: field 1 = list messages + writeFieldBegin(&p, tLIST, 1) + p.WriteByte(tSTRUCT) // list element type + writeI32(&p, int32(len(messages))) + for _, m := range messages { + writeFieldBegin(&p, tSTRING, 1) // LogEntry.category + writeString(&p, category) + writeFieldBegin(&p, tSTRING, 2) // LogEntry.message + writeString(&p, m) + p.WriteByte(tSTOP) // end LogEntry + } + p.WriteByte(tSTOP) // end Log_args + + // Frame the payload with a 4-byte big-endian length prefix (TFramedTransport). + var frame bytes.Buffer + writeI32(&frame, int32(p.Len())) + frame.Write(p.Bytes()) + return frame.Bytes() +} + +// readResultCode reads one framed Thrift reply and extracts the ResultCode. +// Handles both strict and non-strict message headers, since the server's write +// strictness is independent of ours. +func readResultCode(r io.Reader) (int32, error) { + var lenBuf [4]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return -1, fmt.Errorf("read frame length: %w", err) + } + frame := make([]byte, binary.BigEndian.Uint32(lenBuf[:])) + if _, err := io.ReadFull(r, frame); err != nil { + return -1, fmt.Errorf("read frame body: %w", err) + } + br := bytes.NewReader(frame) + + // messageBegin — detect strict vs non-strict from the first i32. + var i32buf [4]byte + if _, err := io.ReadFull(br, i32buf[:]); err != nil { + return -1, fmt.Errorf("read message header: %w", err) + } + if first := int32(binary.BigEndian.Uint32(i32buf[:])); first < 0 { + // strict: first = version|type, then string name, then i32 seqid + var nameLen [4]byte + if _, err := io.ReadFull(br, nameLen[:]); err != nil { + return -1, err + } + if _, err := io.CopyN(io.Discard, br, int64(binary.BigEndian.Uint32(nameLen[:]))); err != nil { + return -1, err + } + if _, err := io.ReadFull(br, i32buf[:]); err != nil { // seqid + return -1, err + } + } else { + // non-strict: first = name length, then name, then i8 type, then i32 seqid + if _, err := io.CopyN(io.Discard, br, int64(first)); err != nil { + return -1, err + } + if _, err := br.ReadByte(); err != nil { // message type + return -1, err + } + if _, err := io.ReadFull(br, i32buf[:]); err != nil { // seqid + return -1, err + } + } + + // Log_result struct: first field is the return value (id 0, type I32). + typeID, err := br.ReadByte() + if err != nil { + return -1, err + } + if typeID == tSTOP { + return 0, nil // no return field present → treat as OK + } + var idBuf [2]byte + if _, err := io.ReadFull(br, idBuf[:]); err != nil { + return -1, err + } + if typeID != tI32 { + return -1, fmt.Errorf("unexpected reply field type %d", typeID) + } + if _, err := io.ReadFull(br, i32buf[:]); err != nil { + return -1, err + } + return int32(binary.BigEndian.Uint32(i32buf[:])), nil +} + +// dialWithRetry retries the initial connection so a cold `docker compose up` +// does not fail before the Scribe daemon is listening. +func dialWithRetry(addr string, attempts int, interval time.Duration) (net.Conn, error) { + var lastErr error + for i := 0; i < attempts; i++ { + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err == nil { + return conn, nil + } + lastErr = err + log.Printf("scribe not reachable at %s (attempt %d/%d): %v", addr, i+1, attempts, err) + time.Sleep(interval) + } + return nil, lastErr +} + +func main() { + host := env("SCRIBE_HOST", "scribe") + port := env("SCRIBE_PORT", "1463") + category := env("SCRIBE_CATEGORY", "example") + count := envInt("SCRIBE_COUNT", 5) + + addr := net.JoinHostPort(host, port) + conn, err := dialWithRetry(addr, 10, 1*time.Second) + if err != nil { + log.Fatalf("could not connect to scribe at %s: %v", addr, err) + } + defer conn.Close() + + // Bound the whole exchange so a wedged server that accepts the connection + // but never replies cannot hang this one-shot sender indefinitely. + conn.SetDeadline(time.Now().Add(10 * time.Second)) + + messages := make([]string, 0, count) + for i := 1; i <= count; i++ { + messages = append(messages, fmt.Sprintf("Hello from Go sender — message %d/%d", i, count)) + } + + if _, err := conn.Write(buildLogFrame(category, messages)); err != nil { + log.Fatalf("send failed: %v", err) + } + + rc, err := readResultCode(bufio.NewReader(conn)) + if err != nil { + log.Fatalf("read reply failed: %v", err) + } + + switch rc { + case 0: + log.Printf("sent %d messages to category %q, result=OK", count, category) + case 1: + log.Printf("sent %d messages to category %q, result=TRY_LATER (scribe busy)", count, category) + os.Exit(2) + default: + log.Printf("sent %d messages to category %q, unexpected result code=%d", count, category, rc) + os.Exit(3) + } +}