mirror of
https://github.com/tiennm99/docker-images.git
synced 2026-10-14 02:16:03 +00:00
feat(scribe): add runnable docker-compose example with Go test sender
Introduces a complete runnable example for the Scribe logging image, featuring: - docker-compose.yml orchestrating scribe daemon and Go test sender - Go test sender application (main.go, go.mod, Dockerfile) - Example README with setup and usage instructions - Updated scribe/README.md with Runnable Example section
This commit is contained in:
1 parent
21220546a8
commit
bd16c305eb
7 files changed
+376
No files matched your search
@@ -23,6 +23,21 @@ Scribe will listen on port **1463** (Thrift binary protocol) and write incoming
|
||||
|
||||
---
|
||||
|
||||
## Runnable Example
|
||||
|
||||
Want to see logs actually written end-to-end? The [`example/`](example/) directory has a
|
||||
`docker compose` setup that boots Scribe plus a small Go sender:
|
||||
|
||||
```bash
|
||||
cd scribe/example
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
The sender pushes a few messages and exits; written log files appear on your host under
|
||||
`example/logs/`. See [`example/README.md`](example/README.md) for details.
|
||||
|
||||
---
|
||||
|
||||
## Custom Configuration
|
||||
|
||||
Mount your own `scribe.conf` over the default one:
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Scribe Runnable Example
|
||||
|
||||
A one-command demo that proves the Scribe config actually writes logs.
|
||||
|
||||
`docker compose up --build` starts the Scribe daemon (using the repo's default
|
||||
[`config/scribe.conf`](../config/scribe.conf)) plus a tiny **Go sender** that connects over
|
||||
Thrift, pushes a few messages, and exits. The resulting log files appear on your host under
|
||||
[`./logs`](./logs).
|
||||
|
||||
> Scribe is receive-only — it writes **nothing** until a client sends it messages. The Go
|
||||
> sender is what makes this example produce log files.
|
||||
|
||||
---
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
cd scribe/example
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
You'll see the sender connect and report:
|
||||
|
||||
```
|
||||
sender-1 | sent 5 messages to category "example", result=OK
|
||||
```
|
||||
|
||||
The sender then exits; the Scribe daemon keeps running. Press `Ctrl+C`, then `docker compose down` to stop it.
|
||||
|
||||
---
|
||||
|
||||
## Verify
|
||||
|
||||
Log files are written on the host under `./logs/<category>/`:
|
||||
|
||||
```bash
|
||||
ls logs/example/
|
||||
cat logs/example/example-*_00000
|
||||
```
|
||||
|
||||
You should see the five `Hello from Go sender — message N/5` lines.
|
||||
|
||||
---
|
||||
|
||||
## Customize
|
||||
|
||||
Adjust the sender via environment variables in [`docker-compose.yml`](./docker-compose.yml):
|
||||
|
||||
| Env | Default | Purpose |
|
||||
|-----|---------|---------|
|
||||
| `SCRIBE_CATEGORY` | `example` | Message category → log subdirectory name |
|
||||
| `SCRIBE_COUNT` | `5` | Number of messages to send |
|
||||
| `SCRIBE_HOST` | `scribe` | Daemon hostname (compose service name) |
|
||||
| `SCRIBE_PORT` | `1463` | Thrift port |
|
||||
|
||||
To change *where/how* Scribe stores logs (rotation, store type, per-category routing), edit
|
||||
[`config/scribe.conf`](../config/scribe.conf) — see the
|
||||
[Configuration Reference](../README.md#configuration-reference).
|
||||
|
||||
The sender ([`sender/main.go`](./sender/main.go)) hand-encodes the Thrift framed binary
|
||||
`Log()` call using only the Go standard library — no Apache Thrift dependency, no generated
|
||||
bindings. For the higher-level client API, see the Python snippet in the
|
||||
[main README](../README.md#sending-log-messages).
|
||||
|
||||
---
|
||||
|
||||
## Troubleshoot
|
||||
|
||||
- **`logs/` stays empty** — no messages were written yet. Check the `sender` output in the
|
||||
compose logs; it exits non-zero and logs the cause if the send failed.
|
||||
- **Permission denied writing logs (native Linux hosts)** — Scribe runs as a non-root user
|
||||
inside the container, which may not be able to write to the bind-mounted host directory.
|
||||
Fix by loosening the local dir before running:
|
||||
|
||||
```bash
|
||||
chmod 777 logs
|
||||
```
|
||||
|
||||
On Docker Desktop (Windows/macOS) this is normally not needed.
|
||||
- **Sender can't connect** — it retries for ~10s on startup. If Scribe takes longer to boot,
|
||||
re-run `docker compose up` or increase the retry window in `sender/main.go`.
|
||||
|
||||
---
|
||||
|
||||
## Stop
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
```
|
||||
@@ -0,0 +1,26 @@
|
||||
# Runnable Scribe example: the daemon plus a one-shot Go sender.
|
||||
# Run from this directory: docker compose up --build
|
||||
# Written logs appear on the host under ./logs/<category>/.
|
||||
services:
|
||||
scribe:
|
||||
image: ghcr.io/tiennm99/scribe:2.2
|
||||
ports:
|
||||
- "1463:1463"
|
||||
volumes:
|
||||
# Reuse the repo's default config (read-only).
|
||||
- ../config/scribe.conf:/etc/scribe/scribe.conf:ro
|
||||
# Host-visible log output — files land here after the sender runs.
|
||||
- ./logs:/var/log/scribe
|
||||
restart: unless-stopped
|
||||
|
||||
sender:
|
||||
build: ./sender
|
||||
depends_on:
|
||||
- scribe
|
||||
# Reaches scribe over the compose network by service name (internal port),
|
||||
# independent of the host 1463 mapping. The sender retries until scribe is up.
|
||||
environment:
|
||||
- SCRIBE_HOST=scribe
|
||||
- SCRIBE_PORT=1463
|
||||
- SCRIBE_CATEGORY=example
|
||||
- SCRIBE_COUNT=5
|
||||
Whitespace-only changes.
@@ -0,0 +1,10 @@
|
||||
# Multi-stage build: compile a static binary, ship it on a minimal runtime.
|
||||
FROM golang:1.22-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod .
|
||||
COPY main.go .
|
||||
RUN CGO_ENABLED=0 go build -o /sender .
|
||||
|
||||
FROM alpine:3.20
|
||||
COPY --from=build /sender /sender
|
||||
ENTRYPOINT ["/sender"]
|
||||
@@ -0,0 +1,3 @@
|
||||
module scribe-example-sender
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,233 @@
|
||||
// Command scribe-example-sender pushes a few LogEntry messages to a Scribe
|
||||
// daemon to demonstrate that the example writes real log files.
|
||||
//
|
||||
// It speaks Scribe's Thrift RPC by hand-encoding the framed binary `Log()` call
|
||||
// using only the Go standard library — no Apache Thrift dependency and no
|
||||
// generated bindings. The Scribe Thrift contract:
|
||||
//
|
||||
// enum ResultCode { OK = 0, TRY_LATER = 1 }
|
||||
// struct LogEntry { 1: string category; 2: string message; }
|
||||
// service scribe { ResultCode Log(1: list<LogEntry> messages); }
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Thrift binary protocol type IDs and message types.
|
||||
const (
|
||||
tSTOP = 0
|
||||
tI32 = 8
|
||||
tSTRING = 11
|
||||
tSTRUCT = 12
|
||||
tLIST = 15
|
||||
|
||||
msgCALL = 1
|
||||
)
|
||||
|
||||
func env(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// --- TBinaryProtocol writers (all integers big-endian / network order) ---
|
||||
|
||||
func writeI16(b *bytes.Buffer, v int16) {
|
||||
var tmp [2]byte
|
||||
binary.BigEndian.PutUint16(tmp[:], uint16(v))
|
||||
b.Write(tmp[:])
|
||||
}
|
||||
|
||||
func writeI32(b *bytes.Buffer, v int32) {
|
||||
var tmp [4]byte
|
||||
binary.BigEndian.PutUint32(tmp[:], uint32(v))
|
||||
b.Write(tmp[:])
|
||||
}
|
||||
|
||||
func writeString(b *bytes.Buffer, s string) {
|
||||
writeI32(b, int32(len(s)))
|
||||
b.WriteString(s)
|
||||
}
|
||||
|
||||
func writeFieldBegin(b *bytes.Buffer, typeID byte, id int16) {
|
||||
b.WriteByte(typeID)
|
||||
writeI16(b, id)
|
||||
}
|
||||
|
||||
// buildLogFrame builds a framed Thrift `Log()` CALL carrying one LogEntry per
|
||||
// message, all under the same category.
|
||||
//
|
||||
// Uses the non-strict TBinaryProtocol message header (string name, byte type,
|
||||
// i32 seqid) to match the README Python client (strictWrite=False); the Scribe
|
||||
// C++ server reads with strict_read=false and accepts it.
|
||||
func buildLogFrame(category string, messages []string) []byte {
|
||||
var p bytes.Buffer
|
||||
|
||||
// messageBegin (non-strict): name, type, seqid
|
||||
writeString(&p, "Log")
|
||||
p.WriteByte(msgCALL)
|
||||
writeI32(&p, 0)
|
||||
|
||||
// Log_args struct: field 1 = list<LogEntry> messages
|
||||
writeFieldBegin(&p, tLIST, 1)
|
||||
p.WriteByte(tSTRUCT) // list element type
|
||||
writeI32(&p, int32(len(messages)))
|
||||
for _, m := range messages {
|
||||
writeFieldBegin(&p, tSTRING, 1) // LogEntry.category
|
||||
writeString(&p, category)
|
||||
writeFieldBegin(&p, tSTRING, 2) // LogEntry.message
|
||||
writeString(&p, m)
|
||||
p.WriteByte(tSTOP) // end LogEntry
|
||||
}
|
||||
p.WriteByte(tSTOP) // end Log_args
|
||||
|
||||
// Frame the payload with a 4-byte big-endian length prefix (TFramedTransport).
|
||||
var frame bytes.Buffer
|
||||
writeI32(&frame, int32(p.Len()))
|
||||
frame.Write(p.Bytes())
|
||||
return frame.Bytes()
|
||||
}
|
||||
|
||||
// readResultCode reads one framed Thrift reply and extracts the ResultCode.
|
||||
// Handles both strict and non-strict message headers, since the server's write
|
||||
// strictness is independent of ours.
|
||||
func readResultCode(r io.Reader) (int32, error) {
|
||||
var lenBuf [4]byte
|
||||
if _, err := io.ReadFull(r, lenBuf[:]); err != nil {
|
||||
return -1, fmt.Errorf("read frame length: %w", err)
|
||||
}
|
||||
frame := make([]byte, binary.BigEndian.Uint32(lenBuf[:]))
|
||||
if _, err := io.ReadFull(r, frame); err != nil {
|
||||
return -1, fmt.Errorf("read frame body: %w", err)
|
||||
}
|
||||
br := bytes.NewReader(frame)
|
||||
|
||||
// messageBegin — detect strict vs non-strict from the first i32.
|
||||
var i32buf [4]byte
|
||||
if _, err := io.ReadFull(br, i32buf[:]); err != nil {
|
||||
return -1, fmt.Errorf("read message header: %w", err)
|
||||
}
|
||||
if first := int32(binary.BigEndian.Uint32(i32buf[:])); first < 0 {
|
||||
// strict: first = version|type, then string name, then i32 seqid
|
||||
var nameLen [4]byte
|
||||
if _, err := io.ReadFull(br, nameLen[:]); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if _, err := io.CopyN(io.Discard, br, int64(binary.BigEndian.Uint32(nameLen[:]))); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if _, err := io.ReadFull(br, i32buf[:]); err != nil { // seqid
|
||||
return -1, err
|
||||
}
|
||||
} else {
|
||||
// non-strict: first = name length, then name, then i8 type, then i32 seqid
|
||||
if _, err := io.CopyN(io.Discard, br, int64(first)); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if _, err := br.ReadByte(); err != nil { // message type
|
||||
return -1, err
|
||||
}
|
||||
if _, err := io.ReadFull(br, i32buf[:]); err != nil { // seqid
|
||||
return -1, err
|
||||
}
|
||||
}
|
||||
|
||||
// Log_result struct: first field is the return value (id 0, type I32).
|
||||
typeID, err := br.ReadByte()
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if typeID == tSTOP {
|
||||
return 0, nil // no return field present → treat as OK
|
||||
}
|
||||
var idBuf [2]byte
|
||||
if _, err := io.ReadFull(br, idBuf[:]); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if typeID != tI32 {
|
||||
return -1, fmt.Errorf("unexpected reply field type %d", typeID)
|
||||
}
|
||||
if _, err := io.ReadFull(br, i32buf[:]); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return int32(binary.BigEndian.Uint32(i32buf[:])), nil
|
||||
}
|
||||
|
||||
// dialWithRetry retries the initial connection so a cold `docker compose up`
|
||||
// does not fail before the Scribe daemon is listening.
|
||||
func dialWithRetry(addr string, attempts int, interval time.Duration) (net.Conn, error) {
|
||||
var lastErr error
|
||||
for i := 0; i < attempts; i++ {
|
||||
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
log.Printf("scribe not reachable at %s (attempt %d/%d): %v", addr, i+1, attempts, err)
|
||||
time.Sleep(interval)
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func main() {
|
||||
host := env("SCRIBE_HOST", "scribe")
|
||||
port := env("SCRIBE_PORT", "1463")
|
||||
category := env("SCRIBE_CATEGORY", "example")
|
||||
count := envInt("SCRIBE_COUNT", 5)
|
||||
|
||||
addr := net.JoinHostPort(host, port)
|
||||
conn, err := dialWithRetry(addr, 10, 1*time.Second)
|
||||
if err != nil {
|
||||
log.Fatalf("could not connect to scribe at %s: %v", addr, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// Bound the whole exchange so a wedged server that accepts the connection
|
||||
// but never replies cannot hang this one-shot sender indefinitely.
|
||||
conn.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
|
||||
messages := make([]string, 0, count)
|
||||
for i := 1; i <= count; i++ {
|
||||
messages = append(messages, fmt.Sprintf("Hello from Go sender — message %d/%d", i, count))
|
||||
}
|
||||
|
||||
if _, err := conn.Write(buildLogFrame(category, messages)); err != nil {
|
||||
log.Fatalf("send failed: %v", err)
|
||||
}
|
||||
|
||||
rc, err := readResultCode(bufio.NewReader(conn))
|
||||
if err != nil {
|
||||
log.Fatalf("read reply failed: %v", err)
|
||||
}
|
||||
|
||||
switch rc {
|
||||
case 0:
|
||||
log.Printf("sent %d messages to category %q, result=OK", count, category)
|
||||
case 1:
|
||||
log.Printf("sent %d messages to category %q, result=TRY_LATER (scribe busy)", count, category)
|
||||
os.Exit(2)
|
||||
default:
|
||||
log.Printf("sent %d messages to category %q, unexpected result code=%d", count, category, rc)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user