diff --git a/README.md b/README.md index d9c6dd9..3b743f1 100644 --- a/README.md +++ b/README.md @@ -9,64 +9,28 @@ Latest releases: [![Maven Central](https://img.shields.io/maven-central/v/pl.tkowalcz.tjahzi/core.svg?label=Core&style=for-the-badge)](https://search.maven.org/search?q=g:pl.tkowalcz.tjahzi) [![Maven Central](https://img.shields.io/maven-central/v/pl.tkowalcz.tjahzi/log4j2-appender.svg?label=Log4j2%20Appender&style=for-the-badge)](https://search.maven.org/search?q=g:pl.tkowalcz.tjahzi) +[![Maven Central](https://img.shields.io/maven-central/v/pl.tkowalcz.tjahzi/logback-appender.svg?label=Logback%20Appender&style=for-the-badge)](https://search.maven.org/search?q=g:pl.tkowalcz.tjahzi) If you find the project useful (or not useful for some reason) please let me know to encourage further development. ⭐ Stars are also welcome ⭐ ;). ## Features Tjahzi allows pushing log data to Loki. It uses protobuf format to push log lines with timestamps and labels to Loki. This -is implemented in the `core` component. On top of that we built the `log4j2 appender`. It's features include: +is implemented in the `core` component. On top of that we built appenders for `log4j2` and `Logback`. These feature: 1. Logging does not allocate objects nor take any locks. 1. Sending data to Loki in batches allocates as little as possible. -1. We also provide a no-dependency version of log4j2 appender. -1. Includes in-house implementation of protobuf wire format for Loki to reduce dependencies and improve performance +1. We also provide a no-dependency versions of these appenders. +1. Includes in-house implementation of protobuf wire format for Loki to reduce dependencies and improve performance. -It is currently used (that I know of) to ship logs from tens of servers at about 10GB of logs per day. +Log4j2 appender is currently used to ship logs from tens of servers at about 10GB of logs per day. ## Getting started -1. Grab the no-dependency version of the appender: - - ```xml - - pl.tkowalcz.tjahzi - log4j2-appender-nodep - 0.9.4 - - ``` - - If you already use a compatible version of Netty in your project then to reduce the size of your dependencies include - regular appender distribution: - - ```xml - - pl.tkowalcz.tjahzi - log4j2-appender - 0.9.4 - - ``` - -1. Add `packages="pl.tkowalcz.tjahzi.log4j2"` attribute to your log4j2 configuration xml. -1. Include minimal appender configuration +For `log4j2` appender quick start guide and detailed discussions see this [README.md](log4j2-appender/README.md). +For `Logback` appender quick start guide and detailed discussions see this [README.md](logback-appender/README.md). -```xml - - ${sys:loki.host} - ${sys:loki.port} - - - %X{tid} [%t] %d{MM-dd HH:mm:ss.SSS} %5p %c{1} - %m%n%exception{full} - - - -``` - -Detailed discussion of how to configure the appender with examples and description of all possible tags check -[Log4j2 appender docs](log4j2-appender/README.md). - -## Design principles +## Core design principles Logging should be lightweight and not interfere with main business tasks of threads that happen to log a message. Asking the logging subsystem to log a message should be as CPU efficient as possible. diff --git a/log4j2-appender/README.md b/log4j2-appender/README.md index 0298159..9c0a314 100644 --- a/log4j2-appender/README.md +++ b/log4j2-appender/README.md @@ -3,14 +3,61 @@ On top of a `Core` component with rather simplistic API we intend to build several layers that make it truly useful. Log4j2 appender seemed like a good first. -## Example Log4j2 configuration +## Quick start guide + +1. Grab the no-dependency version of the appender: + + ```xml + + pl.tkowalcz.tjahzi + log4j2-appender-nodep + 0.9.4 + + ``` + + If you already use a compatible version of Netty in your project then to reduce the size of your dependencies include + regular appender distribution: + + ```xml + + pl.tkowalcz.tjahzi + log4j2-appender + 0.9.4 + + ``` + +1. Add `packages="pl.tkowalcz.tjahzi.log4j2"` attribute to your existing log4j2 configuration file. +1. Include minimal appender configuration: + +```xml + + ${sys:loki.host} + ${sys:loki.port} + + + %X{tid} [%t] %d{MM-dd HH:mm:ss.SSS} %5p %c{1} - %m%n%exception{full} + + + +``` + +1. Reference the appender from inside one of your logger definitions: + +```xml + + + +``` + +## Advanced configuration This example sets up a root logger with a Loki appender. Note that `pl.tkowalcz.tjahzi.log4j2` is added to `packages` attribute of configuration so that the appender can be found. ```xml - + diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java index d9d8f98..527fcc6 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java @@ -52,7 +52,7 @@ public class LabelFactory { if (!duplicatedLabels.isEmpty()) { LOGGER.error( "There are duplicated labels which is not allowed by Loki. " + - "These labels will be deduplicated undeterministically: {}", + "These labels will be deduplicated non-deterministically: {}", duplicatedLabels ); } @@ -67,7 +67,8 @@ public class LabelFactory { return Stream.of(label); } - LOGGER.error("Label '{}' contains invalid characters - ignoring it. {}", + LOGGER.error( + "Ignoring label '{}' - contains invalid characters. {}", label.getName(), GitHubDocs.LABEL_NAMING.getLogMessage() ); @@ -85,7 +86,8 @@ public class LabelFactory { String logLevelLabel ) { if (!Label.hasValidName(logLevelLabel)) { - LOGGER.error("Log level label '{}' contains invalid characters - ignoring it. {}", + LOGGER.error( + "Ignoring log level label '{}' - contains invalid characters. {}", logLevelLabel, GitHubDocs.LABEL_NAMING.getLogMessage() ); diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java index 3355965..3353fd8 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java @@ -101,7 +101,7 @@ public class LokiAppenderBuilder> extends Abstr ); int bufferSizeBytes = getBufferSizeMegabytes() * BYTES_IN_MEGABYTE; - if (TjahziInitializer.isCorrectSize(bufferSizeBytes)) { + if (!TjahziInitializer.isCorrectSize(bufferSizeBytes)) { LOGGER.warn("Invalid log buffer size {} - using nearest power of two greater than provided value, no less than 1MB. {}", bufferSizeBytes, GitHubDocs.LOG_BUFFER_SIZING.getLogMessage() diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AllocationTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AllocationTest.java index 508a30f..8ad1b42 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AllocationTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AllocationTest.java @@ -57,7 +57,7 @@ public class AllocationTest { URI uri = getClass() .getClassLoader() - .getResource("basic-appender-test-log4j2-configuration.xml") + .getResource("basic-appender-test-configuration.xml") .toURI(); ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false)) diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LogLevelLabelConfigurationTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LogLevelLabelConfigurationTest.java index 36dddb2..219ce86 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LogLevelLabelConfigurationTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LogLevelLabelConfigurationTest.java @@ -37,7 +37,7 @@ class LogLevelLabelConfigurationTest { .withExposedPorts(3100); @Test - void shouldWorkWIthNoLogLevelConfigured() throws URISyntaxException { + void shouldWorkWIthNoLogLevelConfigured() throws Exception { // Given System.setProperty("loki.host", loki.getHost()); System.setProperty("loki.port", loki.getFirstMappedPort().toString()); diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTest.java similarity index 97% rename from log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java rename to log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTest.java index 3393f5f..4e1eeda 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTest.java @@ -17,7 +17,6 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.net.URI; -import java.net.URISyntaxException; import java.util.List; import java.util.concurrent.TimeUnit; @@ -26,7 +25,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItems; @Testcontainers -class LokiAppenderLargeBatchesTestTest { +class LokiAppenderLargeBatchesTest { @Container public GenericContainer loki = new GenericContainer("grafana/loki:latest") @@ -41,7 +40,7 @@ class LokiAppenderLargeBatchesTestTest { .withExposedPorts(3100); @Test - void shouldSendData() throws URISyntaxException { + void shouldSendData() throws Exception { // Given System.setProperty("loki.host", loki.getHost()); System.setProperty("loki.port", loki.getFirstMappedPort().toString()); @@ -65,7 +64,7 @@ class LokiAppenderLargeBatchesTestTest { "Souffle cake muffin liquorice tart souffle pie sesame snaps."; long expectedTimestamp = System.currentTimeMillis(); - Logger logger = LogManager.getLogger(LokiAppenderLargeBatchesTestTest.class); + Logger logger = LogManager.getLogger(LokiAppenderLargeBatchesTest.class); // When for (int i = 0; i < 1000; i++) { diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderMonitoringTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderMonitoringTest.java index abd007d..fd255cb 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderMonitoringTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderMonitoringTest.java @@ -21,9 +21,12 @@ class LokiAppenderMonitoringTest { @Test void shouldInjectMonitoringAndUseIt() throws URISyntaxException { // Given + System.setProperty("loki.host", "somewhere"); + System.setProperty("loki.port", "42"); + URI uri = getClass() .getClassLoader() - .getResource("basic-appender-test-log4j2-configuration.xml") + .getResource("basic-appender-test-configuration.xml") .toURI(); LoggerContext context = (LoggerContext) LogManager.getContext(false); diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderShutdownTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderShutdownTest.java index abea607..7dd0744 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderShutdownTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderShutdownTest.java @@ -40,7 +40,7 @@ class LokiAppenderShutdownTest { URI uri = getClass() .getClassLoader() - .getResource("basic-appender-test-log4j2-configuration.xml") + .getResource("basic-appender-test-configuration.xml") .toURI(); ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false)) diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderTest.java index 023dac9..baf8834 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderTest.java @@ -17,7 +17,6 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.net.URI; -import java.net.URISyntaxException; import java.util.List; import java.util.concurrent.TimeUnit; @@ -41,14 +40,14 @@ class LokiAppenderTest { .withExposedPorts(3100); @Test - void shouldSendData() throws URISyntaxException { + void shouldSendData() throws Exception { // Given System.setProperty("loki.host", loki.getHost()); System.setProperty("loki.port", loki.getFirstMappedPort().toString()); URI uri = getClass() .getClassLoader() - .getResource("basic-appender-test-log4j2-configuration.xml") + .getResource("basic-appender-test-configuration.xml") .toURI(); ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false)) diff --git a/log4j2-appender/src/test/resources/appender-test-large-batches.xml b/log4j2-appender/src/test/resources/appender-test-large-batches.xml index 79aa5a9..b418b5a 100644 --- a/log4j2-appender/src/test/resources/appender-test-large-batches.xml +++ b/log4j2-appender/src/test/resources/appender-test-large-batches.xml @@ -1,5 +1,5 @@ - + @@ -7,7 +7,9 @@ - + + 1048576 + ${sys:loki.host} ${sys:loki.port} diff --git a/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-set-to-64.xml b/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-set-to-64.xml index 40a82cf..c19af37 100644 --- a/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-set-to-64.xml +++ b/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-set-to-64.xml @@ -1,5 +1,5 @@ - + diff --git a/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-unset.xml b/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-unset.xml index 663c75b..4e77860 100644 --- a/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-unset.xml +++ b/log4j2-appender/src/test/resources/appender-test-with-buffer-size-megabytes-unset.xml @@ -1,5 +1,5 @@ - + diff --git a/log4j2-appender/src/test/resources/appender-test-with-log-label-set.xml b/log4j2-appender/src/test/resources/appender-test-with-log-label-set.xml index eb1ffa9..1939d05 100644 --- a/log4j2-appender/src/test/resources/appender-test-with-log-label-set.xml +++ b/log4j2-appender/src/test/resources/appender-test-with-log-label-set.xml @@ -1,5 +1,5 @@ - + diff --git a/log4j2-appender/src/test/resources/appender-test-with-log-label-unset.xml b/log4j2-appender/src/test/resources/appender-test-with-log-label-unset.xml index 663c75b..4e77860 100644 --- a/log4j2-appender/src/test/resources/appender-test-with-log-label-unset.xml +++ b/log4j2-appender/src/test/resources/appender-test-with-log-label-unset.xml @@ -1,5 +1,5 @@ - + diff --git a/log4j2-appender/src/test/resources/basic-appender-test-log4j2-configuration.xml b/log4j2-appender/src/test/resources/basic-appender-test-configuration.xml similarity index 87% rename from log4j2-appender/src/test/resources/basic-appender-test-log4j2-configuration.xml rename to log4j2-appender/src/test/resources/basic-appender-test-configuration.xml index b481161..ce125fc 100644 --- a/log4j2-appender/src/test/resources/basic-appender-test-log4j2-configuration.xml +++ b/log4j2-appender/src/test/resources/basic-appender-test-configuration.xml @@ -1,5 +1,5 @@ - + diff --git a/logback-appender-nodep/.gitignore b/logback-appender-nodep/.gitignore new file mode 100644 index 0000000..53868d1 --- /dev/null +++ b/logback-appender-nodep/.gitignore @@ -0,0 +1,2 @@ +src/** +dependency-reduced-pom.xml diff --git a/logback-appender-nodep/README.md b/logback-appender-nodep/README.md new file mode 100644 index 0000000..b7aee8d --- /dev/null +++ b/logback-appender-nodep/README.md @@ -0,0 +1,3 @@ +# Logback Appender - nodep + +This repository contains only `pom.xml` to generate distribution with no external dependencies (other than logback-classic). diff --git a/logback-appender-nodep/pom.xml b/logback-appender-nodep/pom.xml new file mode 100644 index 0000000..b7fe446 --- /dev/null +++ b/logback-appender-nodep/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + + pl.tkowalcz.tjahzi + tjahzi-parent + 0.9.5-SNAPSHOT + + + logback-appender-nodep + logback-appender-nodep + jar + + + + logback-appender + + + + + pl.tkowalcz.tjahzi + logback-appender + 0.1.0-SNAPSHOT + + + + + + + maven-dependency-plugin + + + unpack-sources + generate-sources + + unpack-dependencies + + + + + sources + jar + ${project.build.sourceDirectory} + ${project.build.unpack_includes} + **\/*.java + true + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + ch.qos.logback.logback-classic + + + + + org.agrona + pl.tkowalcz.tjahzi.org.agrona + + + io.netty + pl.tkowalcz.tjahzi.io.netty + + + + true + true + + + + + + + diff --git a/logback-appender/README.md b/logback-appender/README.md new file mode 100644 index 0000000..09a6d3c --- /dev/null +++ b/logback-appender/README.md @@ -0,0 +1,112 @@ +# Logback Appender + +On top of the `Core` component that sends logs to Loki this project gives you Logback appender. + +## Quick start guide + +1. Grab the no-dependency version of the appender: + + ```xml + + pl.tkowalcz.tjahzi + logback-appender-nodep + 0.1.0 + + ``` + + If you already use a compatible version of Netty in your project then to reduce the size of your dependencies include + regular appender distribution: + + ```xml + + pl.tkowalcz.tjahzi + logback-appender + 0.1.0 + + ``` + +1. Include minimal appender configuration in your existing logback configuration: + +```xml + + ${loki.host} + ${loki.port} + + + %-4relative [%thread] %-5level %logger{35} - %msg%n + + + + +``` + +1. Reference the appender from inside one of your logger definitions: + +```xml + + + +``` + +## Advanced configuration + +This example sets up a root logger with Loki appender. The appender definition must point to class `pl.tkowalcz.tjahzi.logback.LokiAppender`. + +```xml + + + ${loki.host} + ${loki.port} + + + %-4relative [%thread] %-5level %logger{35} - %msg%n + + +
+ X-Org-Id + Codewise +
+ + + + + log_level + + + + + + + + +``` + +### Lookups / variable substitution + +Contents of the properties are automatically interpolated by Logback (see [here](http://logback.qos.ch/manual/configuration.html#variableSubstitution)). +All environment, system etc. variable references will be replaced by their values during initialization of the appender. + +## Details + +Let's go through the example config above and analyze configuration options (**Note: Tags are case-insensitive**). + +| Tag | Required | Default value | Description | +|-----|----------|---------------|-------------| +| Host | Y | | Network host address of Loki instance. Either IP address or host name. It will be passed to Netty and end up being resolved by call to `InetSocketAddress.createUnresolved`. | +| Port | Y | | Self-explanatory :) | +| Header | N | (empty list) | This tag can be used multiple times to specify additional headers that are passed to Loki instance. One example is to pass a `X-Scope-OrgID` header when [running Loki in multi-tenant mode](https://grafana.com/docs/loki/latest/operations/authentication/). | +| Label | N | (empty list) | Specify additional labels attached to each log line sent via this appender instance. See also note about [label naming](https://github.com/tkowalcz/tjahzi/wiki/Label-naming). | +| LogLevelLabel | N | null | If defined then log level label of configured name will be added to each line sent to Loki. It will contain Log4j log level e.g. `INFO`, `WARN` etc. See also note about [label naming](https://github.com/tkowalcz/tjahzi/wiki/Label-naming). | +| BufferSizeMegabytes | N | 32 MB | Size of the `log buffer`. Must be power of two between 1MB and 1GB. See [log buffer sizing](https://github.com/tkowalcz/tjahzi/wiki/Log-buffer-sizing) for more explanations. | +| MaxRetries | N | 3 | Maximum number of retries to perform when delivering log message to Loki. Log buffer data is delivered in order, one batch after the other, so too much retries will block delivery of subsequent log batches (on the other hand if we need to retry many times then next batches will probably fail too). | +| ConnectTimeoutMillis | N | 5s | This configures socket connect timeout when connecting to Loki. After unsuccessful connection attempt it will continue to retry indefinitely employing exponential backoff (initial backoff = 250ms, maximum backoff = 30s, multiplier = 3). | +| ReadTimeoutMillis | N | 60s | Sets socket read timeout on Loki connection. | +| UseOffHeapBuffer | N | true | Whether Tjahzi should allocate native buffer for `Log buffer` component. We can go into a rabbit hole of divagations what are the implications of this. Most important in our view is that having 10s or 100s of MB of space taken out of heap is not very friendly to garbage collector which might have to occasionally copy it around. | +| BatchSize | N | 100 KB | Like in [promtail configuration](https://grafana.com/docs/loki/latest/clients/promtail/configuration/) `maximum batch size (in bytes) of logs to accumulate before sending the batch to Loki`.| +| BatchWait | N | 5s | Like in [promtail configuration](https://grafana.com/docs/loki/latest/clients/promtail/configuration/) `maximum amount of time to wait before sending a batch, even if that batch isn't full`.| diff --git a/logback-appender/pom.xml b/logback-appender/pom.xml new file mode 100644 index 0000000..0931358 --- /dev/null +++ b/logback-appender/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + + pl.tkowalcz.tjahzi + tjahzi-parent + 0.9.5-SNAPSHOT + + + logback-appender + logback-appender + jar + 0.1.0-SNAPSHOT + + + + pl.tkowalcz.tjahzi + core + 0.9.5-SNAPSHOT + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + io.dropwizard.metrics + metrics-core + 4.1.17 + provided + + + + com.google.code.java-allocation-instrumenter + java-allocation-instrumenter + 3.3.0 + test + + + io.rest-assured + rest-assured + 4.3.1 + test + + + org.awaitility + awaitility + 4.0.3 + test + + + org.assertj + assertj-core + 3.15.0 + test + + + + + org.junit.jupiter + junit-jupiter-api + 5.6.2 + test + + + org.junit.jupiter + junit-jupiter-params + 5.6.2 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.6.2 + test + + + org.testcontainers + testcontainers + 1.14.3 + test + + + org.testcontainers + junit-jupiter + 1.14.3 + test + + + diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java new file mode 100644 index 0000000..bb18900 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java @@ -0,0 +1,23 @@ +package pl.tkowalcz.tjahzi.logback; + +public class Header { + + private String name; + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java new file mode 100644 index 0000000..25202c9 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java @@ -0,0 +1,50 @@ +package pl.tkowalcz.tjahzi.logback; + +import java.util.Map; +import java.util.regex.Pattern; + +public class Label { + + private static final Pattern LABEL_NAME_PATTER = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); + + private String name; + private String value; + + // For Logback this is just a convenience method for testing + public static Label createLabel(String name, String value) { + Label result = new Label(); + result.setName(name); + result.setValue(value); + + return result; + } + + public boolean hasValidName() { + return hasValidName(getName()); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + // @VisibleForTests + public Map.Entry asEntry() { + return Map.entry(getName(), getValue()); + } + + public static boolean hasValidName(String label) { + return LABEL_NAME_PATTER.matcher(label).matches(); + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java new file mode 100644 index 0000000..087c944 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java @@ -0,0 +1,119 @@ +package pl.tkowalcz.tjahzi.logback; + +import ch.qos.logback.core.spi.ContextAware; +import pl.tkowalcz.tjahzi.github.GitHubDocs; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.counting; + +public class LabelFactory { + + private final ContextAware internalLogger; + + private final String logLevelLabel; + private final Label[] labels; + + public LabelFactory( + ContextAware internalLogger, + String logLevelLabel, + Label... labels + ) { + this.internalLogger = internalLogger; + this.logLevelLabel = logLevelLabel; + this.labels = labels; + } + + public HashMap convertLabelsDroppingInvalid() { + detectAndLogDuplicateLabels(); + return convertAndLogViolations(); + } + + public String validateLogLevelLabel(HashMap existingLabels) { + if (logLevelLabel != null) { + return validateLogLevelLabelAgainst( + existingLabels, + logLevelLabel + ); + } + + return null; + } + + private void detectAndLogDuplicateLabels() { + List duplicatedLabels = stream(labels) + .collect(Collectors.groupingBy(Label::getName, counting())) + .entrySet() + .stream().filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + if (!duplicatedLabels.isEmpty()) { + internalLogger.addWarn( + String.format( + "There are duplicated labels which is not allowed by Loki. " + + "These labels will be deduplicated non-deterministically: %s\n", + duplicatedLabels + ) + ); + } + } + + private HashMap convertAndLogViolations() { + HashMap lokiLabels = new HashMap<>(); + + stream(labels) + .flatMap(label -> { + if (label.hasValidName()) { + return Stream.of(label); + } + + internalLogger.addWarn( + String.format( + "Ignoring label '%s' - contains invalid characters. %s\n", + label.getName(), + GitHubDocs.LABEL_NAMING.getLogMessage() + ) + ); + + return Stream.of(); + } + ) + .forEach(__ -> lokiLabels.put(__.getName(), __.getValue())); + + return lokiLabels; + } + + private String validateLogLevelLabelAgainst( + Map existingLabels, + String logLevelLabel + ) { + if (!Label.hasValidName(logLevelLabel)) { + internalLogger.addWarn( + String.format( + "Ignoring log level label '%s' - contains invalid characters. %s\n", + logLevelLabel, + GitHubDocs.LABEL_NAMING.getLogMessage() + ) + ); + + return null; + } + + if (existingLabels.remove(logLevelLabel) != null) { + internalLogger.addWarn( + String.format( + "Ignoring log level label '%s' - conflicts with label defined in configuration.\n", + logLevelLabel + ) + ); + } + + return logLevelLabel; + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java new file mode 100644 index 0000000..8a7f3cf --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java @@ -0,0 +1,86 @@ +package pl.tkowalcz.tjahzi.logback; + +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; +import ch.qos.logback.classic.spi.ILoggingEvent; +import pl.tkowalcz.tjahzi.LoggingSystem; +import pl.tkowalcz.tjahzi.TjahziLogger; +import pl.tkowalcz.tjahzi.stats.MonitoringModule; +import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper; + +import java.nio.ByteBuffer; +import java.util.Collections; + +public class LokiAppender extends LokiAppenderConfigurator { + + + private PatternLayoutEncoder encoder; + private LoggingSystem loggingSystem; + + private TjahziLogger logger; + private String logLevelLabel; + private MutableMonitoringModuleWrapper monitoringModuleWrapper; + + public PatternLayoutEncoder getEncoder() { + return encoder; + } + + public void setEncoder(PatternLayoutEncoder encoder) { + this.encoder = encoder; + } + + /** + * This is an entry point to set monitoring (statistics) hooks for this appender. This + * API is in beta and is subject to change (and probably will). + */ + public void setMonitoringModule(MonitoringModule monitoringModule) { + monitoringModuleWrapper.setMonitoringModule(monitoringModule); + } + + // @VisibleForTesting + public LoggingSystem getLoggingSystem() { + return loggingSystem; + } + + @Override + protected void append(ILoggingEvent event) { + String logLevel = event.getLevel().toString(); + byte[] logLine = encoder.encode(event); + + logger.log( + event.getTimeStamp(), + Collections.emptyMap(), + logLevelLabel, + logLevel, + ByteBuffer.wrap(logLine) + ); + } + + @Override + public void start() { + if (encoder == null) { + addError("No encoder set for the appender named [" + name + "]."); + return; + } + + encoder.start(); + + LokiAppenderFactory lokiAppenderFactory = new LokiAppenderFactory(this); + loggingSystem = lokiAppenderFactory.createAppender(); + logLevelLabel = lokiAppenderFactory.getLogLevelLabel(); + monitoringModuleWrapper = lokiAppenderFactory.getMonitoringModuleWrapper(); + + logger = loggingSystem.createLogger(); + loggingSystem.start(); + super.start(); + } + + @Override + public void stop() { + loggingSystem.close( + 1000, + thread -> addError("Loki appender was unable to stop thread on shutdown: " + thread) + ); + + super.stop(); + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java new file mode 100644 index 0000000..84f6363 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java @@ -0,0 +1,136 @@ +package pl.tkowalcz.tjahzi.logback; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.UnsynchronizedAppenderBase; + +import java.util.ArrayList; +import java.util.List; + +public abstract class LokiAppenderConfigurator extends UnsynchronizedAppenderBase { + + static final int BYTES_IN_MEGABYTE = 1024 * 1024; + + private String host; + private int port; + + private int connectTimeoutMillis = 5000; + private int readTimeoutMillis = 60_000; + private int maxRetries = 3; + + private int bufferSizeMegabytes = 32; + private boolean useOffHeapBuffer = true; + + private String logLevelLabel; + + private long batchSize = 10_2400; + private long batchWait = 5; + + private int maxRequestsInFlight = 100; + + private final List
headers = new ArrayList<>(); + private final List