From a834b22c8f7959abc243f558e9f9d9f98eeee619 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Sun, 28 Feb 2021 19:04:07 +0100 Subject: [PATCH 01/11] Re #36 - initial stab at implementing logback logger. --- .../tkowalcz/tjahzi/log4j2/LabelFactory.java | 8 +- logback-appender/pom.xml | 86 +++++++++++ .../pl/tkowalcz/tjahzi/logback/Header.java | 23 +++ .../pl/tkowalcz/tjahzi/logback/Label.java | 35 +++++ .../tkowalcz/tjahzi/logback/LabelFactory.java | 103 +++++++++++++ .../tkowalcz/tjahzi/logback/LokiAppender.java | 68 +++++++++ .../logback/LokiAppenderConfigurator.java | 136 ++++++++++++++++++ .../tjahzi/logback/LokiAppenderFactory.java | 79 ++++++++++ .../tjahzi/logback/LokiAppenderTest.java | 121 ++++++++++++++++ .../test/resources/logback-configuration.xml | 33 +++++ .../src/test/resources/loki-config.yaml | 48 +++++++ pom.xml | 1 + 12 files changed, 738 insertions(+), 3 deletions(-) create mode 100644 logback-appender/pom.xml create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java create mode 100644 logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderFactory.java create mode 100644 logback-appender/src/test/java/pl/tkowalcz/tjahzi/logback/LokiAppenderTest.java create mode 100644 logback-appender/src/test/resources/logback-configuration.xml create mode 100644 logback-appender/src/test/resources/loki-config.yaml diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java index d9d8f98..527fcc6 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java @@ -52,7 +52,7 @@ public class LabelFactory { if (!duplicatedLabels.isEmpty()) { LOGGER.error( "There are duplicated labels which is not allowed by Loki. " + - "These labels will be deduplicated undeterministically: {}", + "These labels will be deduplicated non-deterministically: {}", duplicatedLabels ); } @@ -67,7 +67,8 @@ public class LabelFactory { return Stream.of(label); } - LOGGER.error("Label '{}' contains invalid characters - ignoring it. {}", + LOGGER.error( + "Ignoring label '{}' - contains invalid characters. {}", label.getName(), GitHubDocs.LABEL_NAMING.getLogMessage() ); @@ -85,7 +86,8 @@ public class LabelFactory { String logLevelLabel ) { if (!Label.hasValidName(logLevelLabel)) { - LOGGER.error("Log level label '{}' contains invalid characters - ignoring it. {}", + LOGGER.error( + "Ignoring log level label '{}' - contains invalid characters. {}", logLevelLabel, GitHubDocs.LABEL_NAMING.getLogMessage() ); diff --git a/logback-appender/pom.xml b/logback-appender/pom.xml new file mode 100644 index 0000000..995e16b --- /dev/null +++ b/logback-appender/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + + pl.tkowalcz.tjahzi + tjahzi-parent + 0.9.5-SNAPSHOT + + + logback-appender + logback-appender + jar + + + + pl.tkowalcz.tjahzi + core + 0.9.5-SNAPSHOT + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + com.google.code.java-allocation-instrumenter + java-allocation-instrumenter + 3.3.0 + test + + + io.rest-assured + rest-assured + 4.3.1 + test + + + org.awaitility + awaitility + 4.0.3 + test + + + org.assertj + assertj-core + 3.15.0 + test + + + + + org.junit.jupiter + junit-jupiter-api + 5.6.2 + test + + + org.junit.jupiter + junit-jupiter-params + 5.6.2 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.6.2 + test + + + org.testcontainers + testcontainers + 1.14.3 + test + + + org.testcontainers + junit-jupiter + 1.14.3 + test + + + diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java new file mode 100644 index 0000000..bb18900 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Header.java @@ -0,0 +1,23 @@ +package pl.tkowalcz.tjahzi.logback; + +public class Header { + + private String name; + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java new file mode 100644 index 0000000..dbb5028 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/Label.java @@ -0,0 +1,35 @@ +package pl.tkowalcz.tjahzi.logback; + +import java.util.regex.Pattern; + +public class Label { + + private static final Pattern LABEL_NAME_PATTER = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); + + private String name; + private String value; + + public boolean hasValidName() { + return hasValidName(getName()); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public static boolean hasValidName(String label) { + return LABEL_NAME_PATTER.matcher(label).matches(); + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java new file mode 100644 index 0000000..5095755 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LabelFactory.java @@ -0,0 +1,103 @@ +package pl.tkowalcz.tjahzi.logback; + +import pl.tkowalcz.tjahzi.github.GitHubDocs; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.counting; + +public class LabelFactory { + + private final String logLevelLabel; + private final Label[] labels; + + public LabelFactory(String logLevelLabel, Label... labels) { + this.logLevelLabel = logLevelLabel; + this.labels = labels; + } + + public HashMap convertLabelsDroppingInvalid() { + detectAndLogDuplicateLabels(); + return convertAndLogViolations(); + } + + public String validateLogLevelLabel(HashMap existingLabels) { + if (logLevelLabel != null) { + return validateLogLevelLabelAgainst( + existingLabels, + logLevelLabel + ); + } + + return null; + } + + private void detectAndLogDuplicateLabels() { + List duplicatedLabels = stream(labels) + .collect(Collectors.groupingBy(Label::getName, counting())) + .entrySet() + .stream().filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + if (!duplicatedLabels.isEmpty()) { + System.out.printf( + "There are duplicated labels which is not allowed by Loki. " + + "These labels will be deduplicated non-deterministically: %s\n", + duplicatedLabels + ); + } + } + + private HashMap convertAndLogViolations() { + HashMap lokiLabels = new HashMap<>(); + + stream(labels) + .flatMap(label -> { + if (label.hasValidName()) { + return Stream.of(label); + } + + System.out.printf( + "Ignoring label '%s' - contains invalid characters. %s\n", + label.getName(), + GitHubDocs.LABEL_NAMING.getLogMessage() + ); + + return Stream.of(); + } + ) + .forEach(__ -> lokiLabels.put(__.getName(), __.getValue())); + + return lokiLabels; + } + + private String validateLogLevelLabelAgainst( + Map existingLabels, + String logLevelLabel + ) { + if (!Label.hasValidName(logLevelLabel)) { + System.out.printf( + "Ignoring log level label '%s' - contains invalid characters. %s\n", + logLevelLabel, + GitHubDocs.LABEL_NAMING.getLogMessage() + ); + + return null; + } + + if (existingLabels.remove(logLevelLabel) != null) { + System.out.printf( + "Ignoring log level label '%s' - conflicts with label defined in configuration.\n", + logLevelLabel + ); + } + + return logLevelLabel; + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java new file mode 100644 index 0000000..4ca6eb1 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java @@ -0,0 +1,68 @@ +package pl.tkowalcz.tjahzi.logback; + +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; +import ch.qos.logback.classic.spi.ILoggingEvent; +import pl.tkowalcz.tjahzi.LoggingSystem; +import pl.tkowalcz.tjahzi.TjahziLogger; + +import java.nio.ByteBuffer; +import java.util.Collections; + +public class LokiAppender extends LokiAppenderConfigurator { + + private PatternLayoutEncoder encoder; + private LoggingSystem loggingSystem; + + private TjahziLogger logger; + private String logLevelLabel; + + public PatternLayoutEncoder getEncoder() { + return encoder; + } + + public void setEncoder(PatternLayoutEncoder encoder) { + this.encoder = encoder; + } + + @Override + protected void append(ILoggingEvent event) { + String logLevel = event.getLevel().toString(); + byte[] logLine = encoder.encode(event); + + logger.log( + event.getTimeStamp(), + Collections.emptyMap(), + logLevelLabel, + logLevel, + ByteBuffer.wrap(logLine) + ); + } + + @Override + public void start() { + if (encoder == null) { + addError("No encoder set for the appender named [" + name + "]."); + return; + } + + encoder.start(); + + LokiAppenderFactory lokiAppenderFactory = new LokiAppenderFactory(this); + loggingSystem = lokiAppenderFactory.createAppender(); + logLevelLabel = lokiAppenderFactory.getLogLevelLabel(); + + logger = loggingSystem.createLogger(); + loggingSystem.start(); + super.start(); + } + + @Override + public void stop() { + loggingSystem.close( + 1000, + thread -> System.out.println("Loki appender was unable to stop thread on shutdown: " + thread) + ); + + super.stop(); + } +} diff --git a/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java new file mode 100644 index 0000000..84f6363 --- /dev/null +++ b/logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppenderConfigurator.java @@ -0,0 +1,136 @@ +package pl.tkowalcz.tjahzi.logback; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.UnsynchronizedAppenderBase; + +import java.util.ArrayList; +import java.util.List; + +public abstract class LokiAppenderConfigurator extends UnsynchronizedAppenderBase { + + static final int BYTES_IN_MEGABYTE = 1024 * 1024; + + private String host; + private int port; + + private int connectTimeoutMillis = 5000; + private int readTimeoutMillis = 60_000; + private int maxRetries = 3; + + private int bufferSizeMegabytes = 32; + private boolean useOffHeapBuffer = true; + + private String logLevelLabel; + + private long batchSize = 10_2400; + private long batchWait = 5; + + private int maxRequestsInFlight = 100; + + private final List
headers = new ArrayList<>(); + private final List