From c9bbb04e4d91dfd607722754f812bbc9329dfeef Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Sun, 2 May 2021 21:18:07 +0200 Subject: [PATCH] Re #47: Dynamic label substitution for ctx placeholders. --- log4j2-appender/README.md | 9 +- .../tkowalcz/tjahzi/log4j2/AppenderLogic.java | 49 +++++++- .../tkowalcz/tjahzi/log4j2/LabelFactory.java | 81 +++++++++---- .../tjahzi/log4j2/LabelsDescriptor.java | 34 ++++++ .../tkowalcz/tjahzi/log4j2/LokiAppender.java | 5 +- .../tjahzi/log4j2/LokiAppenderBuilder.java | 8 +- .../tjahzi/log4j2/AppenderLogicTest.java | 109 ++++++++++++++++++ .../tjahzi/log4j2/LabelFactoryTest.java | 72 ++++++++---- .../log4j2/LabelsContextSubstitutionTest.java | 107 +++++++++++++++++ ...ontext-substitution-test-configuration.xml | 25 ++++ 10 files changed, 439 insertions(+), 60 deletions(-) create mode 100644 log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelsDescriptor.java create mode 100644 log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogicTest.java create mode 100644 log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelsContextSubstitutionTest.java create mode 100644 log4j2-appender/src/test/resources/labels-context-substitution-test-configuration.xml diff --git a/log4j2-appender/README.md b/log4j2-appender/README.md index a01dfd0..d91f847 100644 --- a/log4j2-appender/README.md +++ b/log4j2-appender/README.md @@ -111,10 +111,13 @@ of configuration so that the appender can be found. ### Lookups / variable substitution -Contents of the properties are automatically interpolated by Log4j2 (see [here](https://logging.apache.org/log4j/log4j-2.2/manual/lookups.html)). +Contents of the properties are [automatically interpolated by Log4j2](https://logging.apache.org/log4j/log4j-2.2/manual/configuration.html#PropertySubstitution). All environment, system etc. variable references will be replaced by their values during initialization of the appender. -Alternatively this process could have been executed for every log message. The latter approach was deemed too expensive. If you need a mechanism -to replace a variable after logging system initialization I would lvie to hear your use case - please file an issue. +The exception is context/MDC (`${ctx:foo}`) value lookup as it make sense only during runtime. + +NOTE: This process could have been executed for every lookup type at runtime (for each log message). This approach was +deemed too expensive. If you need a mechanism to replace a variable (other than context/MDC) after logging system +initialization I would love to hear your use case - please file an issue. ## Details diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogic.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogic.java index 4637f17..d4d15e1 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogic.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogic.java @@ -3,33 +3,58 @@ package pl.tkowalcz.tjahzi.log4j2; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.lookup.Interpolator; +import org.apache.logging.log4j.core.lookup.StrLookup; +import org.apache.logging.log4j.core.lookup.StrSubstitutor; import org.apache.logging.log4j.status.StatusLogger; import pl.tkowalcz.tjahzi.LoggingSystem; import pl.tkowalcz.tjahzi.TjahziLogger; import java.io.Serializable; import java.nio.ByteBuffer; -import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; public class AppenderLogic implements BiConsumer { - protected static final Logger LOGGER = StatusLogger.getLogger(); + private static final Logger LOGGER = StatusLogger.getLogger(); + + private static final Map EMPTY_MAP = new HashMap<>(); private final LoggingSystem loggingSystem; private final String logLevelLabel; private final TjahziLogger logger; + private final Map lokiLabels; + private final StrSubstitutor strSubstitutor; + public AppenderLogic( LoggingSystem loggingSystem, - String logLevelLabel + String logLevelLabel, + Map lokiLabels ) { this.loggingSystem = loggingSystem; this.logLevelLabel = logLevelLabel; logger = loggingSystem.createLogger(); + this.lokiLabels = lokiLabels; + + Interpolator interpolator = new Interpolator(); + strSubstitutor = new StrSubstitutor(new StrLookup() { + @Override + public String lookup(String key) { + return null; + } + + @Override + public String lookup(LogEvent event, String key) { + return interpolator.lookup(event, key); + } + }); } public void append( @@ -54,12 +79,28 @@ public class AppenderLogic implements BiConsumer { public void accept(LogEvent event, ByteBuffer byteBuffer) { String logLevel = event.getLevel().toString(); + Map dynamicLabels = processDynamicLabelsIfAny(event); + logger.log( event.getTimeMillis(), - Collections.emptyMap(), + dynamicLabels, logLevelLabel, logLevel, byteBuffer ); } + + private Map processDynamicLabelsIfAny(LogEvent event) { + if (lokiLabels.isEmpty()) { + return EMPTY_MAP; + } + + LinkedHashMap dynamicLabels = new LinkedHashMap<>(); + for (Map.Entry label : lokiLabels.entrySet()) { + String replaced = strSubstitutor.replace(event, label.getValue()); + dynamicLabels.put(label.getKey(), replaced); + } + + return dynamicLabels; + } } diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java index 527fcc6..0fb7431 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java @@ -4,19 +4,24 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.status.StatusLogger; import pl.tkowalcz.tjahzi.github.GitHubDocs; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Arrays.stream; import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.toMap; public class LabelFactory { private static final Logger LOGGER = StatusLogger.getLogger(); + private static final Pattern DYNAMIC_LABEL_PATTERN = Pattern.compile("\\$\\{ctx:[^}:]+}"); + private static final Predicate> IS_DYNAMIC_LABEL = (entry) -> DYNAMIC_LABEL_PATTERN.matcher(entry.getValue()).find(); + private final String logLevelLabel; private final Label[] labels; @@ -25,32 +30,47 @@ public class LabelFactory { this.labels = labels; } - public HashMap convertLabelsDroppingInvalid() { + public LabelsDescriptor convertLabelsDroppingInvalid() { detectAndLogDuplicateLabels(); - return convertAndLogViolations(); - } - public String validateLogLevelLabel(HashMap existingLabels) { - if (logLevelLabel != null) { - return validateLogLevelLabelAgainst( - existingLabels, - logLevelLabel - ); - } + Map allLabels = convertAndLogViolations(); - return null; + Map dynamicLabels = allLabels + .entrySet() + .stream() + .filter(IS_DYNAMIC_LABEL) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map staticLabels = allLabels + .entrySet() + .stream() + .filter(IS_DYNAMIC_LABEL.negate()) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + + String actualLogLevelLabel = validateLogLevelLabel( + logLevelLabel, + staticLabels, + dynamicLabels + ); + + return new LabelsDescriptor( + actualLogLevelLabel, + staticLabels, + dynamicLabels + ); } private void detectAndLogDuplicateLabels() { List duplicatedLabels = stream(labels) .collect(Collectors.groupingBy(Label::getName, counting())) .entrySet() - .stream().filter(entry -> entry.getValue() > 1) + .stream() + .filter(entry -> entry.getValue() > 1) .map(Map.Entry::getKey) .collect(Collectors.toList()); if (!duplicatedLabels.isEmpty()) { - LOGGER.error( + LOGGER.warn( "There are duplicated labels which is not allowed by Loki. " + "These labels will be deduplicated non-deterministically: {}", duplicatedLabels @@ -58,10 +78,8 @@ public class LabelFactory { } } - private HashMap convertAndLogViolations() { - HashMap lokiLabels = new HashMap<>(); - - stream(labels) + private Map convertAndLogViolations() { + return stream(labels) .flatMap(label -> { if (label.hasValidName()) { return Stream.of(label); @@ -76,15 +94,22 @@ public class LabelFactory { return Stream.of(); } ) - .forEach(__ -> lokiLabels.put(__.getName(), __.getValue())); - - return lokiLabels; + .collect(toMap( + Label::getName, + Label::getValue, + (original, duplicate) -> duplicate) + ); } - private String validateLogLevelLabelAgainst( - Map existingLabels, - String logLevelLabel + private static String validateLogLevelLabel( + String logLevelLabel, + Map staticLabels, + Map dynamicLabels ) { + if (logLevelLabel == null) { + return null; + } + if (!Label.hasValidName(logLevelLabel)) { LOGGER.error( "Ignoring log level label '{}' - contains invalid characters. {}", @@ -95,7 +120,13 @@ public class LabelFactory { return null; } - if (existingLabels.remove(logLevelLabel) != null) { + if (staticLabels.remove(logLevelLabel) != null) { + LOGGER.error("Log level label '{} conflicts with label defined in configuration - ignoring it.", + logLevelLabel + ); + } + + if (dynamicLabels.remove(logLevelLabel) != null) { LOGGER.error("Log level label '{} conflicts with label defined in configuration - ignoring it.", logLevelLabel ); diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelsDescriptor.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelsDescriptor.java new file mode 100644 index 0000000..476f94f --- /dev/null +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelsDescriptor.java @@ -0,0 +1,34 @@ +package pl.tkowalcz.tjahzi.log4j2; + +import java.util.Map; + +public class LabelsDescriptor { + + private final String logLevelLabel; + + private final Map staticLabels; + private final Map dynamicLabels; + + public LabelsDescriptor( + String logLevelLabel, + Map staticLabels, + Map dynamicLabels + ) { + this.logLevelLabel = logLevelLabel; + this.staticLabels = staticLabels; + this.dynamicLabels = dynamicLabels; + } + + public String getLogLevelLabel() { + return logLevelLabel; + } + + public Map getStaticLabels() { + return staticLabels; + } + + public Map getDynamicLabels() { + return dynamicLabels; + } +} + diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppender.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppender.java index f81d33c..309c46f 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppender.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppender.java @@ -14,6 +14,7 @@ import pl.tkowalcz.tjahzi.stats.MonitoringModule; import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper; import java.io.Serializable; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -43,6 +44,7 @@ public class LokiAppender extends AbstractAppender { boolean ignoreExceptions, Property[] properties, String logLevelLabel, + Map lokiLabels, LoggingSystem loggingSystem, MutableMonitoringModuleWrapper monitoringModuleWrapper ) { @@ -59,7 +61,8 @@ public class LokiAppender extends AbstractAppender { this.loggingSystem = loggingSystem; this.appenderLogic = new AppenderLogic( loggingSystem, - logLevelLabel + logLevelLabel, + lokiLabels ); } diff --git a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java index 3e67f03..8f1196e 100644 --- a/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java +++ b/log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java @@ -15,7 +15,6 @@ import pl.tkowalcz.tjahzi.http.NettyHttpClient; import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper; import pl.tkowalcz.tjahzi.stats.StandardMonitoringModule; -import java.util.HashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -117,13 +116,13 @@ public class LokiAppenderBuilder> extends Abstr } LabelFactory labelFactory = new LabelFactory(logLevelLabel, labels); - HashMap lokiLabels = labelFactory.convertLabelsDroppingInvalid(); - logLevelLabel = labelFactory.validateLogLevelLabel(lokiLabels); + LabelsDescriptor labelsDescriptor = labelFactory.convertLabelsDroppingInvalid(); + logLevelLabel = labelsDescriptor.getLogLevelLabel(); LoggingSystem loggingSystem = new TjahziInitializer().createLoggingSystem( httpClient, monitoringModuleWrapper, - lokiLabels, + labelsDescriptor.getStaticLabels(), batchSize, TimeUnit.SECONDS.toMillis(batchWait), bufferSizeBytes, @@ -137,6 +136,7 @@ public class LokiAppenderBuilder> extends Abstr isIgnoreExceptions(), getPropertyArray(), logLevelLabel, + labelsDescriptor.getDynamicLabels(), loggingSystem, monitoringModuleWrapper ); diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogicTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogicTest.java new file mode 100644 index 0000000..accfbb7 --- /dev/null +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogicTest.java @@ -0,0 +1,109 @@ +package pl.tkowalcz.tjahzi.log4j2; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.impl.JdkMapAdapterStringMap; +import org.apache.logging.log4j.core.impl.MutableLogEvent; +import org.junit.jupiter.api.Test; +import pl.tkowalcz.tjahzi.LoggingSystem; +import pl.tkowalcz.tjahzi.TjahziLogger; + +import java.nio.ByteBuffer; +import java.util.Map; + +import static org.mockito.Mockito.*; + +class AppenderLogicTest { + + @Test + void shouldWorkWithNoDynamicLabels() { + // Given + TjahziLogger tjahziLogger = mock(TjahziLogger.class); + + LoggingSystem loggingSystem = mock(LoggingSystem.class); + when(loggingSystem.createLogger()).thenReturn(tjahziLogger); + + long timestamp = 42L; + String logLevelLabel = "log_level"; + Level logLevel = Level.INFO; + + AppenderLogic logic = new AppenderLogic( + loggingSystem, + logLevelLabel, + Map.of() + ); + + MutableLogEvent logEvent = new MutableLogEvent(); + logEvent.setLevel(logLevel); + logEvent.setLoggerName("foobar"); + logEvent.setTimeMillis(timestamp); + logEvent.setContextData(new JdkMapAdapterStringMap(Map.of())); + + // When + logic.accept(logEvent, ByteBuffer.allocate(1024)); + + // Then + verify(tjahziLogger).log( + eq(timestamp), + eq(Map.of()), + eq(logLevelLabel), + eq("INFO"), + any() + ); + } + + @Test + void shouldPerformSubstitutionOnDynamicLabels() { + // Given + TjahziLogger tjahziLogger = mock(TjahziLogger.class); + + LoggingSystem loggingSystem = mock(LoggingSystem.class); + when(loggingSystem.createLogger()).thenReturn(tjahziLogger); + + long timestamp = 42L; + String logLevelLabel = "log_level"; + Level logLevel = Level.INFO; + + Map dynamicLabels = Map.of( + "id", "foo_${ctx:foo}_bar_${ctx:bar}", + "tennant", "${ctx:tennant}", + "region", "region_${ctx:region}" + ); + + Map contextMap = Map.of( + "foo", "123", + // "bar"... - missing bar + "tennant", "david", + "region", "us_east_1" + ); + + Map expected = Map.of( + "id", "foo_123_bar_${ctx:bar}", + "tennant", "david", + "region", "region_us_east_1" + ); + + AppenderLogic logic = new AppenderLogic( + loggingSystem, + logLevelLabel, + dynamicLabels + ); + + MutableLogEvent logEvent = new MutableLogEvent(); + logEvent.setLevel(logLevel); + logEvent.setLoggerName("foobar"); + logEvent.setTimeMillis(timestamp); + logEvent.setContextData(new JdkMapAdapterStringMap(contextMap)); + + // When + logic.accept(logEvent, ByteBuffer.allocate(1024)); + + // Then + verify(tjahziLogger).log( + eq(timestamp), + eq(expected), + eq(logLevelLabel), + eq("INFO"), + any() + ); + } +} diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelFactoryTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelFactoryTest.java index 4e5a8e3..7429aee 100644 --- a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelFactoryTest.java +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelFactoryTest.java @@ -2,7 +2,6 @@ package pl.tkowalcz.tjahzi.log4j2; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -24,13 +23,47 @@ class LabelFactoryTest { ); // When - HashMap actual = labelFactory.convertLabelsDroppingInvalid(); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(actual).containsOnly( - asEntry(thisShouldStay), - asEntry(thisTooShouldStay) + assertThat(actual.getStaticLabels()) + .containsOnly( + asEntry(thisShouldStay), + asEntry(thisTooShouldStay) + ); + } + + @Test + void shouldIdentifyDynamicLabels() { + // Given + Label thisIsAStaticLabel = Label.createLabel("ip", "127.0.0.1"); + Label thisIsAlsoAStaticLabel = Label.createLabel("region", "us-east-1"); + Label thisIsADynamicLabel = Label.createLabel("tenant", "${ctx:tenant}"); + Label thisIsAnotherDynamicLabel = Label.createLabel("tenant", "foo_${ctx:baz}_bar"); + + LabelFactory labelFactory = new LabelFactory( + "log_level", + thisIsAStaticLabel, + thisIsAlsoAStaticLabel, + thisIsADynamicLabel, + thisIsAnotherDynamicLabel ); + + // When + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); + + // Then + assertThat(actual.getStaticLabels()) + .containsOnly( + asEntry(thisIsAStaticLabel), + asEntry(thisIsAlsoAStaticLabel) + ); + + assertThat(actual.getDynamicLabels()) + .containsOnly( + asEntry(thisIsADynamicLabel), + asEntry(thisIsAnotherDynamicLabel) + ); } @Test @@ -48,10 +81,10 @@ class LabelFactoryTest { ); // When - HashMap actual = labelFactory.convertLabelsDroppingInvalid(); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(actual).containsOnly( + assertThat(actual.getStaticLabels()).containsOnly( asEntry(thisShouldStay), asEntry(thisShouldStayToo) ); @@ -70,10 +103,10 @@ class LabelFactoryTest { ); // When - HashMap actual = labelFactory.convertLabelsDroppingInvalid(); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(actual).containsOnly( + assertThat(actual.getStaticLabels()).containsOnly( asEntry(label1), asEntry(label2) ); @@ -93,22 +126,15 @@ class LabelFactoryTest { thisShouldStay ); - HashMap labels = new HashMap<>( - Map.ofEntries( - asEntry(thisShouldStay), - asEntry(thisShouldBeRemovedDueToConflict) - ) - ); - // When - String actual = labelFactory.validateLogLevelLabel(labels); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(labels).containsOnly( + assertThat(actual.getStaticLabels()).containsOnly( asEntry(thisShouldStay) ); - assertThat(actual).isEqualTo(logLevelLabel); + assertThat(actual.getLogLevelLabel()).isEqualTo(logLevelLabel); } @Test @@ -119,10 +145,10 @@ class LabelFactoryTest { LabelFactory labelFactory = new LabelFactory(logLevelLabel); // When - String actual = labelFactory.validateLogLevelLabel(new HashMap<>()); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(actual).isNull(); + assertThat(actual.getLogLevelLabel()).isNull(); } @Test @@ -132,10 +158,10 @@ class LabelFactoryTest { LabelFactory labelFactory = new LabelFactory(logLevelLabel); // When - String actual = labelFactory.validateLogLevelLabel(new HashMap<>()); + LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid(); // Then - assertThat(actual).isNull(); + assertThat(actual.getLogLevelLabel()).isNull(); } public Map.Entry asEntry(Label label) { diff --git a/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelsContextSubstitutionTest.java b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelsContextSubstitutionTest.java new file mode 100644 index 0000000..ba3c812 --- /dev/null +++ b/log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelsContextSubstitutionTest.java @@ -0,0 +1,107 @@ +package pl.tkowalcz.tjahzi.log4j2; + +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import io.restassured.parsing.Parser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.awaitility.Durations; +import org.junit.jupiter.api.Test; +import org.slf4j.MDC; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.net.URI; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.core.Every.everyItem; + +@Testcontainers +class LabelsContextSubstitutionTest { + + @Container + public GenericContainer loki = new GenericContainer("grafana/loki:latest") + .withCommand("-config.file=/etc/loki-config.yaml") + .withClasspathResourceMapping("loki-config.yaml", + "/etc/loki-config.yaml", + BindMode.READ_ONLY) + .waitingFor( + Wait.forHttp("/ready") + .forPort(3100) + ) + .withExposedPorts(3100); + + @Test + void shouldSendData() throws Exception { + // Given + System.setProperty("loki.host", loki.getHost()); + System.setProperty("loki.port", loki.getFirstMappedPort().toString()); + + URI uri = getClass() + .getClassLoader() + .getResource("labels-context-substitution-test-configuration.xml") + .toURI(); + + ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false)) + .setConfigLocation(uri); + + String expectedLogLine = "Test"; + long expectedTimestamp = System.currentTimeMillis(); + + Logger logger = LogManager.getLogger(LokiAppenderTest.class); + + // When + MDC.put("object", "bust_ticket"); + MDC.put("owner", "wally"); + logger.info("Test1"); + + MDC.put("object", "comb"); + MDC.put("owner", "jennifer"); + logger.info("Test2"); + + MDC.clear(); + logger.info("Test3"); + + // Then + RestAssured.port = loki.getFirstMappedPort(); + RestAssured.baseURI = "http://" + loki.getHost(); + RestAssured.registerParser("text/plain", Parser.JSON); + + Awaitility + .await() + .atMost(Durations.TEN_SECONDS) + .pollInterval(Durations.ONE_SECOND) + .ignoreExceptions() + .untilAsserted(() -> { + given() + .contentType(ContentType.URLENC) + .urlEncodingEnabled(false) + .formParam("query=%7Bserver%3D%22127.0.0.1%22%7D") + .when() + .get("/loki/api/v1/query_range") + .then() + .log() + .all() + .statusCode(200) + .body("status", equalTo("success")) + .body("data.result.size()", equalTo(3)) + .body("data.result.stream.server", everyItem(equalTo("127.0.0.1"))) + .body("data.result.stream.object", contains("prefix_${ctx:object}", "prefix_bust_ticket", "prefix_comb")) + .body("data.result.stream.owner", contains("${ctx:owner}_suffix", "wally_suffix", "jennifer_suffix")) + .body("data.result.values", + hasItems( + hasItems(hasItems(("LokiAppenderTest - Test3"))), + hasItems(hasItems(("LokiAppenderTest - Test1"))), + hasItems(hasItems(("LokiAppenderTest - Test2"))) + ) + ); + }); + } +} diff --git a/log4j2-appender/src/test/resources/labels-context-substitution-test-configuration.xml b/log4j2-appender/src/test/resources/labels-context-substitution-test-configuration.xml new file mode 100644 index 0000000..0522457 --- /dev/null +++ b/log4j2-appender/src/test/resources/labels-context-substitution-test-configuration.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + ${sys:loki.host} + ${sys:loki.port} + + + + %c{1} - %m + + +
+