Merge pull request #52 from tkowalcz/47-dynamic-label-substitution

47 dynamic label substitution

core/src/test/java/pl/tkowalcz/tjahzi/LoggingSystemTest.java

@@ -106,9 +106,7 @@ class LoggingSystemTest {
 
         // Then
         RestAssured.port = loki.getFirstMappedPort();
-//        RestAssured.port = 3100;
         RestAssured.baseURI = "http://" + loki.getHost();
-//        RestAssured.baseURI = "http://localhost";
         RestAssured.registerParser("text/plain", Parser.JSON);
 
         Awaitility

core/src/test/resources/loki-config.yaml

@@ -39,9 +39,6 @@ limits_config:
   reject_old_samples: true
   reject_old_samples_max_age: 168h
 
-chunk_store_config:
-  max_look_back_period: 0s
-
 table_manager:
   retention_deletes_enabled: false
   retention_period: 0s

log4j2-appender/README.md

@@ -111,10 +111,13 @@ of configuration so that the appender can be found.
 
 ### Lookups / variable substitution
 
-Contents of the properties are automatically interpolated by Log4j2 (see [here](https://logging.apache.org/log4j/log4j-2.2/manual/lookups.html)).
+Contents of the properties are [automatically interpolated by Log4j2](https://logging.apache.org/log4j/log4j-2.2/manual/configuration.html#PropertySubstitution).
 All environment, system etc. variable references will be replaced by their values during initialization of the appender.
-Alternatively this process could have been executed for every log message. The latter approach was deemed too expensive. If you need a mechanism
-to replace a variable after logging system initialization I would lvie to hear your use case - please file an issue.
+The exception is context/MDC (`${ctx:foo}`) value lookup as it make sense only during runtime.
+
+NOTE: This process could have been executed for every lookup type at runtime (for each log message). This approach was 
+deemed too expensive. If you need a mechanism to replace a variable (other than context/MDC) after logging system 
+initialization I would love to hear your use case - please file an issue.
 
 ## Details
 

log4j2-appender/pom.xml

@@ -1,4 +1,5 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
 
@@ -33,6 +34,20 @@
         </dependency>
 
         <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>2.13.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.9.0</version>
+            <scope>test</scope>
+        </dependency>
+
+
         <dependency>
             <groupId>com.google.code.java-allocation-instrumenter</groupId>
             <artifactId>java-allocation-instrumenter</artifactId>

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogic.java

@@ -3,33 +3,58 @@ package pl.tkowalcz.tjahzi.log4j2;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.Layout;
 import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.lookup.Interpolator;
+import org.apache.logging.log4j.core.lookup.StrLookup;
+import org.apache.logging.log4j.core.lookup.StrSubstitutor;
 import org.apache.logging.log4j.status.StatusLogger;
 import pl.tkowalcz.tjahzi.LoggingSystem;
 import pl.tkowalcz.tjahzi.TjahziLogger;
 
 import java.io.Serializable;
 import java.nio.ByteBuffer;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
 public class AppenderLogic implements BiConsumer<LogEvent, ByteBuffer> {
 
-    protected static final Logger LOGGER = StatusLogger.getLogger();
+    private static final Logger LOGGER = StatusLogger.getLogger();
+
+    private static final Map<String, String> EMPTY_MAP = new HashMap<>();
 
     private final LoggingSystem loggingSystem;
 
     private final String logLevelLabel;
     private final TjahziLogger logger;
 
+    private final Map<String, String> lokiLabels;
+    private final StrSubstitutor strSubstitutor;
+
     public AppenderLogic(
             LoggingSystem loggingSystem,
-            String logLevelLabel
+            String logLevelLabel,
+            Map<String, String> lokiLabels
     ) {
         this.loggingSystem = loggingSystem;
         this.logLevelLabel = logLevelLabel;
 
         logger = loggingSystem.createLogger();
+        this.lokiLabels = lokiLabels;
+
+        Interpolator interpolator = new Interpolator();
+        strSubstitutor = new StrSubstitutor(new StrLookup() {
+            @Override
+            public String lookup(String key) {
+                return null;
+            }
+
+            @Override
+            public String lookup(LogEvent event, String key) {
+                return interpolator.lookup(event, key);
+            }
+        });
     }
 
     public void append(
@@ -54,12 +79,28 @@ public class AppenderLogic implements BiConsumer<LogEvent, ByteBuffer> {
     public void accept(LogEvent event, ByteBuffer byteBuffer) {
         String logLevel = event.getLevel().toString();
 
+        Map<String, String> dynamicLabels = processDynamicLabelsIfAny(event);
+
         logger.log(
                 event.getTimeMillis(),
-                Collections.emptyMap(),
+                dynamicLabels,
                 logLevelLabel,
                 logLevel,
                 byteBuffer
         );
     }
+
+    private Map<String, String> processDynamicLabelsIfAny(LogEvent event) {
+        if (lokiLabels.isEmpty()) {
+            return EMPTY_MAP;
+        }
+
+        LinkedHashMap<String, String> dynamicLabels = new LinkedHashMap<>();
+        for (Map.Entry<String, String> label : lokiLabels.entrySet()) {
+            String replaced = strSubstitutor.replace(event, label.getValue());
+            dynamicLabels.put(label.getKey(), replaced);
+        }
+
+        return dynamicLabels;
+    }
 }

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelFactory.java

@@ -4,19 +4,24 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.status.StatusLogger;
 import pl.tkowalcz.tjahzi.github.GitHubDocs;
 
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.util.Arrays.stream;
 import static java.util.stream.Collectors.counting;
+import static java.util.stream.Collectors.toMap;
 
 public class LabelFactory {
 
     private static final Logger LOGGER = StatusLogger.getLogger();
 
+    private static final Pattern DYNAMIC_LABEL_PATTERN = Pattern.compile("\\$\\{ctx:[^}:]+}");
+    private static final Predicate<Map.Entry<String, String>> IS_DYNAMIC_LABEL = (entry) -> DYNAMIC_LABEL_PATTERN.matcher(entry.getValue()).find();
+
     private final String logLevelLabel;
     private final Label[] labels;
 
@@ -25,32 +30,47 @@ public class LabelFactory {
         this.labels = labels;
     }
 
-    public HashMap<String, String> convertLabelsDroppingInvalid() {
+    public LabelsDescriptor convertLabelsDroppingInvalid() {
         detectAndLogDuplicateLabels();
-        return convertAndLogViolations();
-    }
 
-    public String validateLogLevelLabel(HashMap<String, String> existingLabels) {
-        if (logLevelLabel != null) {
-            return validateLogLevelLabelAgainst(
-                    existingLabels,
-                    logLevelLabel
-            );
-        }
+        Map<String, String> allLabels = convertAndLogViolations();
 
-        return null;
+        Map<String, String> dynamicLabels = allLabels
+                .entrySet()
+                .stream()
+                .filter(IS_DYNAMIC_LABEL)
+                .collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        Map<String, String> staticLabels = allLabels
+                .entrySet()
+                .stream()
+                .filter(IS_DYNAMIC_LABEL.negate())
+                .collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        String actualLogLevelLabel = validateLogLevelLabel(
+                logLevelLabel,
+                staticLabels,
+                dynamicLabels
+        );
+
+        return new LabelsDescriptor(
+                actualLogLevelLabel,
+                staticLabels,
+                dynamicLabels
+        );
     }
 
     private void detectAndLogDuplicateLabels() {
         List<String> duplicatedLabels = stream(labels)
                 .collect(Collectors.groupingBy(Label::getName, counting()))
                 .entrySet()
-                .stream().filter(entry -> entry.getValue() > 1)
+                .stream()
+                .filter(entry -> entry.getValue() > 1)
                 .map(Map.Entry::getKey)
                 .collect(Collectors.toList());
 
         if (!duplicatedLabels.isEmpty()) {
-            LOGGER.error(
+            LOGGER.warn(
                     "There are duplicated labels which is not allowed by Loki. " +
                             "These labels will be deduplicated non-deterministically: {}",
                     duplicatedLabels
@@ -58,10 +78,8 @@ public class LabelFactory {
         }
     }
 
-    private HashMap<String, String> convertAndLogViolations() {
-        HashMap<String, String> lokiLabels = new HashMap<>();
-
-        stream(labels)
+    private Map<String, String> convertAndLogViolations() {
+        return stream(labels)
                 .flatMap(label -> {
                             if (label.hasValidName()) {
                                 return Stream.of(label);
@@ -76,15 +94,22 @@ public class LabelFactory {
                             return Stream.of();
                         }
                 )
-                .forEach(__ -> lokiLabels.put(__.getName(), __.getValue()));
-
-        return lokiLabels;
+                .collect(toMap(
+                        Label::getName,
+                        Label::getValue,
+                        (original, duplicate) -> duplicate)
+                );
     }
 
-    private String validateLogLevelLabelAgainst(
-            Map<String, String> existingLabels,
-            String logLevelLabel
+    private static String validateLogLevelLabel(
+            String logLevelLabel,
+            Map<String, String> staticLabels,
+            Map<String, String> dynamicLabels
     ) {
+        if (logLevelLabel == null) {
+            return null;
+        }
+
         if (!Label.hasValidName(logLevelLabel)) {
             LOGGER.error(
                     "Ignoring log level label '{}' - contains invalid characters. {}",
@@ -95,7 +120,13 @@ public class LabelFactory {
             return null;
         }
 
-        if (existingLabels.remove(logLevelLabel) != null) {
+        if (staticLabels.remove(logLevelLabel) != null) {
+            LOGGER.error("Log level label '{} conflicts with label defined in configuration - ignoring it.",
+                    logLevelLabel
+            );
+        }
+
+        if (dynamicLabels.remove(logLevelLabel) != null) {
             LOGGER.error("Log level label '{} conflicts with label defined in configuration - ignoring it.",
                     logLevelLabel
             );

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LabelsDescriptor.java

@@ -0,0 +1,34 @@
+package pl.tkowalcz.tjahzi.log4j2;
+
+import java.util.Map;
+
+public class LabelsDescriptor {
+
+    private final String logLevelLabel;
+
+    private final Map<String, String> staticLabels;
+    private final Map<String, String> dynamicLabels;
+
+    public LabelsDescriptor(
+            String logLevelLabel,
+            Map<String, String> staticLabels,
+            Map<String, String> dynamicLabels
+    ) {
+        this.logLevelLabel = logLevelLabel;
+        this.staticLabels = staticLabels;
+        this.dynamicLabels = dynamicLabels;
+    }
+
+    public String getLogLevelLabel() {
+        return logLevelLabel;
+    }
+
+    public Map<String, String> getStaticLabels() {
+        return staticLabels;
+    }
+
+    public Map<String, String> getDynamicLabels() {
+        return dynamicLabels;
+    }
+}
+

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppender.java

@@ -14,6 +14,7 @@ import pl.tkowalcz.tjahzi.stats.MonitoringModule;
 import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper;
 
 import java.io.Serializable;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
@@ -43,6 +44,7 @@ public class LokiAppender extends AbstractAppender {
             boolean ignoreExceptions,
             Property[] properties,
             String logLevelLabel,
+            Map<String, String> lokiLabels,
             LoggingSystem loggingSystem,
             MutableMonitoringModuleWrapper monitoringModuleWrapper
     ) {
@@ -59,7 +61,8 @@ public class LokiAppender extends AbstractAppender {
         this.loggingSystem = loggingSystem;
         this.appenderLogic = new AppenderLogic(
                 loggingSystem,
-                logLevelLabel
+                logLevelLabel,
+                lokiLabels
         );
     }
 

log4j2-appender/src/main/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderBuilder.java

@@ -15,7 +15,6 @@ import pl.tkowalcz.tjahzi.http.NettyHttpClient;
 import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper;
 import pl.tkowalcz.tjahzi.stats.StandardMonitoringModule;
 
-import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
@@ -117,13 +116,13 @@ public class LokiAppenderBuilder<B extends LokiAppenderBuilder<B>> extends Abstr
         }
 
         LabelFactory labelFactory = new LabelFactory(logLevelLabel, labels);
-        HashMap<String, String> lokiLabels = labelFactory.convertLabelsDroppingInvalid();
-        logLevelLabel = labelFactory.validateLogLevelLabel(lokiLabels);
+        LabelsDescriptor labelsDescriptor = labelFactory.convertLabelsDroppingInvalid();
+        logLevelLabel = labelsDescriptor.getLogLevelLabel();
 
         LoggingSystem loggingSystem = new TjahziInitializer().createLoggingSystem(
                 httpClient,
                 monitoringModuleWrapper,
-                lokiLabels,
+                labelsDescriptor.getStaticLabels(),
                 batchSize,
                 TimeUnit.SECONDS.toMillis(batchWait),
                 bufferSizeBytes,
@@ -137,6 +136,7 @@ public class LokiAppenderBuilder<B extends LokiAppenderBuilder<B>> extends Abstr
                 isIgnoreExceptions(),
                 getPropertyArray(),
                 logLevelLabel,
+                labelsDescriptor.getDynamicLabels(),
                 loggingSystem,
                 monitoringModuleWrapper
         );

log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/AppenderLogicTest.java

@@ -0,0 +1,109 @@
+package pl.tkowalcz.tjahzi.log4j2;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.impl.JdkMapAdapterStringMap;
+import org.apache.logging.log4j.core.impl.MutableLogEvent;
+import org.junit.jupiter.api.Test;
+import pl.tkowalcz.tjahzi.LoggingSystem;
+import pl.tkowalcz.tjahzi.TjahziLogger;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import static org.mockito.Mockito.*;
+
+class AppenderLogicTest {
+
+    @Test
+    void shouldWorkWithNoDynamicLabels() {
+        // Given
+        TjahziLogger tjahziLogger = mock(TjahziLogger.class);
+
+        LoggingSystem loggingSystem = mock(LoggingSystem.class);
+        when(loggingSystem.createLogger()).thenReturn(tjahziLogger);
+
+        long timestamp = 42L;
+        String logLevelLabel = "log_level";
+        Level logLevel = Level.INFO;
+
+        AppenderLogic logic = new AppenderLogic(
+                loggingSystem,
+                logLevelLabel,
+                Map.of()
+        );
+
+        MutableLogEvent logEvent = new MutableLogEvent();
+        logEvent.setLevel(logLevel);
+        logEvent.setLoggerName("foobar");
+        logEvent.setTimeMillis(timestamp);
+        logEvent.setContextData(new JdkMapAdapterStringMap(Map.of()));
+
+        // When
+        logic.accept(logEvent, ByteBuffer.allocate(1024));
+
+        // Then
+        verify(tjahziLogger).log(
+                eq(timestamp),
+                eq(Map.of()),
+                eq(logLevelLabel),
+                eq("INFO"),
+                any()
+        );
+    }
+
+    @Test
+    void shouldPerformSubstitutionOnDynamicLabels() {
+        // Given
+        TjahziLogger tjahziLogger = mock(TjahziLogger.class);
+
+        LoggingSystem loggingSystem = mock(LoggingSystem.class);
+        when(loggingSystem.createLogger()).thenReturn(tjahziLogger);
+
+        long timestamp = 42L;
+        String logLevelLabel = "log_level";
+        Level logLevel = Level.INFO;
+
+        Map<String, String> dynamicLabels = Map.of(
+                "id", "foo_${ctx:foo}_bar_${ctx:bar}",
+                "tennant", "${ctx:tennant}",
+                "region", "region_${ctx:region}"
+        );
+
+        Map<String, String> contextMap = Map.of(
+                "foo", "123",
+                // "bar"... - missing bar
+                "tennant", "david",
+                "region", "us_east_1"
+        );
+
+        Map<String, String> expected = Map.of(
+                "id", "foo_123_bar_${ctx:bar}",
+                "tennant", "david",
+                "region", "region_us_east_1"
+        );
+
+        AppenderLogic logic = new AppenderLogic(
+                loggingSystem,
+                logLevelLabel,
+                dynamicLabels
+        );
+
+        MutableLogEvent logEvent = new MutableLogEvent();
+        logEvent.setLevel(logLevel);
+        logEvent.setLoggerName("foobar");
+        logEvent.setTimeMillis(timestamp);
+        logEvent.setContextData(new JdkMapAdapterStringMap(contextMap));
+
+        // When
+        logic.accept(logEvent, ByteBuffer.allocate(1024));
+
+        // Then
+        verify(tjahziLogger).log(
+                eq(timestamp),
+                eq(expected),
+                eq(logLevelLabel),
+                eq("INFO"),
+                any()
+        );
+    }
+}

log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelFactoryTest.java

@@ -2,7 +2,6 @@ package pl.tkowalcz.tjahzi.log4j2;
 
 import org.junit.jupiter.api.Test;
 
-import java.util.HashMap;
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -24,13 +23,47 @@ class LabelFactoryTest {
         );
 
         // When
-        HashMap<String, String> actual = labelFactory.convertLabelsDroppingInvalid();
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(actual).containsOnly(
-                asEntry(thisShouldStay),
-                asEntry(thisTooShouldStay)
+        assertThat(actual.getStaticLabels())
+                .containsOnly(
+                        asEntry(thisShouldStay),
+                        asEntry(thisTooShouldStay)
+                );
+    }
+
+    @Test
+    void shouldIdentifyDynamicLabels() {
+        // Given
+        Label thisIsAStaticLabel = Label.createLabel("ip", "127.0.0.1");
+        Label thisIsAlsoAStaticLabel = Label.createLabel("region", "us-east-1");
+        Label thisIsADynamicLabel = Label.createLabel("tenant", "${ctx:tenant}");
+        Label thisIsAnotherDynamicLabel = Label.createLabel("tenant", "foo_${ctx:baz}_bar");
+
+        LabelFactory labelFactory = new LabelFactory(
+                "log_level",
+                thisIsAStaticLabel,
+                thisIsAlsoAStaticLabel,
+                thisIsADynamicLabel,
+                thisIsAnotherDynamicLabel
         );
+
+        // When
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
+
+        // Then
+        assertThat(actual.getStaticLabels())
+                .containsOnly(
+                        asEntry(thisIsAStaticLabel),
+                        asEntry(thisIsAlsoAStaticLabel)
+                );
+
+        assertThat(actual.getDynamicLabels())
+                .containsOnly(
+                        asEntry(thisIsADynamicLabel),
+                        asEntry(thisIsAnotherDynamicLabel)
+                );
     }
 
     @Test
@@ -48,10 +81,10 @@ class LabelFactoryTest {
         );
 
         // When
-        HashMap<String, String> actual = labelFactory.convertLabelsDroppingInvalid();
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(actual).containsOnly(
+        assertThat(actual.getStaticLabels()).containsOnly(
                 asEntry(thisShouldStay),
                 asEntry(thisShouldStayToo)
         );
@@ -70,10 +103,10 @@ class LabelFactoryTest {
         );
 
         // When
-        HashMap<String, String> actual = labelFactory.convertLabelsDroppingInvalid();
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(actual).containsOnly(
+        assertThat(actual.getStaticLabels()).containsOnly(
                 asEntry(label1),
                 asEntry(label2)
         );
@@ -93,22 +126,15 @@ class LabelFactoryTest {
                 thisShouldStay
         );
 
-        HashMap<String, String> labels = new HashMap<>(
-                Map.ofEntries(
-                        asEntry(thisShouldStay),
-                        asEntry(thisShouldBeRemovedDueToConflict)
-                )
-        );
-
         // When
-        String actual = labelFactory.validateLogLevelLabel(labels);
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(labels).containsOnly(
+        assertThat(actual.getStaticLabels()).containsOnly(
                 asEntry(thisShouldStay)
         );
 
-        assertThat(actual).isEqualTo(logLevelLabel);
+        assertThat(actual.getLogLevelLabel()).isEqualTo(logLevelLabel);
     }
 
     @Test
@@ -119,10 +145,10 @@ class LabelFactoryTest {
         LabelFactory labelFactory = new LabelFactory(logLevelLabel);
 
         // When
-        String actual = labelFactory.validateLogLevelLabel(new HashMap<>());
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(actual).isNull();
+        assertThat(actual.getLogLevelLabel()).isNull();
     }
 
     @Test
@@ -132,10 +158,10 @@ class LabelFactoryTest {
         LabelFactory labelFactory = new LabelFactory(logLevelLabel);
 
         // When
-        String actual = labelFactory.validateLogLevelLabel(new HashMap<>());
+        LabelsDescriptor actual = labelFactory.convertLabelsDroppingInvalid();
 
         // Then
-        assertThat(actual).isNull();
+        assertThat(actual.getLogLevelLabel()).isNull();
     }
 
     public Map.Entry<String, String> asEntry(Label label) {

log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LabelsContextSubstitutionTest.java

@@ -0,0 +1,107 @@
+package pl.tkowalcz.tjahzi.log4j2;
+
+import io.restassured.RestAssured;
+import io.restassured.http.ContentType;
+import io.restassured.parsing.Parser;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
+import org.junit.jupiter.api.Test;
+import org.slf4j.MDC;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.net.URI;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.core.Every.everyItem;
+
+@Testcontainers
+class LabelsContextSubstitutionTest {
+
+    @Container
+    public GenericContainer loki = new GenericContainer("grafana/loki:latest")
+            .withCommand("-config.file=/etc/loki-config.yaml")
+            .withClasspathResourceMapping("loki-config.yaml",
+                    "/etc/loki-config.yaml",
+                    BindMode.READ_ONLY)
+            .waitingFor(
+                    Wait.forHttp("/ready")
+                            .forPort(3100)
+            )
+            .withExposedPorts(3100);
+
+    @Test
+    void shouldSendData() throws Exception {
+        // Given
+        System.setProperty("loki.host", loki.getHost());
+        System.setProperty("loki.port", loki.getFirstMappedPort().toString());
+
+        URI uri = getClass()
+                .getClassLoader()
+                .getResource("labels-context-substitution-test-configuration.xml")
+                .toURI();
+
+        ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false))
+                .setConfigLocation(uri);
+
+        String expectedLogLine = "Test";
+        long expectedTimestamp = System.currentTimeMillis();
+
+        Logger logger = LogManager.getLogger(LokiAppenderTest.class);
+
+        // When
+        MDC.put("object", "bust_ticket");
+        MDC.put("owner", "wally");
+        logger.info("Test1");
+
+        MDC.put("object", "comb");
+        MDC.put("owner", "jennifer");
+        logger.info("Test2");
+
+        MDC.clear();
+        logger.info("Test3");
+
+        // Then
+        RestAssured.port = loki.getFirstMappedPort();
+        RestAssured.baseURI = "http://" + loki.getHost();
+        RestAssured.registerParser("text/plain", Parser.JSON);
+
+        Awaitility
+                .await()
+                .atMost(Durations.TEN_SECONDS)
+                .pollInterval(Durations.ONE_SECOND)
+                .ignoreExceptions()
+                .untilAsserted(() -> {
+                    given()
+                            .contentType(ContentType.URLENC)
+                            .urlEncodingEnabled(false)
+                            .formParam("query=%7Bserver%3D%22127.0.0.1%22%7D")
+                            .when()
+                            .get("/loki/api/v1/query_range")
+                            .then()
+                            .log()
+                            .all()
+                            .statusCode(200)
+                            .body("status", equalTo("success"))
+                            .body("data.result.size()", equalTo(3))
+                            .body("data.result.stream.server", everyItem(equalTo("127.0.0.1")))
+                            .body("data.result.stream.object", contains("prefix_${ctx:object}", "prefix_bust_ticket", "prefix_comb"))
+                            .body("data.result.stream.owner", contains("${ctx:owner}_suffix", "wally_suffix", "jennifer_suffix"))
+                            .body("data.result.values",
+                                    hasItems(
+                                            hasItems(hasItems(("LokiAppenderTest - Test3"))),
+                                            hasItems(hasItems(("LokiAppenderTest - Test1"))),
+                                            hasItems(hasItems(("LokiAppenderTest - Test2")))
+                                    )
+                            );
+                });
+    }
+}

log4j2-appender/src/test/resources/labels-context-substitution-test-configuration.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration packages="pl.tkowalcz.tjahzi.log4j2">
+    <Loggers>
+        <Root level="INFO">
+            <AppenderRef ref="Loki"/>
+        </Root>
+    </Loggers>
+
+    <appenders>
+        <Loki name="Loki">
+            <host>${sys:loki.host}</host>
+            <port>${sys:loki.port}</port>
+
+            <ThresholdFilter level="ALL"/>
+            <PatternLayout>
+                <Pattern>%c{1} - %m</Pattern>
+            </PatternLayout>
+
+            <Header name="server" value="127.0.0.1"/>
+            <Label name="server" value="127.0.0.1"/>
+            <Label name="object" value="prefix_${ctx:object}"/>
+            <Label name="owner" value="${ctx:owner}_suffix"/>
+        </Loki>
+    </appenders>
+</configuration>

log4j2-appender/src/test/resources/loki-config.yaml

@@ -40,9 +40,6 @@ limits_config:
   reject_old_samples_max_age: 168h
   max_entries_limit_per_query: 10000
 
-chunk_store_config:
-  max_look_back_period: 0s
-
 table_manager:
   retention_deletes_enabled: false
   retention_period: 0s

logback-appender/src/test/resources/loki-config.yaml

@@ -40,9 +40,6 @@ limits_config:
   reject_old_samples_max_age: 168h
   max_entries_limit_per_query: 10000
 
-chunk_store_config:
-  max_look_back_period: 0s
-
 table_manager:
   retention_deletes_enabled: false
   retention_period: 0s