Merge pull request #42 from tkowalcz/40-logback-encoder

40 logback encoder

README.md

@@ -23,6 +23,9 @@ is implemented in the `core` component. On top of that we built appenders for `l
 1. We also provide a no-dependency versions of these appenders. 
 1. Includes in-house implementation of protobuf wire format for Loki to reduce dependencies and improve performance. 
 
+These points apply to log4j2 appender. Logback design does not allow allocation free operation, but we go to great
+lengths to reduce allocations and even provide an optimized version of pattern layout.
+
 Log4j2 appender is currently used to ship logs from tens of servers at about 10GB of logs per day.
 
 ## Getting started

logback-appender-nodep/pom.xml

@@ -11,6 +11,7 @@
     <name>logback-appender-nodep</name>
     <artifactId>logback-appender-nodep</artifactId>
     <packaging>jar</packaging>
+    <version>0.1.1-SNAPSHOT</version>
 
     <properties>
         <!--add comma separated artifact names from dependencies, to be included in unpacking-->

logback-appender/README.md

@@ -28,14 +28,13 @@ On top of the `Core` component that sends logs to Loki this project gives you Lo
 1. Include minimal appender configuration in your existing logback configuration:
 
 ```xml
-
 <appender name="Loki" class="pl.tkowalcz.tjahzi.logback.LokiAppender">
     <host>${loki.host}</host>
     <port>${loki.port}</port>
 
-    <encoder>
+    <efficientLayout>
         <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-    </encoder>
+    </efficientLayout>
 
     <label>
         <name>server</name>
@@ -47,27 +46,43 @@ On top of the `Core` component that sends logs to Loki this project gives you Lo
 1. Reference the appender from inside one of your logger definitions:
 
 ```xml
-
 <root level="debug">
     <appender-ref ref="Loki"/>
 </root>
 ```
 
+## Custom pattern layout and decreasing allocations
+
+Tjahzi provides its own pattern layout class `ch.qos.logback.core.pattern.EfficientPatternLayout`. It is based on
+existing `PatternLayout` from Logback and uses exact same code for formatting events. The key difference is that it
+swaps out some parts of the implementation and introduces reusable thread local buffers for constructing log line and
+allocation free string to bytes encoder.
+
+Tjahzi appender supports all kinds of layouts. To enable `EfficientPatternLayout` us following configuration:
+
+```xml
+<appender name="Loki" class="pl.tkowalcz.tjahzi.logback.LokiAppender">
+    ...
+    <efficientLayout>
+        <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+    </efficientLayout>
+</appender>
+```
+
 ## Advanced configuration
 
 This example sets up a root logger with Loki appender. The appender definition must point to
 class `pl.tkowalcz.tjahzi.logback.LokiAppender`.
 
 ```xml
-
 <configuration>
     <appender name="Loki" class="pl.tkowalcz.tjahzi.logback.LokiAppender">
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>
@@ -88,7 +103,6 @@ class `pl.tkowalcz.tjahzi.logback.LokiAppender`.
         <appender-ref ref="Loki"/>
     </root>
 </configuration>
-
 ``` 
 
 ### Lookups / variable substitution
@@ -107,6 +121,25 @@ Let's go through the example config above and analyze configuration options (**N
 |-----|-------------|
 | Host | Network host address of Loki instance. Either IP address or host name. It will be passed to Netty and end up being resolved by call to `InetSocketAddress.createUnresolved`. |
 | Port | Self-explanatory :) |
+| Encoder | You need to specify pattern for the layout. See section below |
+
+#### Pattern definition
+
+You can define layout and pattern in a traditional Logback way:
+
+```xml
+<encoder>
+    <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+</encoder>
+```
+
+or use Tjahzi optimized efficient, low allocation encoder:
+
+```xml
+<efficientLayout>
+    <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+</efficientLayout>
+```
 
 #### Optional configuration parameters
 

logback-appender/pom.xml

@@ -1,4 +1,5 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
 
@@ -33,6 +34,22 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-core</artifactId>
+            <version>1.2.3</version>
+            <classifier>tests</classifier>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.google.code.java-allocation-instrumenter</groupId>
             <artifactId>java-allocation-instrumenter</artifactId>
@@ -59,6 +76,19 @@
         </dependency>
 
         <!-- Testcontainers -->
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>1.14.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>1.14.3</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
@@ -78,16 +108,37 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>testcontainers</artifactId>
-            <version>1.14.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>1.14.3</version>
+            <groupId>org.junit.vintage</groupId>
+            <artifactId>junit-vintage-engine</artifactId>
+            <version>5.6.2</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
+
+    <profiles>
+        <profile>
+            <id>allocation-profiling</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-surefire-plugin</artifactId>
+                        <version>3.0.0-M5</version>
+                        <configuration>
+                            <argLine>
+                                -javaagent:"${settings.localRepository}/com/google/code/java-allocation-instrumenter/java-allocation-instrumenter/3.3.0/java-allocation-instrumenter-3.3.0.jar"
+                            </argLine>
+                        </configuration>
+                        <dependencies>
+                            <dependency>
+                                <groupId>com.google.code.java-allocation-instrumenter</groupId>
+                                <artifactId>java-allocation-instrumenter</artifactId>
+                                <version>3.3.0</version>
+                            </dependency>
+                        </dependencies>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>

logback-appender/src/main/java/ch/qos/logback/core/pattern/EfficientPatternLayout.java

@@ -0,0 +1,59 @@
+package ch.qos.logback.core.pattern;
+
+import ch.qos.logback.classic.PatternLayout;
+import ch.qos.logback.classic.pattern.EnsureExceptionHandling;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+public class EfficientPatternLayout extends PatternLayoutBase<ILoggingEvent> {
+
+    private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0);
+
+    public EfficientPatternLayout() {
+        this.postCompileProcessor = new EnsureExceptionHandling();
+    }
+
+    public Map<String, String> getDefaultConverterMap() {
+        return PatternLayout.defaultConverterMap;
+    }
+
+    public ByteBuffer doEfficientLayout(ILoggingEvent event) {
+        if (!isStarted()) {
+            return EMPTY_BUFFER;
+        }
+
+        return efficientWriteLoopOnConverters(event);
+    }
+
+    protected ByteBuffer efficientWriteLoopOnConverters(ILoggingEvent event) {
+        StringBuilder strBuilder = StringBuilders.threadLocal();
+        Converter<ILoggingEvent> c = head;
+        while (c != null) {
+            c.write(strBuilder, event);
+            c = c.getNext();
+        }
+
+        Encoder encoder = Encoders.threadLocal();
+        encoder.encode(strBuilder);
+
+        return encoder.getBuffer();
+    }
+
+    @Override
+    protected String getPresentationHeaderPrefix() {
+        return PatternLayout.HEADER_PREFIX;
+    }
+
+    @Override
+    public String doLayout(ILoggingEvent event) {
+        ByteBuffer byteBuffer = doEfficientLayout(event);
+
+        return new String(
+                byteBuffer.array(),
+                byteBuffer.position(),
+                byteBuffer.limit()
+        );
+    }
+}

logback-appender/src/main/java/ch/qos/logback/core/pattern/Encoder.java

@@ -0,0 +1,60 @@
+package ch.qos.logback.core.pattern;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.StandardCharsets;
+
+public class Encoder {
+
+    /**
+     * 10kb should be enough for everyone. Even in case of 1000s of threads we will allocate at most 10s of MB of buffers.
+     * Larger messages will be fragmented.
+     */
+    private static final int ENCODER_BUFFER_SIZE = Integer.getInteger("tjahzi.logback.layout.encoder.bufferSize", 10 * 1024);
+
+    private final ByteBuffer buffer = ByteBuffer.allocate(ENCODER_BUFFER_SIZE);
+
+    private final char[] temporaryTextArray = new char[1024];
+    private final CharBuffer temporaryTextBuffer = CharBuffer.wrap(temporaryTextArray);
+    private final CharsetEncoder charsetEncoder = StandardCharsets.UTF_8.newEncoder();
+
+    public void encode(StringBuilder input) {
+        int offset = 0;
+        int sizeOfInput = input.length();
+        while (sizeOfInput > 0) {
+            int length = Math.min(sizeOfInput, temporaryTextArray.length);
+            sizeOfInput -= length;
+
+            encodeRound(input, offset, length, sizeOfInput == 0);
+            offset += length;
+        }
+
+        buffer.flip();
+    }
+
+    public ByteBuffer getBuffer() {
+        return buffer;
+    }
+
+    public void clear() {
+        buffer.clear();
+        charsetEncoder.reset();
+    }
+
+    private void encodeRound(StringBuilder input, int offset, int length, boolean endOfInput) {
+        input.getChars(offset, offset + length, temporaryTextArray, 0);
+        temporaryTextBuffer.clear().limit(length);
+
+        CoderResult coderResult = charsetEncoder.encode(
+                temporaryTextBuffer,
+                buffer,
+                endOfInput
+        );
+
+        if (coderResult.isError()) {
+            throw new RuntimeException(coderResult.toString());
+        }
+    }
+}

logback-appender/src/main/java/ch/qos/logback/core/pattern/Encoders.java

@@ -0,0 +1,13 @@
+package ch.qos.logback.core.pattern;
+
+public class Encoders {
+
+    private static final ThreadLocal<Encoder> THREAD_LOCAL = ThreadLocal.withInitial(Encoder::new);
+
+    public static Encoder threadLocal() {
+        Encoder result = THREAD_LOCAL.get();
+        result.clear();
+
+        return result;
+    }
+}

logback-appender/src/main/java/ch/qos/logback/core/pattern/StringBuilders.java

@@ -0,0 +1,13 @@
+package ch.qos.logback.core.pattern;
+
+public class StringBuilders {
+
+    private static final ThreadLocal<StringBuilder> THREAD_LOCAL = ThreadLocal.withInitial(StringBuilder::new);
+
+    public static StringBuilder threadLocal() {
+        StringBuilder result = THREAD_LOCAL.get();
+        result.setLength(0);
+
+        return result;
+    }
+}

logback-appender/src/main/java/pl/tkowalcz/tjahzi/logback/LokiAppender.java

@@ -2,6 +2,7 @@ package pl.tkowalcz.tjahzi.logback;
 
 import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
 import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.pattern.EfficientPatternLayout;
 import pl.tkowalcz.tjahzi.LoggingSystem;
 import pl.tkowalcz.tjahzi.TjahziLogger;
 import pl.tkowalcz.tjahzi.stats.MonitoringModule;
@@ -9,17 +10,28 @@ import pl.tkowalcz.tjahzi.stats.MutableMonitoringModuleWrapper;
 
 import java.nio.ByteBuffer;
 import java.util.Collections;
+import java.util.function.Function;
 
 public class LokiAppender extends LokiAppenderConfigurator {
 
-
+    private EfficientPatternLayout efficientLayout;
     private PatternLayoutEncoder encoder;
+
     private LoggingSystem loggingSystem;
+    private Function<ILoggingEvent, ByteBuffer> actualEncoder;
 
     private TjahziLogger logger;
     private String logLevelLabel;
     private MutableMonitoringModuleWrapper monitoringModuleWrapper;
 
+    public EfficientPatternLayout getEfficientLayout() {
+        return efficientLayout;
+    }
+
+    public void setEfficientLayout(EfficientPatternLayout efficientLayout) {
+        this.efficientLayout = efficientLayout;
+    }
+
     public PatternLayoutEncoder getEncoder() {
         return encoder;
     }
@@ -44,25 +56,30 @@ public class LokiAppender extends LokiAppenderConfigurator {
     @Override
     protected void append(ILoggingEvent event) {
         String logLevel = event.getLevel().toString();
-        byte[] logLine = encoder.encode(event);
+        ByteBuffer logLine = actualEncoder.apply(event);
 
         logger.log(
                 event.getTimeStamp(),
                 Collections.emptyMap(),
                 logLevelLabel,
                 logLevel,
-                ByteBuffer.wrap(logLine)
+                logLine
         );
     }
 
     @Override
     public void start() {
         if (encoder == null) {
-            addError("No encoder set for the appender named [" + name + "].");
-            return;
-        }
+            if (efficientLayout == null) {
+                addError("No encoder set for the appender named [" + name + "]. You can also try setting efficientLayout instead.");
+                return;
+            }
 
-        encoder.start();
+            actualEncoder = efficientLayout::doEfficientLayout;
+        } else {
+            encoder.start();
+            actualEncoder = event -> ByteBuffer.wrap(encoder.encode(event));
+        }
 
         LokiAppenderFactory lokiAppenderFactory = new LokiAppenderFactory(this);
         loggingSystem = lokiAppenderFactory.createAppender();

logback-appender/src/test/java/ch/qos/logback/core/pattern/EfficientPatternLayoutTest.java

@@ -0,0 +1,333 @@
+package ch.qos.logback.core.pattern;
+
+import ch.qos.logback.classic.ClassicTestConstants;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.LoggingEvent;
+import ch.qos.logback.classic.testUtil.SampleConverter;
+import ch.qos.logback.core.Context;
+import ch.qos.logback.core.joran.spi.JoranException;
+import ch.qos.logback.core.pattern.parser.AbstractPatternLayoutBaseTest;
+import ch.qos.logback.core.testUtil.StringListAppender;
+import ch.qos.logback.core.util.OptionHelper;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.MDC;
+
+import static ch.qos.logback.classic.ClassicTestConstants.ISO_REGEX;
+import static ch.qos.logback.classic.ClassicTestConstants.MAIN_REGEX;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Adapted from ch.qos.logback.classic.PatternLayoutTest
+ */
+public class EfficientPatternLayoutTest extends AbstractPatternLayoutBaseTest<ILoggingEvent> {
+
+    private EfficientPatternLayout patternLayout;
+
+    private final LoggerContext loggerContext = new LoggerContext();
+    private final Logger logger = loggerContext.getLogger(EfficientPatternLayoutTest.class);
+    private final Logger root = loggerContext.getLogger(Logger.ROOT_LOGGER_NAME);
+
+    @Before
+    public void setUp() {
+        patternLayout = getPatternLayoutBase();
+        patternLayout.setContext(loggerContext);
+    }
+
+    @Override
+    public EfficientPatternLayout getPatternLayoutBase() {
+        return new EfficientPatternLayout();
+    }
+
+    @Override
+    public ILoggingEvent getEventObject() {
+        return new LoggingEvent(
+                ch.qos.logback.core.pattern.FormattingConverter.class.getName(),
+                logger,
+                Level.INFO,
+                "Some message",
+                null,
+                null
+        );
+    }
+
+    @Override
+    public Context getContext() {
+        return loggerContext;
+    }
+
+    @Test
+    public void testOK() {
+        // Given
+        String pattern = "%d %le [%t] %lo{30} - %m%n";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        // 2021-02-01 22:38:06,212 INFO [main] c.q.l.pattern.EfficientPatternLayoutTest - Some message
+        String regex = ISO_REGEX + " INFO " + MAIN_REGEX + " c.q.l.c.p.EfficientPatternLayoutTest - Some message\\s*";
+        assertThat(val).matches(regex);
+    }
+
+    @Test
+    public void testNoExeptionHandler() {
+        // Given
+        String pattern = "%m%n";
+        ILoggingEvent loggingEvent = new LoggingEvent(
+                ch.qos.logback.core.pattern.FormattingConverter.class.getName(),
+                logger,
+                Level.INFO,
+                "Some message",
+                new Exception("Bogus exception"),
+                null
+        );
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(loggingEvent);
+
+        // Then
+        assertThat(val).contains("java.lang.Exception: Bogus exception");
+    }
+
+    @Test
+    public void testCompositePattern() {
+        // Given
+        String pattern = "%-69(%d %lo{20}) - %m%n";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        //                                                          / Ten spaces \
+        // 2021-03-18 21:55:54,250 c.q.l.c.p.EfficientPatternLayoutTest         - Some message
+        String regex = ISO_REGEX + " c.q.l.c.p.EfficientPatternLayoutTest {10}- Some message\\s*";
+        assertThat(val).matches(regex);
+    }
+
+    @Test
+    public void contextProperty() {
+        // Given
+        String pattern = "%property{a}";
+        loggerContext.putProperty("a", "b");
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        assertThat(val).isEqualTo("b");
+    }
+
+    @Test
+    public void testNopExceptionHandler() {
+        // Given
+        ILoggingEvent loggingEvent = new LoggingEvent(
+                ch.qos.logback.core.pattern.FormattingConverter.class.getName(),
+                logger,
+                Level.INFO,
+                "Some message",
+                new Exception("Bogus exception"),
+                null
+        );
+
+        // When
+        patternLayout.setPattern("%nopex %m%n");
+        patternLayout.start();
+        String val = patternLayout.doLayout(loggingEvent);
+
+        // Then
+        assertThat(val).doesNotContain("java.lang.Exception: Bogus exception");
+    }
+
+    @Test
+    public void testWithParenthesis() {
+        // Given
+        ILoggingEvent loggingEvent = new LoggingEvent(
+                ch.qos.logback.core.pattern.FormattingConverter.class.getName(),
+                logger,
+                Level.INFO,
+                "Some message",
+                null,
+                null
+        );
+
+        // When
+        patternLayout.setPattern("\\(%msg:%msg\\) %msg");
+        patternLayout.start();
+        String val = patternLayout.doLayout(loggingEvent);
+
+        // Then
+        assertThat(val).isEqualTo("(Some message:Some message) Some message");
+    }
+
+    @Test
+    public void testWithLettersComingFromLog4j() {
+        // Given
+        // Letters: p = level and c = logger
+        String pattern = "%d %p [%t] %c{30} - %m%n";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        // 2021-02-01 22:38:06,212 INFO [main] c.q.l.pattern.EfficientPatternLayoutTest - Some message
+        String regex = ClassicTestConstants.ISO_REGEX + " INFO " + MAIN_REGEX + " c.q.l.c.p.EfficientPatternLayoutTest - Some message\\s*";
+        assertThat(val).matches(regex);
+    }
+
+    @Test
+    public void mdcWithDefaultValue() {
+        // Given
+        String pattern = "%msg %mdc{foo} %mdc{bar:-[null]}";
+        MDC.put("foo", "foo");
+
+        // When
+        patternLayout.setPattern(OptionHelper.substVars(pattern, loggerContext));
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        assertThat(val).isEqualTo("Some message foo [null]");
+    }
+
+    @Test
+    public void contextNameTest() {
+        // Given
+        String pattern = "%contextName";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+
+        loggerContext.setName("aValue");
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        assertThat(val).isEqualTo("aValue");
+    }
+
+    @Test
+    public void cnTest() {
+        // Given
+        String pattern = "%cn";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+
+        loggerContext.setName("aValue");
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        assertEquals("aValue", val);
+    }
+
+    @Test
+    public void testConversionRuleSupportInPatternLayout() throws JoranException {
+        // Given
+        String configurationFile = "src/test/resources/joran/conversionRule/patternLayout0.xml";
+        String msg = "Simon says";
+
+        // When
+        configure(configurationFile);
+        logger.debug(msg);
+
+        // Then
+        StringListAppender<ILoggingEvent> sla = (StringListAppender<ILoggingEvent>) root.getAppender("LIST");
+        assertThat(sla).isNotNull();
+        assertThat(sla.strList).containsExactly(SampleConverter.SAMPLE_STR + " - " + msg);
+    }
+
+    @Test
+    public void smokeReplace() {
+        // Given
+        String pattern = "%replace(a1234b){'\\d{4}', 'XXXX'}";
+
+        // When
+        patternLayout.setPattern(pattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(getEventObject());
+
+        // Then
+        assertThat(val).isEqualTo("aXXXXb");
+    }
+
+    @Test
+    public void replaceNewline() {
+        // Given
+        String pattern = "%replace(A\nB){'\n', '\n\t'}";
+        ILoggingEvent loggingEvent = new LoggingEvent(
+                ch.qos.logback.core.pattern.FormattingConverter.class.getName(),
+                logger,
+                Level.INFO,
+                "",
+                null,
+                null
+        );
+
+        String substPattern = OptionHelper.substVars(pattern, null, loggerContext);
+        assertEquals(pattern, substPattern);
+
+        // When
+        patternLayout.setPattern(substPattern);
+        patternLayout.start();
+        String val = patternLayout.doLayout(loggingEvent);
+
+        // Then
+        assertThat(val).isEqualTo("A\n\tB");
+    }
+
+    @Test
+    public void replaceWithJoran() throws JoranException {
+        // Given
+        String configurationFile = "src/test/resources/joran/pattern/replace0.xml";
+        String msg = "And the number is 4111111111110000, expiring on 12/2010";
+
+        // When
+        configure(configurationFile);
+        logger.debug(msg);
+
+        // Then
+        StringListAppender<ILoggingEvent> sla = (StringListAppender<ILoggingEvent>) root.getAppender("LIST");
+        assertThat(sla).isNotNull();
+        assertThat(sla.strList).containsExactly("And the number is XXXX, expiring on 12/2010");
+    }
+
+    @Test
+    public void replaceWithJoran_NEWLINE() throws JoranException {
+        // Given
+        String configurationFile = "src/test/resources/joran/pattern/replaceNewline.xml";
+        String msg = "A\nC";
+
+        // When
+        loggerContext.putProperty("TAB", "\t");
+        configure(configurationFile);
+        logger.debug(msg);
+
+        // Then
+        StringListAppender<ILoggingEvent> sla = (StringListAppender<ILoggingEvent>) root.getAppender("LIST");
+        assertThat(sla).isNotNull();
+        assertThat(sla.strList).containsExactly("A\n\tC");
+    }
+
+    private void configure(String file) throws JoranException {
+        JoranConfigurator jc = new JoranConfigurator();
+        jc.setContext(loggerContext);
+        jc.doConfigure(file);
+    }
+}

logback-appender/src/test/java/ch/qos/logback/core/pattern/EncoderTest.java

@@ -0,0 +1,102 @@
+package ch.qos.logback.core.pattern;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+import static java.nio.ByteBuffer.wrap;
+import static java.util.Arrays.copyOf;
+import static org.assertj.core.api.Assertions.assertThat;
+
+class EncoderTest {
+
+    @Test
+    void shouldHandleEmptyInput() {
+        // Given
+        Encoder encoder = Encoders.threadLocal();
+        String expected = "ABC";
+
+        // When
+        encoder.encode(new StringBuilder(expected));
+
+        // Then
+        ByteBuffer actual = encoder.getBuffer();
+        ByteBuffer actual1 = wrap(expected.getBytes(StandardCharsets.UTF_8));
+
+        assertThat(actual).isEqualTo(actual1);
+    }
+
+    @Test
+    void shouldEncode() {
+        // Given
+        Encoder encoder = Encoders.threadLocal();
+        String expected = "ABC";
+
+        // When
+        encoder.encode(new StringBuilder(expected));
+
+        // Then
+        ByteBuffer actual = encoder.getBuffer();
+        ByteBuffer actual1 = wrap(expected.getBytes(StandardCharsets.UTF_8));
+
+        assertThat(actual).isEqualTo(actual1);
+    }
+
+    @Test
+    void shouldClearInternalBuffer() {
+        // Given
+        Encoder encoder = new Encoder();
+
+        StringBuilder firstText = new StringBuilder("first");
+        StringBuilder secondText = new StringBuilder("second");
+
+        encoder.encode(firstText);
+        assertThat(encoder.getBuffer()).isEqualTo(wrap(firstText.toString().getBytes()));
+
+        // When
+        encoder.clear();
+
+        // Then
+        assertThat(encoder.getBuffer().position()).isZero();
+
+        encoder.encode(secondText);
+        assertThat(encoder.getBuffer()).isEqualTo(wrap(secondText.toString().getBytes()));
+    }
+
+    @Test
+    void shouldEncodeTextLargerThanTempArray() {
+        // Given
+        Encoder encoder = new Encoder();
+
+        String toEncode = RandomStringUtils.randomAlphanumeric(4 * 1024 + 41);
+        ByteBuffer expected = wrap(toEncode.getBytes());
+
+        // When
+        encoder.encode(new StringBuilder(toEncode));
+
+        // Then
+        assertThat(encoder.getBuffer()).isEqualTo(expected);
+    }
+
+    @Test
+    void shouldTruncateLongerMessages() {
+        // Given
+        Encoder encoder = new Encoder();
+
+        String toEncode = RandomStringUtils.randomAlphanumeric(40 * 1024 + 41);
+        ByteBuffer expected = wrap(
+                copyOf(
+                        toEncode.getBytes(),
+                        10 * 1024
+                )
+        );
+
+        // When
+        encoder.encode(new StringBuilder(toEncode));
+
+        // Then
+        assertThat(encoder.getBuffer()).isEqualTo(expected);
+    }
+}

logback-appender/src/test/java/ch/qos/logback/core/pattern/EncodersTest.java

@@ -0,0 +1,55 @@
+package ch.qos.logback.core.pattern;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class EncodersTest {
+
+    @Test
+    void shouldClearBufferBeforeUse() {
+        // Given
+        Encoder encoder = Encoders.threadLocal();
+        encoder.encode(new StringBuilder("abc"));
+
+        // When
+        Encoder actual = Encoders.threadLocal();
+
+        // Then
+        assertThat(actual.getBuffer().position()).isZero();
+    }
+
+    @Test
+    void shouldReturnSameBufferInsideSameThread() {
+        // When
+        Encoder encoder1 = Encoders.threadLocal();
+        Encoder encoder2 = Encoders.threadLocal();
+
+        // Then
+        assertThat(encoder1).isSameAs(encoder2);
+    }
+
+    @Test
+    void shouldReturnDifferentBufferInsideDifferentThreads() throws InterruptedException {
+        // Given
+        AtomicReference<Encoder> captor1 = new AtomicReference<>();
+        AtomicReference<Encoder> captor2 = new AtomicReference<>();
+
+        // When
+        Thread thread1 = new Thread(() -> captor1.set(Encoders.threadLocal()));
+        thread1.start();
+        thread1.join();
+
+        Thread thread2 = new Thread(() -> captor2.set(Encoders.threadLocal()));
+        thread2.start();
+        thread2.join();
+
+        // Then
+        assertThat(captor1.get()).isNotNull();
+        assertThat(captor2.get()).isNotNull();
+
+        assertThat(captor1.get()).isNotSameAs(captor2.get());
+    }
+}

logback-appender/src/test/java/ch/qos/logback/core/pattern/StringBuildersTest.java

@@ -0,0 +1,55 @@
+package ch.qos.logback.core.pattern;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class StringBuildersTest {
+
+    @Test
+    void shouldClearBufferBeforeUse() {
+        // Given
+        StringBuilder stringBuilder = StringBuilders.threadLocal();
+        stringBuilder.append("abc");
+
+        // When
+        StringBuilder actual = StringBuilders.threadLocal();
+
+        // Then
+        assertThat(actual).isEmpty();
+    }
+
+    @Test
+    void shouldReturnSameBufferInsideSameThread() {
+        // When
+        StringBuilder stringBuilder1 = StringBuilders.threadLocal();
+        StringBuilder stringBuilder2 = StringBuilders.threadLocal();
+
+        // Then
+        assertThat(stringBuilder1).isSameAs(stringBuilder2);
+    }
+
+    @Test
+    void shouldReturnDifferentBufferInsideDifferentThreads() throws InterruptedException {
+        // Given
+        AtomicReference<StringBuilder> captor1 = new AtomicReference<>();
+        AtomicReference<StringBuilder> captor2 = new AtomicReference<>();
+
+        // When
+        Thread thread1 = new Thread(() -> captor1.set(StringBuilders.threadLocal()));
+        thread1.start();
+        thread1.join();
+
+        Thread thread2 = new Thread(() -> captor2.set(StringBuilders.threadLocal()));
+        thread2.start();
+        thread2.join();
+
+        // Then
+        assertThat(captor1.get()).isNotNull();
+        assertThat(captor2.get()).isNotNull();
+
+        assertThat(captor1.get()).isNotSameAs(captor2.get());
+    }
+}

logback-appender/src/test/java/pl/tkowalcz/tjahzi/logback/AllocationTest.java

@@ -0,0 +1,119 @@
+package pl.tkowalcz.tjahzi.logback;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import com.google.monitoring.runtime.instrumentation.AllocationRecorder;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.shaded.com.google.common.util.concurrent.Uninterruptibles;
+
+import java.net.URI;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Testcontainers
+public class AllocationTest {
+
+    @Container
+    public GenericContainer loki = new GenericContainer("grafana/loki:latest")
+            .withCommand("-config.file=/etc/loki-config.yaml")
+            .withClasspathResourceMapping("loki-config.yaml",
+                    "/etc/loki-config.yaml",
+                    BindMode.READ_ONLY)
+            .waitingFor(
+                    Wait.forHttp("/ready")
+                            .forPort(3100)
+            )
+            .withExposedPorts(3100);
+
+    @Test
+    @Disabled
+    void shouldSendData() throws Exception {
+        AtomicLong totalAllocatedMemory = new AtomicLong();
+        Map<String, AtomicLong> allocatedMemory = new HashMap<>();
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            allocatedMemory
+                    .entrySet()
+                    .stream()
+                    .sorted(Comparator.<Map.Entry<String, AtomicLong>>comparingLong(e -> e.getValue().get()).reversed())
+                    .limit(30)
+                    .forEach(System.out::println);
+
+            System.out.println("totalAllocatedMemory = " + totalAllocatedMemory);
+        }));
+
+        // Given
+        System.setProperty("loki.host", loki.getHost());
+        System.setProperty("loki.port", loki.getFirstMappedPort().toString());
+
+        URI uri = getClass()
+                .getClassLoader()
+                .getResource("basic-appender-test-configuration.xml")
+                .toURI();
+
+        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+
+        JoranConfigurator configurator = new JoranConfigurator();
+        configurator.setContext(context);
+
+        context.reset();
+        configurator.doConfigure(uri.toURL());
+
+        String logLine = "Cupcake ipsum dolor sit amet cake wafer. " +
+                "Souffle jelly beans biscuit topping. " +
+                "Danish bonbon gummies powder caramels. ";
+
+        Logger logger = context.getLogger(AllocationTest.class);
+
+        // When
+        System.out.println("Warmup...");
+        for (int i = 0; i < 2000; i++) {
+            logger.info(logLine);
+            Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+        }
+
+        AllocationRecorder.addSampler((count, desc, newObj, size) -> {
+            if (desc.contains("AllocationTest")) {
+                return;
+            }
+
+            if (desc.contains("instrumentation")) {
+                return;
+            }
+
+            if (desc.contains("asm")) {
+                return;
+            }
+
+            if (desc.contains("Object")) {
+                return;
+            }
+
+            if(desc.contains("LoggingEvent")) {
+                return;
+            }
+
+            allocatedMemory.computeIfAbsent(desc, __ -> new AtomicLong()).addAndGet(size);
+            totalAllocatedMemory.addAndGet(size);
+        });
+
+        System.out.println("Test...");
+        for (int i = 0; i < 1000; i++) {
+            logger.info(logLine);
+            Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
+        }
+
+        // Then
+
+    }
+}

logback-appender/src/test/resources/appender-test-large-batches.xml

@@ -8,9 +8,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/appender-test-with-buffer-size-megabytes-set-to-64.xml

@@ -6,9 +6,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/appender-test-with-buffer-size-megabytes-unset.xml

@@ -3,9 +3,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/appender-test-with-log-label-set.xml

@@ -3,9 +3,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/appender-test-with-log-label-unset.xml

@@ -3,9 +3,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/basic-appender-test-configuration.xml

@@ -3,9 +3,9 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
+        <efficientLayout>
             <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        </efficientLayout>
 
         <header>
             <name>X-Org-Id</name>

logback-appender/src/test/resources/joran/conversionRule/patternLayout0.xml

@@ -0,0 +1,30 @@
+<!--
+/**
+ * Logback: the reliable, generic, fast and flexible logging framework.
+ * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
+ *
+ * This program and the accompanying materials are dual-licensed under
+ * either the terms of the Eclipse Public License v1.0 as published by
+ * the Eclipse Foundation
+ *
+ *   or (per the licensee's choosing)
+ *
+ * under the terms of the GNU Lesser General Public License version 2.1
+ * as published by the Free Software Foundation.
+ */
+-->
+<configuration>
+
+    <conversionRule conversionWord="sample"
+                    converterClass="ch.qos.logback.classic.testUtil.SampleConverter"/>
+
+    <appender name="LIST" class="ch.qos.logback.core.testUtil.StringListAppender">
+        <layout class="ch.qos.logback.core.pattern.EfficientPatternLayout">
+            <Pattern>%sample - %msg</Pattern>
+        </layout>
+    </appender>
+
+    <root level="debug">
+        <appender-ref ref="LIST"/>
+    </root>
+</configuration>

logback-appender/src/test/resources/joran/pattern/replace0.xml

@@ -0,0 +1,31 @@
+<!--
+/**
+ * Logback: the reliable, generic, fast and flexible logging framework.
+ * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
+ *
+ * This program and the accompanying materials are dual-licensed under
+ * either the terms of the Eclipse Public License v1.0 as published by
+ * the Eclipse Foundation
+ *
+ *   or (per the licensee's choosing)
+ *
+ * under the terms of the GNU Lesser General Public License version 2.1
+ * as published by the Free Software Foundation.
+ */
+-->
+<configuration>
+
+    <conversionRule conversionWord="sample"
+                    converterClass="ch.qos.logback.classic.testUtil.SampleConverter"/>
+
+    <appender name="LIST" class="ch.qos.logback.core.testUtil.StringListAppender">
+        <layout class="ch.qos.logback.core.pattern.EfficientPatternLayout">
+            <Pattern>%replace(%msg){'\d{14,16}', 'XXXX'}</Pattern>
+        </layout>
+    </appender>
+
+    <root level="debug">
+        <appender-ref ref="LIST"/>
+    </root>
+</configuration>
+

logback-appender/src/test/resources/joran/pattern/replaceNewline.xml

@@ -0,0 +1,30 @@
+<!--
+/**
+ * Logback: the reliable, generic, fast and flexible logging framework.
+ * Copyright (C) 1999-2015, QOS.ch. All rights reserved.
+ *
+ * This program and the accompanying materials are dual-licensed under
+ * either the terms of the Eclipse Public License v1.0 as published by
+ * the Eclipse Foundation
+ *
+ *   or (per the licensee's choosing)
+ *
+ * under the terms of the GNU Lesser General Public License version 2.1
+ * as published by the Free Software Foundation.
+ */
+-->
+<configuration>
+
+    <conversionRule conversionWord="sample"
+                    converterClass="ch.qos.logback.classic.testUtil.SampleConverter"/>
+
+    <appender name="LIST" class="ch.qos.logback.core.testUtil.StringListAppender">
+        <layout class="ch.qos.logback.core.pattern.EfficientPatternLayout">
+            <Pattern>%replace(%msg){'(\n)', '$1${TAB}'}</Pattern>
+        </layout>
+    </appender>
+
+    <root level="debug">
+        <appender-ref ref="LIST"/>
+    </root>
+</configuration>

logback-appender/src/test/resources/logback-configuration.xml

@@ -3,9 +3,7 @@
         <host>${loki.host}</host>
         <port>${loki.port}</port>
 
-        <encoder>
-            <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
-        </encoder>
+        <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
 
         <header>
             <name>X-Org-Id</name>