From 989f6014c62127fe62b6c77ef691b0443a4b64d9 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 00:07:40 +0100 Subject: [PATCH 01/16] Working to make CircleCI trigger on tag. --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 419abf1..9f97caa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,9 @@ workflows: main: jobs: - build-and-test + filters: + tags: + only: /.*/ - circleci-maven-release-orb/run-maven-release: filters: branches: From 110c6f922dfb3465835b895911c857713a12aa2e Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 00:08:44 +0100 Subject: [PATCH 02/16] Of course there was an error in yaml... --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f97caa..2bf1797 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ workflows: version: 2 main: jobs: - - build-and-test + - build-and-test: filters: tags: only: /.*/ From d023fb45dbced77e9006f49afc2c94da7ed310a4 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 00:37:39 +0100 Subject: [PATCH 03/16] Never version of maven release orb. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bf1797..568cec0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - circleci-maven-release-orb: sonatype-nexus-community/circleci-maven-release-orb@0.0.15 + circleci-maven-release-orb: sonatype-nexus-community/circleci-maven-release-orb@0.0.16 jobs: build-and-test: From bad0d0d5d1913364286e80b9d303f6463b2eb992 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 08:46:32 +0100 Subject: [PATCH 04/16] Trying to make circleci checkout repo without detached HEAD. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 568cec0..127bbd1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ workflows: - circleci-maven-release-orb/run-maven-release: filters: branches: - ignore: /.*/ + only: /master/ tags: only: /.*/ requires: From bf1ebd6c76488924d3503994380a9eff59020641 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 07:53:49 +0000 Subject: [PATCH 05/16] [skip ci] [maven-release-plugin] prepare release tjahzi-0.7 --- loki-log4j2-appender/pom.xml | 5 ++--- pom.xml | 4 ++-- tjahzi-core/pom.xml | 5 ++--- tjahzi-protobuf/pom.xml | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index 92a91ad..3f58d45 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -1,12 +1,11 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-log4j2-appender diff --git a/pom.xml b/pom.xml index fbd50e6..2921519 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.7-SNAPSHOT + 0.7 Tjahzi Tjahzi is a low overhead logging client for Loki @@ -20,7 +20,7 @@ scm:git:git@github.com:tkowalcz/tjahzi.git https://github.com/tkowalcz/tjahzi - HEAD + tjahzi-0.7 diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index 879a39e..b6fce59 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -1,12 +1,11 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 4260bf4..8d7f68a 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-loki-protobuf From ee2bacaaf2bf28c44fd2d01d2ef8ccfcffb749bb Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 22:32:05 +0100 Subject: [PATCH 06/16] Changing how releases are performed. --- .circleci/config.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 127bbd1..3fc2710 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,20 +46,21 @@ executors: machine: image: ubuntu-2004:202008-01 +parameters: + run_workflow_release: + default: false + type: boolean + workflows: version: 2 main: jobs: - - build-and-test: - filters: - tags: - only: /.*/ + - build-and-test + release: + when: << pipeline.parameters.run_workflow_release >> + jobs: + - build-and-test - circleci-maven-release-orb/run-maven-release: - filters: - branches: - only: /master/ - tags: - only: /.*/ requires: - build-and-test executor: new-java-maven-executor From d37c0987587f16b92280c1d35c34549f80445848 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Tue, 26 Jan 2021 22:33:55 +0100 Subject: [PATCH 07/16] Revert "[skip ci] [maven-release-plugin] prepare release tjahzi-0.7" This reverts commit bf1ebd6c --- loki-log4j2-appender/pom.xml | 5 +++-- tjahzi-core/pom.xml | 5 +++-- tjahzi-protobuf/pom.xml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index 3f58d45..92a91ad 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -1,11 +1,12 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7 + 0.7-SNAPSHOT tjahzi-log4j2-appender diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index b6fce59..879a39e 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -1,11 +1,12 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7 + 0.7-SNAPSHOT tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 8d7f68a..4260bf4 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7 + 0.7-SNAPSHOT tjahzi-loki-protobuf From f5b61bed1494580d2575be067efd79b25af96f66 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Wed, 27 Jan 2021 23:19:51 +0100 Subject: [PATCH 08/16] Revert "[skip ci] [maven-release-plugin] prepare release tjahzi-0.7" This reverts commit bf1ebd6c --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2921519..fbd50e6 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.7 + 0.7-SNAPSHOT Tjahzi Tjahzi is a low overhead logging client for Loki @@ -20,7 +20,7 @@ scm:git:git@github.com:tkowalcz/tjahzi.git https://github.com/tkowalcz/tjahzi - tjahzi-0.7 + HEAD From e63a7f3e155b9d9819a729f90a4ec894d4d000ac Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Wed, 27 Jan 2021 22:31:56 +0000 Subject: [PATCH 09/16] [skip ci] [maven-release-plugin] prepare release tjahzi-0.7 --- loki-log4j2-appender/pom.xml | 5 ++--- pom.xml | 4 ++-- tjahzi-core/pom.xml | 5 ++--- tjahzi-protobuf/pom.xml | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index 92a91ad..3f58d45 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -1,12 +1,11 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-log4j2-appender diff --git a/pom.xml b/pom.xml index fbd50e6..2921519 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.7-SNAPSHOT + 0.7 Tjahzi Tjahzi is a low overhead logging client for Loki @@ -20,7 +20,7 @@ scm:git:git@github.com:tkowalcz/tjahzi.git https://github.com/tkowalcz/tjahzi - HEAD + tjahzi-0.7 diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index 879a39e..b6fce59 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -1,12 +1,11 @@ - + 4.0.0 pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 4260bf4..8d7f68a 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7-SNAPSHOT + 0.7 tjahzi-loki-protobuf From 6083bc1627798920ba1fbca09340d6f93f36a039 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Thu, 28 Jan 2021 00:53:15 +0100 Subject: [PATCH 10/16] Manually preparing for 0.8 release. --- loki-log4j2-appender/pom.xml | 2 +- pom.xml | 2 +- tjahzi-core/pom.xml | 2 +- tjahzi-protobuf/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index 3f58d45..2bed228 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7 + 0.8-SNAPSHOT tjahzi-log4j2-appender diff --git a/pom.xml b/pom.xml index 2921519..c374d41 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.7 + 0.8-SNAPSHOT Tjahzi Tjahzi is a low overhead logging client for Loki diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index b6fce59..d0950b0 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7 + 0.8-SNAPSHOT tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 8d7f68a..1ba49a1 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.7 + 0.8-SNAPSHOT tjahzi-loki-protobuf From 30955907464e9e78006d15f956b9e1cebfe204aa Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Thu, 28 Jan 2021 00:05:33 +0000 Subject: [PATCH 11/16] [skip ci] [maven-release-plugin] prepare release tjahzi-0.8 --- loki-log4j2-appender/pom.xml | 2 +- pom.xml | 4 ++-- tjahzi-core/pom.xml | 2 +- tjahzi-protobuf/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index 2bed228..ef51645 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8-SNAPSHOT + 0.8 tjahzi-log4j2-appender diff --git a/pom.xml b/pom.xml index c374d41..35e8878 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.8-SNAPSHOT + 0.8 Tjahzi Tjahzi is a low overhead logging client for Loki @@ -20,7 +20,7 @@ scm:git:git@github.com:tkowalcz/tjahzi.git https://github.com/tkowalcz/tjahzi - tjahzi-0.7 + tjahzi-0.8 diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index d0950b0..802170d 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8-SNAPSHOT + 0.8 tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 1ba49a1..45bf0dc 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8-SNAPSHOT + 0.8 tjahzi-loki-protobuf From 7b8bf0d298079ba6b26ad9d821197e48b9316540 Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Thu, 28 Jan 2021 00:05:40 +0000 Subject: [PATCH 12/16] [skip ci] [maven-release-plugin] prepare for next development iteration --- loki-log4j2-appender/pom.xml | 2 +- pom.xml | 4 ++-- tjahzi-core/pom.xml | 2 +- tjahzi-protobuf/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/loki-log4j2-appender/pom.xml b/loki-log4j2-appender/pom.xml index ef51645..8c473c0 100644 --- a/loki-log4j2-appender/pom.xml +++ b/loki-log4j2-appender/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8 + 0.9-SNAPSHOT tjahzi-log4j2-appender diff --git a/pom.xml b/pom.xml index 35e8878..9d209fd 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi pom - 0.8 + 0.9-SNAPSHOT Tjahzi Tjahzi is a low overhead logging client for Loki @@ -20,7 +20,7 @@ scm:git:git@github.com:tkowalcz/tjahzi.git https://github.com/tkowalcz/tjahzi - tjahzi-0.8 + tjahzi-0.7 diff --git a/tjahzi-core/pom.xml b/tjahzi-core/pom.xml index 802170d..377831c 100644 --- a/tjahzi-core/pom.xml +++ b/tjahzi-core/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8 + 0.9-SNAPSHOT tjahzi-core diff --git a/tjahzi-protobuf/pom.xml b/tjahzi-protobuf/pom.xml index 45bf0dc..01198cd 100644 --- a/tjahzi-protobuf/pom.xml +++ b/tjahzi-protobuf/pom.xml @@ -5,7 +5,7 @@ pl.tkowalcz tjahzi - 0.8 + 0.9-SNAPSHOT tjahzi-loki-protobuf From 0539d70240809bb419f732087515c38222ec9e4c Mon Sep 17 00:00:00 2001 From: "tomasz.kowalczewski" Date: Sun, 31 Jan 2021 11:47:21 +0100 Subject: [PATCH 13/16] Re #26: Fixed bug caused by Snappy class from Netty being able to compress max 32kb of data. Now it is called multiple times but since it appends uncompressed data size at the beginning I had to copy the class contents and remove that (we only need it once). Also streamlined dependency on netty-all. --- .../LokiAppenderLargeBatchesTestTest.java | 124 +++++++++ .../resources/appender-test-large-batches.xml | 23 ++ .../src/test/resources/loki-config.yaml | 1 + tjahzi-core/pom.xml | 12 +- .../pl/tkowalcz/tjahzi/LogBufferAgent.java | 19 +- .../tkowalcz/tjahzi/http/NettyHttpClient.java | 33 ++- .../java/pl/tkowalcz/tjahzi/http/Snappy.java | 256 ++++++++++++++++++ .../tkowalcz/tjahzi/LogBufferAgentTest.java | 5 +- .../pl/tkowalcz/tjahzi/ReconnectTest.java | 4 +- .../pl/tkowalcz/tjahzi/http/SnappyTest.java | 44 +++ 10 files changed, 496 insertions(+), 25 deletions(-) create mode 100644 loki-log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java create mode 100644 loki-log4j2-appender/src/test/resources/appender-test-large-batches.xml create mode 100644 tjahzi-core/src/main/java/pl/tkowalcz/tjahzi/http/Snappy.java create mode 100644 tjahzi-core/src/test/java/pl/tkowalcz/tjahzi/http/SnappyTest.java diff --git a/loki-log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java b/loki-log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java new file mode 100644 index 0000000..3393f5f --- /dev/null +++ b/loki-log4j2-appender/src/test/java/pl/tkowalcz/tjahzi/log4j2/LokiAppenderLargeBatchesTestTest.java @@ -0,0 +1,124 @@ +package pl.tkowalcz.tjahzi.log4j2; + +import io.restassured.RestAssured; +import io.restassured.http.ContentType; +import io.restassured.parsing.Parser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.awaitility.Durations; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItems; + +@Testcontainers +class LokiAppenderLargeBatchesTestTest { + + @Container + public GenericContainer loki = new GenericContainer("grafana/loki:latest") + .withCommand("-config.file=/etc/loki-config.yaml") + .withClasspathResourceMapping("loki-config.yaml", + "/etc/loki-config.yaml", + BindMode.READ_ONLY) + .waitingFor( + Wait.forHttp("/ready") + .forPort(3100) + ) + .withExposedPorts(3100); + + @Test + void shouldSendData() throws URISyntaxException { + // Given + System.setProperty("loki.host", loki.getHost()); + System.setProperty("loki.port", loki.getFirstMappedPort().toString()); + + URI uri = getClass() + .getClassLoader() + .getResource("appender-test-large-batches.xml") + .toURI(); + + ((org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false)) + .setConfigLocation(uri); + + String expectedLogLine = "Cupcake ipsum dolor sit amet cake wafer. " + + "Souffle jelly beans biscuit topping. " + + "Danish bonbon gummies powder caramels. " + + "Danish jelly beans sweet roll topping jelly beans oat cake toffee. " + + "Chocolate cake sesame snaps brownie biscuit cheesecake. " + + "Ice cream dessert sweet donut marshmallow. " + + "Muffin bear claw cookie jelly-o sugar plum jelly beans apple pie fruitcake cookie. " + + "Tootsie roll carrot cake pastry jujubes jelly beans chupa chups. " + + "Souffle cake muffin liquorice tart souffle pie sesame snaps."; + + long expectedTimestamp = System.currentTimeMillis(); + Logger logger = LogManager.getLogger(LokiAppenderLargeBatchesTestTest.class); + + // When + for (int i = 0; i < 1000; i++) { + logger.info(i + " " + expectedLogLine); + } + + // Then + RestAssured.port = loki.getFirstMappedPort(); + RestAssured.baseURI = "http://" + loki.getHost(); + RestAssured.registerParser("text/plain", Parser.JSON); + + Awaitility + .await() + .atMost(Durations.ONE_MINUTE) + .pollInterval(Durations.ONE_SECOND) + .ignoreExceptions() + .untilAsserted(() -> { + given() + .contentType(ContentType.URLENC) + .urlEncodingEnabled(false) + .formParam("&start=" + expectedTimestamp + "&limit=1000&query=%7Bserver%3D%22127.0.0.1%22%7D") + .when() + .get("/loki/api/v1/query_range") + .then() + .log() + .all() + .statusCode(200) + .body("status", equalTo("success")) + .body("data.result.size()", equalTo(1)) + .body("data.result[0].stream.server", equalTo("127.0.0.1")) + .body("data.result[0].values.size()", equalTo(1000)) + .body("data.result[0].values", hasItems(new BaseMatcher<>() { + @Override + public boolean matches(Object o) { + List list = (List) o; + if (list.size() != 2) { + return false; + } + + long actualTimestamp = Long.parseLong(list.get(0).toString()); + String actualLogLine = list.get(1).toString(); + + return actualLogLine.contains(expectedLogLine) + && (expectedTimestamp - actualTimestamp) < TimeUnit.MINUTES.toMillis(1); + } + + @Override + public void describeTo(Description description) { + + } + } + + )); + }); + } +} diff --git a/loki-log4j2-appender/src/test/resources/appender-test-large-batches.xml b/loki-log4j2-appender/src/test/resources/appender-test-large-batches.xml new file mode 100644 index 0000000..79aa5a9 --- /dev/null +++ b/loki-log4j2-appender/src/test/resources/appender-test-large-batches.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + ${sys:loki.host} + ${sys:loki.port} + + + + %X{tid} [%t] %d{MM-dd HH:mm:ss.SSS} %5p %c{1} - %m%n%exception{full} + + +
+