diff --git a/.gitignore b/.gitignore index 3aad54886..25ce0649f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ build/ rpm/ rpmbuild/ *.sh +# ignore benchmark outputs +io.aiven.kafka.tieredstorage*/ diff --git a/Makefile b/Makefile index 3dcce492a..c0cc76a4a 100644 --- a/Makefile +++ b/Makefile @@ -63,3 +63,8 @@ docker_image: build .PHONY: docker_push docker_push: docker push $(IMAGE_TAG) + +# Prepare kernel to capture CPU events +async_profiler_cpu_kernel-prep: + sudo sh -c 'echo 1 >/proc/sys/kernel/perf_event_paranoid' + sudo sh -c 'echo 0 >/proc/sys/kernel/kptr_restrict' diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..19e38b70d --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,121 @@ +### Benchmarks module + +> Borrowed from https://github.com/apache/kafka/blob/trunk/jmh-benchmarks + +This module contains benchmarks written using [JMH](https://openjdk.java.net/projects/code-tools/jmh/) from OpenJDK. + +### Running benchmarks + +If you want to set specific JMH flags or only run certain benchmarks, passing arguments via +gradle tasks is cumbersome. These are simplified by the provided `jmh.sh` script. + +The default behavior is to run all benchmarks: + + ./benchmarks/jmh.sh + +Pass a pattern or name after the command to select the benchmarks: + + ./benchmarks/jmh.sh TransformBench + +Check which benchmarks that match the provided pattern: + + ./benchmarks/jmh.sh -l TransformBench + +Run a specific test and override the number of forks, iterations and warm-up iteration to `2`: + + ./benchmarks/jmh.sh -f 2 -i 2 -wi 2 TransformBench + +Run a specific test with async and GC profilers on Linux and flame graph output: + + ./benchmarks/jmh.sh -prof gc -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph TransformBench + +The following sections cover async profiler and GC profilers in more detail. + +### Using JMH with async profiler + +It's good practice to check profiler output for micro-benchmarks in order to verify that they represent the expected +application behavior and measure what you expect to measure. Some example pitfalls include the use of expensive mocks +or accidental inclusion of test setup code in the benchmarked code. JMH includes +[async-profiler](https://github.com/jvm-profiling-tools/async-profiler) integration that makes this easy: + + ./benchmarks/jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so + +or if having async-profiler on environment variable `export LD_LIBRARY_PATH=/opt/async-profiler-2.9-linux-x64/build/` + + ./benchmarks/jmh.sh -prof async + +With flame graph output (the semicolon is escaped to ensure it is not treated as a command separator): + + ./benchmarks/jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph + +Simultaneous cpu, allocation and lock profiling with async profiler 2.0 and jfr output (the semicolon is +escaped to ensure it is not treated as a command separator): + + ./benchmarks/jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=jfr\;alloc\;lock TransformBench + +A number of arguments can be passed to configure async profiler, run the following for a description: + + ./benchmarks/jmh.sh -prof async:help + +### Using JMH GC profiler + +It's good practice to run your benchmark with `-prof gc` to measure its allocation rate: + + ./benchmarks/jmh.sh -prof gc + +Of particular importance is the `norm` alloc rates, which measure the allocations per operation rather than allocations +per second which can increase when you have make your code faster. + +### Running JMH outside gradle + +The JMH benchmarks can be run outside gradle as you would with any executable jar file: + + java -jar ./benchmarks/build/libs/kafka-benchmarks-*.jar -f2 TransformBench + +### Gradle Tasks + +If no benchmark mode is specified, the default is used which is throughput. It is assumed that users run +the gradle tasks with `./gradlew` from the root of the Kafka project. + +* `benchmarks:shadowJar` - creates the uber jar required to run the benchmarks. + +* `benchmarks:jmh` - runs the `clean` and `shadowJar` tasks followed by all the benchmarks. + +### JMH Options +Some common JMH options are: + +```text + + -e Benchmarks to exclude from the run. + + -f How many times to fork a single benchmark. Use 0 to + disable forking altogether. Warning: disabling + forking may have detrimental impact on benchmark + and infrastructure reliability, you might want + to use different warmup mode instead. + + -i Number of measurement iterations to do. Measurement + iterations are counted towards the benchmark score. + (default: 1 for SingleShotTime, and 5 for all other + modes) + + -l List the benchmarks that match a filter, and exit. + + -lprof List profilers, and exit. + + -o Redirect human-readable output to a given file. + + -prof Use profilers to collect additional benchmark data. + Some profilers are not available on all JVMs and/or + all OSes. Please see the list of available profilers + with -lprof. + + -v Verbosity mode. Available modes are: [SILENT, NORMAL, + EXTRA] + + -wi Number of warmup iterations to do. Warmup iterations + are not counted towards the benchmark score. (default: + 0 for SingleShotTime, and 5 for all other modes) +``` + +To view all options run jmh with the -h flag. diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle new file mode 100644 index 000000000..f6d7626dd --- /dev/null +++ b/benchmarks/build.gradle @@ -0,0 +1,64 @@ +/* + * Copyright 2021 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// JMH execution borrowed from https://github.com/apache/kafka/blob/trunk/jmh-benchmarks + + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveBaseName = 'kafka-ts-benchmarks' +} + +ext { + jmhVersion = "1.36" +} + +dependencies { + implementation project(':core') + implementation group: "org.apache.kafka", name: "kafka-storage-api", version: kafkaVersion + implementation group: "org.apache.kafka", name: "kafka-clients", version: kafkaVersion + + implementation "org.openjdk.jmh:jmh-core:$jmhVersion" + implementation "org.openjdk.jmh:jmh-core-benchmarks:$jmhVersion" + annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$jmhVersion" + + implementation "org.slf4j:slf4j-log4j12:1.7.36" +} + +jar { + manifest { + attributes "Main-Class": "org.openjdk.jmh.Main" + } +} + +tasks.register('jmh', JavaExec) { + dependsOn ':benchmarks:clean' + dependsOn ':benchmarks:shadowJar' + + mainClass = "-jar" + + doFirst { + if (System.getProperty("jmhArgs")) { + args System.getProperty("jmhArgs").split(' ') + } + args = [shadowJar.getArchiveFile(), *args] + } +} + +javadoc { + enabled = false +} diff --git a/benchmarks/src/main/resources/log4j.properties b/benchmarks/src/main/resources/log4j.properties new file mode 100644 index 000000000..3a64848a3 --- /dev/null +++ b/benchmarks/src/main/resources/log4j.properties @@ -0,0 +1,21 @@ +# +# Copyright 2023 Aiven Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n diff --git a/build.gradle b/build.gradle index 0a1dc3d41..5532f2946 100644 --- a/build.gradle +++ b/build.gradle @@ -24,6 +24,7 @@ plugins { id 'info.solidsoft.pitest' version '1.15.0' apply false id "java-library" id "distribution" + id 'com.github.johnrengelman.shadow' version '8.1.1' apply false } apply plugin: 'info.solidsoft.pitest.aggregator' diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index e4415b5e5..59d549a77 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -21,6 +21,7 @@ + @@ -30,6 +31,7 @@ + diff --git a/settings.gradle b/settings.gradle index 1d6840abc..45cb1db53 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,3 +24,4 @@ include 'storage:gcs' include 'storage:s3' include 'e2e' include 'commons' +include 'benchmarks'