Merge pull request #34 from DiceTechnology/feature/benchmarks

Feature #16 Add benchmarks

README.md

@@ -164,7 +164,19 @@ new MaxmindLineParser(
 )
 ```
 
-The Maxmind reader is can load a database with any precision (for example City or Country) and from both the Lite and commercial versions.
+The Maxmind reader can load a database with any precision (for example City or Country) and from both the Lite and commercial versions.
 
 # Benchmark
-wip
+Performance of the library depends on a number of variables including:
+- CPU
+- type of local disk
+- OS
+
+However, on a 2017 MBP with SSD, MacOS and 16Gb RAM we observed the following performance for
+when loading the full Maxmind and DbIp databases (9.4M ip ranges):
+
+- Initial load from ip data files: 35s
+- Single threaded lookup of 1000 distinct ip addresses: 100ms
+
+Benchmarking on other machine - WIP
+Benchmarking heap and off-heap memory usage - WIP

pom.xml

@@ -87,6 +87,7 @@
     <junit.version>4.12</junit.version>
     <mockito.version>2.7.22</mockito.version>
     <jococo.version>0.8.1</jococo.version>
+    <jmh.version>1.21</jmh.version>
   </properties>
 
   <build>
@@ -305,6 +306,12 @@
       <version>${mockito.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <version>${jmh.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>

src/main/java/technology/dice/dicewhere/provider/maxmind/reading/MaxmindDbReader.java

@@ -48,6 +48,6 @@ protected Stream<String> lines() throws IOException {
 
   @Override
   public ProviderKey provider() {
-    return new MaxmindProviderKey();
+    return MaxmindProviderKey.of();
   }
 }

src/test/java/technology/dice/dicewhere/Main.java

@@ -14,6 +14,7 @@
 import java.util.Optional;
 import java.util.Scanner;
 import java.util.concurrent.ExecutionException;
+
 import technology.dice.dicewhere.api.api.IP;
 import technology.dice.dicewhere.api.api.IPResolver;
 import technology.dice.dicewhere.api.api.IpInformation;
@@ -24,6 +25,7 @@
 import technology.dice.dicewhere.reading.RawLine;
 
 public class Main {
+
   public static void maisn(String[] args) throws IOException {
     String IPV4 = "192.168.4.5";
     String IPV6 = "0:0:0:0:0:ffff:c0a8:405";

src/test/java/technology/dice/dicewhere/api/IPResolverBenchmark.java

@@ -0,0 +1,135 @@
+package technology.dice.dicewhere.api;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.ClassloaderProfiler;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.HotspotClassloadingProfiler;
+import org.openjdk.jmh.profile.HotspotMemoryProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import technology.dice.dicewhere.api.api.IPResolver;
+import technology.dice.dicewhere.provider.ProviderKey;
+import technology.dice.dicewhere.provider.dbip.reading.DbIpLineReader;
+import technology.dice.dicewhere.provider.maxmind.MaxmindProviderKey;
+import technology.dice.dicewhere.provider.maxmind.reading.MaxmindDbReader;
+import technology.dice.dicewhere.reading.LineReaderListener;
+import technology.dice.dicewhere.reading.RawLine;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+@State(Scope.Thread)
+@Fork(value = 1, jvmArgsAppend = "-Djmh.stack.lines=3")
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+public class IPResolverBenchmark {
+
+  private static final String IPV4 = "192.168.4.5";
+  private static final String IPV6 = "0:0:0:0:0:ffff:c0a8:405";
+
+  private static final String MAXMIND_RESOURCES_FOLDER =
+      "/Users/zorg/Downloads/where/GeoIP2-City-CSV_20180911";
+  private static final String RESOURCES_FOLDER = "/Users/zorg/Downloads/where";
+
+  private IPResolver resolver;
+
+  public static void main(String[] args) throws IOException, RunnerException {
+
+    Options opt =
+        new OptionsBuilder()
+            .include(IPResolverBenchmark.class.getSimpleName())
+            .addProfiler(GCProfiler.class)
+            .addProfiler(ClassloaderProfiler.class)
+            .addProfiler(HotspotMemoryProfiler.class)
+            .addProfiler(HotspotClassloadingProfiler.class)
+            .addProfiler(MaxMemoryProfiler.class)
+            .forks(1)
+            .build();
+
+    new Runner(opt).run();
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.SampleTime)
+  @Warmup(iterations = 5)
+  @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.MILLISECONDS)
+  public void testIPV4() throws UnknownHostException {
+    for (int i = 0; i < 4; ++i) {
+      for (int b = 0; b < 255; ++b) {
+        resolver.resolve("192.168." + i + "." + b , MaxmindProviderKey.of());
+      }
+    }
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.SampleTime)
+  @Warmup(iterations = 1)
+  @Measurement(iterations = 5, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+  public void testIPV6() throws UnknownHostException {
+    for (int i = 0; i < 1000; ++i) {
+      resolver.resolve(IPV6, MaxmindProviderKey.of());
+    }
+  }
+
+  @TearDown
+  public void tearDown() {}
+
+  @Setup
+  public void setUp() throws IOException {
+    MaxmindDbReader maxmindDbReader =
+        new MaxmindDbReader(
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Locations-en.csv"),
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Blocks-IPv4.csv"),
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Blocks-IPv6.csv"));
+
+    DbIpLineReader dbIpLineReader = new DbIpLineReader(Paths.get(RESOURCES_FOLDER + "/dbip-full-2018-09.csv"));
+
+    LineReaderListener lineReaderListener =
+        new LineReaderListener() {
+
+          @Override
+          public void lineRead(ProviderKey provider, RawLine rawLine, long elapsedMillis) {
+            if (rawLine.getLineNumber() % 100000 == 0) {
+              System.out.println(
+                  Thread.currentThread().getName()
+                      + " ##### Read "
+                      + rawLine.getLineNumber()
+                      + " records so far in + "
+                      + elapsedMillis / 1e3
+                      + " seconds.");
+            }
+          }
+
+          @Override
+          public void finished(ProviderKey provider, long linesProcessed, long elapsedMillis) {
+            System.out.println(
+                "Finished processing "
+                    + linesProcessed
+                    + " lines in "
+                    + elapsedMillis / 1e3
+                    + " seconds");
+          }
+        };
+
+    IPResolver.Builder resolverBuilder =
+        new IPResolver.Builder()
+            .withProvider(maxmindDbReader)
+            .withProvider(dbIpLineReader)
+            .withReaderListener(lineReaderListener);
+
+    resolver = resolverBuilder.build();
+  }
+}

src/test/java/technology/dice/dicewhere/api/MaxMemoryProfiler.java

@@ -0,0 +1,38 @@
+package technology.dice.dicewhere.api;
+
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.profile.InternalProfiler;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.results.IterationResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ScalarResult;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collection;
+
+public class MaxMemoryProfiler implements InternalProfiler {
+
+  @Override
+  public String getDescription() {
+    return "Max memory heap profiler";
+  }
+
+  @Override
+  public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) {}
+
+  @Override
+  public Collection<? extends Result> afterIteration(
+      BenchmarkParams benchmarkParams, IterationParams iterationParams, IterationResult result) {
+
+    long heap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
+    long nonHeap = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage().getUsed();
+
+    Collection<ScalarResult> results = new ArrayList<>();
+    results.add(new ScalarResult("Current memory usage of the heap", heap, "bytes", AggregationPolicy.MAX));
+    results.add(new ScalarResult("Current memory usage of non-heap memory", nonHeap, "bytes", AggregationPolicy.MAX));
+
+    return results;
+  }
+}

src/test/java/technology/dice/dicewhere/parsing/provider/LoaderBenchmark.java

@@ -0,0 +1,124 @@
+package technology.dice.dicewhere.parsing.provider;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.ClassloaderProfiler;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.HotspotClassloadingProfiler;
+import org.openjdk.jmh.profile.HotspotMemoryProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import technology.dice.dicewhere.api.IPResolverBenchmark;
+import technology.dice.dicewhere.api.MaxMemoryProfiler;
+import technology.dice.dicewhere.api.api.IPResolver;
+import technology.dice.dicewhere.provider.ProviderKey;
+import technology.dice.dicewhere.provider.dbip.reading.DbIpLineReader;
+import technology.dice.dicewhere.provider.maxmind.MaxmindProviderKey;
+import technology.dice.dicewhere.provider.maxmind.reading.MaxmindDbReader;
+import technology.dice.dicewhere.reading.LineReaderListener;
+import technology.dice.dicewhere.reading.RawLine;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
+
+@State(Scope.Benchmark)
+@Fork(value = 1, jvmArgsAppend = "-Djmh.stack.lines=3")
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+public class LoaderBenchmark {
+
+  private static final String IPV4 = "192.168.4.5";
+
+  private static final String MAXMIND_RESOURCES_FOLDER =
+      "/Users/zorg/Downloads/where/GeoIP2-City-CSV_20180911";
+  private static final String RESOURCES_FOLDER = "/Users/zorg/Downloads/where";
+
+  private static final LineReaderListener lineReaderListener =
+      new LineReaderListener() {
+
+        @Override
+        public void lineRead(ProviderKey provider, RawLine rawLine, long elapsedMillis) {
+          if (rawLine.getLineNumber() % 100000 == 0) {
+            System.out.println(
+                Thread.currentThread().getName()
+                    + " ##### Read "
+                    + rawLine.getLineNumber()
+                    + " records so far in + "
+                    + elapsedMillis / 1e3
+                    + " seconds.");
+          }
+        }
+
+        @Override
+        public void finished(ProviderKey provider, long linesProcessed, long elapsedMillis) {
+          System.out.println(
+              "Finished processing "
+                  + linesProcessed
+                  + " lines in "
+                  + elapsedMillis / 1e3
+                  + " seconds");
+        }
+      };
+
+  public static void main(String[] args) throws IOException, RunnerException {
+
+    Options opt =
+        new OptionsBuilder()
+            .include(IPResolverBenchmark.class.getSimpleName())
+            .addProfiler(GCProfiler.class)
+            .addProfiler(ClassloaderProfiler.class)
+            .addProfiler(HotspotMemoryProfiler.class)
+            .addProfiler(HotspotClassloadingProfiler.class)
+            .addProfiler(MaxMemoryProfiler.class)
+            .forks(1)
+            .build();
+
+    new Runner(opt).run();
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.SampleTime)
+  @Warmup(iterations = 0)
+  @Measurement(iterations = 1, time = 5, timeUnit = TimeUnit.MILLISECONDS)
+  public void testDbIp() throws IOException {
+
+    DbIpLineReader dbIpLineReader =
+        new DbIpLineReader(Paths.get(RESOURCES_FOLDER + "/dbip-full-2018-09.csv"));
+
+    IPResolver.Builder resolverBuilder =
+        new IPResolver.Builder()
+            .withProvider(dbIpLineReader)
+            .withReaderListener(lineReaderListener);
+
+    resolverBuilder.build().resolve(IPV4, MaxmindProviderKey.of());
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.SampleTime)
+  @Warmup(iterations = 0)
+  @Measurement(iterations = 1, time = 5, timeUnit = TimeUnit.MILLISECONDS)
+  public void testMaxmind() throws IOException {
+
+    MaxmindDbReader maxmindDbReader =
+        new MaxmindDbReader(
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Locations-en.csv"),
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Blocks-IPv4.csv"),
+            Paths.get(MAXMIND_RESOURCES_FOLDER + "/GeoIP2-City-Blocks-IPv6.csv"));
+
+    IPResolver.Builder resolverBuilder =
+        new IPResolver.Builder()
+            .withProvider(maxmindDbReader)
+            .withReaderListener(lineReaderListener);
+
+    resolverBuilder.build().resolve(IPV4, MaxmindProviderKey.of());
+  }
+}