geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wireba...@apache.org
Subject [geode-benchmarks] branch develop updated: GEODE-6145: Allow multiple results for a single probe analyzer (#14)
Date Wed, 05 Dec 2018 17:59:43 GMT
This is an automated email from the ASF dual-hosted git repository.

wirebaron pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git


The following commit(s) were added to refs/heads/develop by this push:
     new ef2e72f  GEODE-6145: Allow multiple results for a single probe analyzer (#14)
ef2e72f is described below

commit ef2e72fec401d6a801744f7a530d9c026c9d77b0
Author: Brian Rowe <browe@pivotal.io>
AuthorDate: Wed Dec 5 09:59:39 2018 -0800

    GEODE-6145: Allow multiple results for a single probe analyzer (#14)
    
    This change modifies the ProbeResultParser interface to return a list of
    ResultData objects.  It takes advantage of this new functionality to return
    6 results from the HDRHistogramParser (median latency, 90, 99, 99.9 percentile
    latencies, average latency, and latency standard deviation).  This is probably
    too noisy for the long run, but could prove useful as we evaluate different
    platforms.
    
    This change also changes the BenchmarkRunAnalyzer (invoked via gradlew
    analyzeRun) to take the baseline directory as its first argument.
---
 .../perftest/analysis/BenchmarkRunAnalyzer.java    | 24 ++++++++++++++++------
 .../geode/perftest/analysis/ProbeResultParser.java | 17 +++++++++++----
 .../analysis/YardstickHdrHistogramParser.java      | 19 +++++++++--------
 .../analysis/YardstickPercentileSensorParser.java  | 22 +++++++++-----------
 .../analysis/YardstickThroughputSensorParser.java  | 10 ++++-----
 .../analysis/BenchmarkRunAnalyzerTest.java         |  2 +-
 6 files changed, 57 insertions(+), 37 deletions(-)

diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
index 764f622..57ae207 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java
@@ -20,6 +20,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -55,7 +56,7 @@ public class BenchmarkRunAnalyzer {
     probes.add(probeResultParser);
   }
 
-  public BenchmarkRunResult analyzeTestRun(File testResultDir, File baselineResultDir)
+  public BenchmarkRunResult analyzeTestRun(File baselineResultDir, File testResultDir)
       throws IOException {
     List<File> benchmarkDirs = Arrays.asList(testResultDir.listFiles());
     benchmarkDirs.sort(File::compareTo);
@@ -72,22 +73,33 @@ public class BenchmarkRunAnalyzer {
       final BenchmarkRunResult.BenchmarkResult benchmarkResult =
           result.addBenchmark(testDir.getName());
       for (ProbeResultParser probe : probes) {
-        double testResult = getTestResult(testYardstickDirs, probe);
-        double baselineResult = getTestResult(baselineYardstickDirs, probe);
+        List<ProbeResultParser.ResultData> testResults = getTestResult(testYardstickDirs,
probe);
+        List<ProbeResultParser.ResultData> baselineResults =
+            getTestResult(baselineYardstickDirs, probe);
 
-        benchmarkResult.addProbeResult(probe.getResultDescription(), baselineResult, testResult);
+        Iterator<ProbeResultParser.ResultData> testResultIter = testResults.iterator();
+        Iterator<ProbeResultParser.ResultData> baselineResultIter = baselineResults.iterator();
+
+        while (testResultIter.hasNext()) {
+          ProbeResultParser.ResultData testResult = testResultIter.next();
+          ProbeResultParser.ResultData baselineResult = baselineResultIter.next();
+
+          benchmarkResult.addProbeResult(testResult.description, baselineResult.value,
+              testResult.value);
+        }
       }
     }
 
     return result;
   }
 
-  private double getTestResult(List<File> resultDirs, ProbeResultParser probe) throws
IOException {
+  private List<ProbeResultParser.ResultData> getTestResult(List<File> resultDirs,
+      ProbeResultParser probe) throws IOException {
     probe.reset();
     for (File outputDirectory : resultDirs) {
       probe.parseResults(outputDirectory);
     }
-    return probe.getProbeResult();
+    return probe.getProbeResults();
   }
 
   private List<File> getYardstickOutputForBenchmarkDir(File benchmarkDir) throws IOException
{
diff --git a/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
b/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
index 1e94329..dd1935e 100644
--- a/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java
@@ -16,8 +16,10 @@ package org.apache.geode.perftest.analysis;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
 public interface ProbeResultParser {
+
   // Given a output directory for a benchmark, parse out the data for the desired probe.
Note that
   // this method may be passed several csv files for a run and is expected to appropriately
   // aggregate the result of interest.
@@ -26,9 +28,16 @@ public interface ProbeResultParser {
   // Reset the parser to a clean state where parseResults can be called again
   void reset();
 
-  // Get a single float value summarizing the data for the probe.
-  double getProbeResult();
+  // Get the {description, value} pairs for the probe
+  List<ResultData> getProbeResults();
+
+  class ResultData {
+    public String description;
+    public double value;
 
-  // Get a text description of what the probe result is depicting
-  String getResultDescription();
+    public ResultData(String description, double value) {
+      this.description = description;
+      this.value = value;
+    }
+  }
 }
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
index 8b2c6da..19d1749 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickHdrHistogramParser.java
@@ -17,6 +17,8 @@ package org.apache.geode.perftest.yardstick.analysis;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.HdrHistogram.Histogram;
 import org.HdrHistogram.HistogramLogReader;
@@ -31,7 +33,6 @@ import org.apache.geode.perftest.yardstick.hdrhistogram.HdrHistogramWriter;
  */
 public class YardstickHdrHistogramParser implements ProbeResultParser {
   public static final String sensorOutputFile = HdrHistogramWriter.FILE_NAME;
-  public static final String probeResultDescription = "HDR 99th percentile latency";
 
   public Histogram histogram = null;
 
@@ -55,13 +56,15 @@ public class YardstickHdrHistogramParser implements ProbeResultParser
{
   }
 
   @Override
-  // Default probe result is the 99th percentile latency for the benchmark
-  public double getProbeResult() {
-    return histogram.getValueAtPercentile(99);
-  }
+  public List<ResultData> getProbeResults() {
+    List<ResultData> results = new ArrayList<>(3);
+    results.add(new ResultData("median latency", histogram.getValueAtPercentile(50)));
+    results.add(new ResultData("90th percentile latency", histogram.getValueAtPercentile(90)));
+    results.add(new ResultData("99th percentile latency", histogram.getValueAtPercentile(99)));
+    results.add(new ResultData("99.9th percentile latency", histogram.getValueAtPercentile(99.9)));
+    results.add(new ResultData("average latency", histogram.getMean()));
+    results.add(new ResultData("latency standard deviation", histogram.getStdDeviation()));
 
-  @Override
-  public String getResultDescription() {
-    return probeResultDescription;
+    return results;
   }
 }
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
index fb843ca..d99a538 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java
@@ -21,6 +21,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.List;
 
 import org.yardstickframework.probes.PercentileProbe;
 
@@ -32,7 +33,7 @@ import org.apache.geode.perftest.analysis.ProbeResultParser;
  */
 public class YardstickPercentileSensorParser implements ProbeResultParser {
   public static final String sensorOutputFile = "PercentileProbe.csv";
-  public static final String probeResultDescription = "99th percentile latency";
+  public static final String probeResultDescription = "YS 99th percentile latency";
 
   private class SensorBucket {
     public int latencyBucket;
@@ -74,17 +75,6 @@ public class YardstickPercentileSensorParser implements ProbeResultParser
{
     buckets = new ArrayList<>();
   }
 
-  @Override
-  // Default probe result is the 99th percentile latency for the benchmark
-  public double getProbeResult() {
-    return getPercentile(99);
-  }
-
-  @Override
-  public String getResultDescription() {
-    return probeResultDescription;
-  }
-
   private void normalizeBuckets() {
     float totalPercentage = 0;
     for (SensorBucket bucket : buckets) {
@@ -131,4 +121,12 @@ public class YardstickPercentileSensorParser implements ProbeResultParser
{
 
     return targetBucket.latencyBucket + bucketSize * percentileLocationInTargetBucket;
   }
+
+  @Override
+  public List<ResultData> getProbeResults() {
+    List<ResultData> results = new ArrayList<>(1);
+    results.add(new ResultData(probeResultDescription, getPercentile(99)));
+
+    return results;
+  }
 }
diff --git a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
index 340f3c3..081b263 100644
--- a/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
+++ b/harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java
@@ -56,13 +56,11 @@ public class YardstickThroughputSensorParser implements ProbeResultParser
{
   }
 
   @Override
-  public double getProbeResult() {
-    return getAverageThroughput();
-  }
+  public List<ResultData> getProbeResults() {
+    List<ResultData> results = new ArrayList<>(1);
+    results.add(new ResultData(probeResultDescription, getAverageThroughput()));
 
-  @Override
-  public String getResultDescription() {
-    return probeResultDescription;
+    return results;
   }
 
   public double getAverageThroughput() {
diff --git a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
index 12e37d1..f38f355 100644
--- a/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
+++ b/harness/src/test/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzerTest.java
@@ -81,7 +81,7 @@ public class BenchmarkRunAnalyzerTest {
 
     StringWriter writer = new StringWriter();
 
-    BenchmarkRunResult results = harvester.analyzeTestRun(testFolder, baseFolder);
+    BenchmarkRunResult results = harvester.analyzeTestRun(baseFolder, testFolder);
 
     BenchmarkRunResult expectedBenchmarkResult = new BenchmarkRunResult();
     BenchmarkRunResult.BenchmarkResult resultA = expectedBenchmarkResult.addBenchmark("BenchmarkA");


Mime
View raw message