cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brandonwilli...@apache.org
Subject git commit: Add aggregate statistics to cassandra-stress Patch by Ryan McGuire, reviewed by brandonwilliams for CASSANDRA-5243
Date Tue, 12 Feb 2013 21:13:25 GMT
Updated Branches:
  refs/heads/trunk dd4d798dc -> c1e5b0ffa


Add aggregate statistics to cassandra-stress
Patch by Ryan McGuire, reviewed by brandonwilliams for CASSANDRA-5243


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c1e5b0ff
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c1e5b0ff
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c1e5b0ff

Branch: refs/heads/trunk
Commit: c1e5b0ffafd28411ef2c36ace94bf011c0484b3f
Parents: dd4d798
Author: Brandon Williams <brandonwilliams@apache.org>
Authored: Tue Feb 12 15:12:59 2013 -0600
Committer: Brandon Williams <brandonwilliams@apache.org>
Committed: Tue Feb 12 15:12:59 2013 -0600

----------------------------------------------------------------------
 .../src/org/apache/cassandra/stress/Session.java   |   12 ++
 .../org/apache/cassandra/stress/StressAction.java  |   19 ++-
 .../apache/cassandra/stress/StressStatistics.java  |  124 +++++++++++++++
 3 files changed, 152 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e5b0ff/tools/stress/src/org/apache/cassandra/stress/Session.java
----------------------------------------------------------------------
diff --git a/tools/stress/src/org/apache/cassandra/stress/Session.java b/tools/stress/src/org/apache/cassandra/stress/Session.java
index 804e4e8..58181a0 100644
--- a/tools/stress/src/org/apache/cassandra/stress/Session.java
+++ b/tools/stress/src/org/apache/cassandra/stress/Session.java
@@ -108,6 +108,7 @@ public class Session implements Serializable
         availableOptions.addOption("Z",  "compaction-strategy",  true,   "CompactionStrategy
to use.");
         availableOptions.addOption("U",  "comparator",           true,   "Column Comparator
to use. Currently supported types are: TimeUUIDType, AsciiType, UTF8Type.");
         availableOptions.addOption("tf", "transport-factory",    true,   "Fully-qualified
TTransportFactory class name for creating a connection. Note: For Thrift over SSL, use org.apache.cassandra.stress.SSLTransportFactory.");
+        availableOptions.addOption("ns", "no-statistics",        false,  "Turn off the aggegate
statistics that is normally output after completion.");
         availableOptions.addOption("ts", SSL_TRUSTSTORE,         true, "SSL: full path to
truststore");
         availableOptions.addOption("tspw", SSL_TRUSTSTORE_PW,    true, "SSL: full path to
truststore");
         availableOptions.addOption("prtcl", SSL_PROTOCOL,        true, "SSL: connections
protocol to use (default: TLS)");
@@ -138,6 +139,7 @@ public class Session implements Serializable
     private boolean enable_cql    = false;
     private boolean use_prepared  = false;
     private boolean trace         = false;
+    private boolean captureStatistics = true;
 
     private final String outFileName;
 
@@ -405,6 +407,11 @@ public class Session implements Serializable
                 timeUUIDComparator = false;
             }
 
+            if (cmd.hasOption("ns"))
+            {
+                captureStatistics = false;
+            }
+
             if(cmd.hasOption(SSL_TRUSTSTORE))
                 encOptions.truststore = cmd.getOptionValue(SSL_TRUSTSTORE);
 
@@ -582,6 +589,11 @@ public class Session implements Serializable
         return use_prepared;
     }
 
+    public boolean outputStatistics()
+    {
+        return captureStatistics;
+    }
+
     /**
      * Create Keyspace with Standard and Super/Counter column families
      */

http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e5b0ff/tools/stress/src/org/apache/cassandra/stress/StressAction.java
----------------------------------------------------------------------
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressAction.java b/tools/stress/src/org/apache/cassandra/stress/StressAction.java
index ca71aba..9aa128f 100644
--- a/tools/stress/src/org/apache/cassandra/stress/StressAction.java
+++ b/tools/stress/src/org/apache/cassandra/stress/StressAction.java
@@ -89,6 +89,8 @@ public class StressAction extends Thread
         int interval = client.getProgressInterval();
         int epochIntervals = client.getProgressInterval() * 10;
         long testStartTime = System.currentTimeMillis();
+        
+        StressStatistics stats = new StressStatistics(client, output);
 
         while (!terminate)
         {
@@ -142,6 +144,14 @@ public class StressAction extends Thread
                                              keyDelta / interval,
                                              latency.getMedian(), latency.get95thPercentile(),
latency.get999thPercentile(),
                                              currentTimeInSeconds));
+
+                if (client.outputStatistics()) {
+                    stats.addIntervalStats(total, 
+                                           opDelta / interval, 
+                                           keyDelta / interval, 
+                                           latency, 
+                                           currentTimeInSeconds);
+                        }
             }
         }
 
@@ -156,11 +166,14 @@ public class StressAction extends Thread
             if (consumer.getReturnCode() == FAILURE)
                 returnCode = FAILURE;
 
-        if (returnCode == SUCCESS)
+        if (returnCode == SUCCESS) {            
+            if (client.outputStatistics())
+                stats.printStats();
             // marking an end of the output to the client
-            output.println("END");
-        else
+            output.println("END");            
+        } else {
             output.println("FAILURE");
+        }
 
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e5b0ff/tools/stress/src/org/apache/cassandra/stress/StressStatistics.java
----------------------------------------------------------------------
diff --git a/tools/stress/src/org/apache/cassandra/stress/StressStatistics.java b/tools/stress/src/org/apache/cassandra/stress/StressStatistics.java
new file mode 100644
index 0000000..4afac5a
--- /dev/null
+++ b/tools/stress/src/org/apache/cassandra/stress/StressStatistics.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress;
+
+import com.yammer.metrics.stats.Snapshot;
+import java.io.PrintStream;
+import org.apache.commons.lang.time.DurationFormatUtils;
+
+/**
+ * Gathers and aggregates statistics for an operation
+ */
+public class StressStatistics
+{
+    
+    private Session client;
+    private PrintStream output;
+
+    private long durationInSeconds;
+    /** The sum of the interval_op_rate values collected by tallyAverages */
+    private int tallyOpRateSum;
+    /** The number of interval_op_rate values collected by tallyAverages */
+    private int tallyOpRateCount;
+    /** The sum of the interval_key_rate values collected by tallyAverages */
+    private int tallyKeyRateSum;
+    /** The number of interval_key_rate values collected by tallyAverages */
+    private int tallyKeyRateCount;
+
+    /** The sum of the latency values collected by tallyAverages */
+    private double tallyLatencySum;
+    /** The number of latency values collected by tallyAverages */
+    private int tallyLatencyCount;
+    /** The sum of the 95%tile latency values collected by tallyAverages */
+    private double tally95thLatencySum;
+    /** The number of 95%tile latency values collected by tallyAverages */
+    private int tally95thLatencyCount;
+    /** The sum of the 99.9%tile latency values collected by tallyAverages */
+    private double tally999thLatencySum;
+    /** The number of 99.9%tile latency values collected by tallyAverages */
+    private int tally999thLatencyCount;
+    
+
+    public StressStatistics(Session client, PrintStream out)
+    {
+        this.client = client;
+        this.output = out;
+
+        tallyOpRateSum = 0;
+        tallyOpRateCount = 0;
+    }
+
+    /**
+     * Collect statistics per-interval
+     */
+    public void addIntervalStats(int totalOperations, int intervalOpRate, 
+                                 int intervalKeyRate, Snapshot latency, 
+                                 long currentTimeInSeconds)
+    {
+        this.tallyAverages(totalOperations, intervalKeyRate, intervalKeyRate, 
+                                latency, currentTimeInSeconds);
+    }
+
+    /**
+     * Collect interval_op_rate and interval_key_rate averages
+     */
+    private void tallyAverages(int totalOperations, int intervalOpRate, 
+                                 int intervalKeyRate, Snapshot latency, 
+                                 long currentTimeInSeconds)
+    {
+        //Skip the first and last 10% of values.
+        //The middle values of the operation are the ones worthwhile
+        //to collect and average:
+        if (totalOperations > (0.10 * client.getNumKeys()) &&
+            totalOperations < (0.90 * client.getNumKeys())) {
+                tallyOpRateSum += intervalOpRate;
+                tallyOpRateCount += 1;
+                tallyKeyRateSum += intervalKeyRate;
+                tallyKeyRateCount += 1;
+                tallyLatencySum += latency.getMedian();
+                tallyLatencyCount += 1;
+                tally95thLatencySum += latency.get95thPercentile();
+                tally95thLatencyCount += 1;
+                tally999thLatencySum += latency.get999thPercentile();
+                tally999thLatencyCount += 1;
+            }
+        durationInSeconds = currentTimeInSeconds;
+    }
+
+    public void printStats()
+    {
+        output.println("\n");
+        if (tallyOpRateCount > 0) {
+            output.println("Averages from the middle 80% of values:");
+            output.println(String.format("interval_op_rate          : %d", 
+                                         (tallyOpRateSum / tallyOpRateCount)));
+            output.println(String.format("interval_key_rate         : %d", 
+                                         (tallyKeyRateSum / tallyKeyRateCount)));
+            output.println(String.format("latency median            : %.1f", 
+                                         (tallyLatencySum / tallyLatencyCount)));
+            output.println(String.format("latency 95th percentile   : %.1f",
+                                         (tally95thLatencySum / tally95thLatencyCount)));
+            output.println(String.format("latency 99.9th percentile : %.1f", 
+                                         (tally999thLatencySum / tally999thLatencyCount)));
+        }
+        output.println("Total operation time      : " + DurationFormatUtils.formatDuration(
+            durationInSeconds*1000, "HH:mm:ss", true));
+    }
+
+}


Mime
View raw message