Author: atm
Date: Fri Jul 27 00:22:54 2012
New Revision: 1366245
URL: http://svn.apache.org/viewvc?rev=1366245&view=rev
Log:
HDFS-3650. Use MutableQuantiles to provide latency histograms for various operations. Contributed
by Andrew Wang.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1366245&r1=1366244&r2=1366245&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Fri Jul 27 00:22:54 2012
@@ -893,6 +893,25 @@ public class Configuration implements It
}
return Integer.parseInt(valueString);
}
+
+ /**
+ * Get the value of the <code>name</code> property as a set of comma-delimited
+ * <code>int</code> values.
+ *
+ * If no such property exists, an empty array is returned.
+ *
+ * @param name property name
+ * @return property value interpreted as an array of comma-delimited
+ * <code>int</code> values
+ */
+ public int[] getInts(String name) {
+ String[] strings = getTrimmedStrings(name);
+ int[] ints = new int[strings.length];
+ for (int i = 0; i < strings.length; i++) {
+ ints[i] = Integer.parseInt(strings[i]);
+ }
+ return ints;
+ }
/**
* Set the value of the <code>name</code> property to an <code>int</code>.
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java?rev=1366245&r1=1366244&r2=1366245&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
Fri Jul 27 00:22:54 2012
@@ -45,7 +45,8 @@ import com.google.common.annotations.Vis
@InterfaceStability.Evolving
public class MutableQuantiles extends MutableMetric {
- static final Quantile[] quantiles = { new Quantile(0.50, 0.050),
+ @VisibleForTesting
+ public static final Quantile[] quantiles = { new Quantile(0.50, 0.050),
new Quantile(0.75, 0.025), new Quantile(0.90, 0.010),
new Quantile(0.95, 0.005), new Quantile(0.99, 0.001) };
@@ -90,8 +91,7 @@ public class MutableQuantiles extends Mu
"Number of %s for %s with %ds interval", lsName, desc, interval));
// Construct the MetricsInfos for the quantiles, converting to percentiles
quantileInfos = new MetricsInfo[quantiles.length];
- String nameTemplate = ucName + "%dthPercentile" + interval + "sInterval"
- + uvName;
+ String nameTemplate = ucName + "%dthPercentile" + uvName;
String descTemplate = "%d percentile " + lvName + " with " + interval
+ " second interval for " + desc;
for (int i = 0; i < quantiles.length; i++) {
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java?rev=1366245&r1=1366244&r2=1366245&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
Fri Jul 27 00:22:54 2012
@@ -31,8 +31,8 @@ import java.nio.channels.WritableByteCha
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.metrics2.lib.MutableRate;
-import org.apache.hadoop.util.Progressable;
/**
* This implements an output stream that can have a timeout while writing.
@@ -179,9 +179,9 @@ public class SocketOutputStream extends
* @param fileCh FileChannel to transfer data from.
* @param position position within the channel where the transfer begins
* @param count number of bytes to transfer.
- * @param waitForWritableTime updated by the nanoseconds spent waiting for
- * the socket to become writable
- * @param transferTime updated by the nanoseconds spent transferring data
+ * @param waitForWritableTime nanoseconds spent waiting for the socket
+ * to become writable
+ * @param transferTime nanoseconds spent transferring data
*
* @throws EOFException
* If end of input file is reached before requested number of
@@ -195,8 +195,8 @@ public class SocketOutputStream extends
* {@link FileChannel#transferTo(long, long, WritableByteChannel)}.
*/
public void transferToFully(FileChannel fileCh, long position, int count,
- MutableRate waitForWritableTime,
- MutableRate transferToTime) throws IOException {
+ LongWritable waitForWritableTime,
+ LongWritable transferToTime) throws IOException {
long waitTime = 0;
long transferTime = 0;
while (count > 0) {
@@ -236,12 +236,12 @@ public class SocketOutputStream extends
waitTime += wait - start;
transferTime += transfer - wait;
}
-
+
if (waitForWritableTime != null) {
- waitForWritableTime.add(waitTime);
+ waitForWritableTime.set(waitTime);
}
if (transferToTime != null) {
- transferToTime.add(transferTime);
+ transferToTime.set(transferTime);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java?rev=1366245&r1=1366244&r2=1366245&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
Fri Jul 27 00:22:54 2012
@@ -150,7 +150,7 @@ public class TestMutableMetrics {
info("FooNumOps", "Number of ops for stat with 5s interval"),
(long) 2000);
Quantile[] quants = MutableQuantiles.quantiles;
- String name = "Foo%dthPercentile5sIntervalLatency";
+ String name = "Foo%dthPercentileLatency";
String desc = "%d percentile latency with 5 second interval for stat";
for (Quantile q : quants) {
int percentile = (int) (100 * q.quantile);
@@ -176,7 +176,7 @@ public class TestMutableMetrics {
"Latency", 5);
Quantile[] quants = MutableQuantiles.quantiles;
- String name = "Foo%dthPercentile5sIntervalLatency";
+ String name = "Foo%dthPercentileLatency";
String desc = "%d percentile latency with 5 second interval for stat";
// Push values for three intervals
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java?rev=1366245&r1=1366244&r2=1366245&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java
Fri Jul 27 00:22:54 2012
@@ -23,7 +23,9 @@ import static com.google.common.base.Pre
import org.hamcrest.Description;
import org.junit.Assert;
+import static org.mockito.AdditionalMatchers.geq;
import static org.mockito.Mockito.*;
+
import org.mockito.stubbing.Answer;
import org.mockito.internal.matchers.GreaterThan;
import org.mockito.invocation.InvocationOnMock;
@@ -39,7 +41,11 @@ import org.apache.hadoop.metrics2.Metric
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.util.Quantile;
+
import static org.apache.hadoop.metrics2.lib.Interns.*;
+import static org.apache.hadoop.test.MetricsAsserts.eqName;
/**
* Helpers for metrics source tests
@@ -328,4 +334,23 @@ public class MetricsAsserts {
MetricsSource source) {
assertGaugeGt(name, greater, getMetrics(source));
}
+
+ /**
+ * Asserts that the NumOps and quantiles for a metric have been changed at
+ * some point to a non-zero value.
+ *
+ * @param prefix of the metric
+ * @param rb MetricsRecordBuilder with the metric
+ */
+ public static void assertQuantileGauges(String prefix,
+ MetricsRecordBuilder rb) {
+ verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l));
+ for (Quantile q : MutableQuantiles.quantiles) {
+ String nameTemplate = prefix + "%dthPercentileLatency";
+ int percentile = (int) (100 * q.quantile);
+ verify(rb).addGauge(
+ eqName(info(String.format(nameTemplate, percentile), "")),
+ geq(0l));
+ }
+ }
}
|