ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject [2/2] ambari git commit: AMBARI-15694 : AMS returns truncated results when it exceeds the metrics service default result limit config (avijayan)
Date Mon, 11 Apr 2016 23:37:33 GMT
AMBARI-15694 : AMS returns truncated results when it exceeds the metrics service default result
limit config (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/602fb5d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/602fb5d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/602fb5d2

Branch: refs/heads/trunk
Commit: 602fb5d2058e241b99eb98f9ecfb88332a6c75af
Parents: 128f26c
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Mon Apr 11 16:19:18 2016 -0700
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Mon Apr 11 16:37:19 2016 -0700

----------------------------------------------------------------------
 .../timeline/query/PhoenixTransactSQL.java      |  17 ++-
 .../timeline/TestPhoenixTransactSQL.java        | 132 +++++++++++++++++++
 2 files changed, 144 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/602fb5d2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
index c8cef27..0efa68f 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java
@@ -28,6 +28,7 @@ import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -484,17 +485,23 @@ public class PhoenixTransactSQL {
         rowsPerMetric = TimeUnit.MILLISECONDS.toHours(range);
         break;
       case MINUTES:
-        rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range)/2; //2 minute data in METRIC_AGGREGATE_MINUTE
table.
+        rowsPerMetric = TimeUnit.MILLISECONDS.toMinutes(range)/5; //5 minute data in METRIC_AGGREGATE_MINUTE
table.
         break;
       default:
         rowsPerMetric = TimeUnit.MILLISECONDS.toSeconds(range)/10; //10 second data in METRIC_AGGREGATE
table
     }
 
-    long totalRowsRequested = rowsPerMetric * condition.getMetricNames().size();
+    List<String> hostNames = condition.getHostnames();
+    int numHosts = (hostNames == null || hostNames.isEmpty()) ? 1 : condition.getHostnames().size();
+
+    long totalRowsRequested = rowsPerMetric * condition.getMetricNames().size() * numHosts;
+
     if (totalRowsRequested > PhoenixHBaseAccessor.RESULTSET_LIMIT) {
-      throw new PrecisionLimitExceededException("Requested precision (" + precision + ")
for given time range causes " +
-        "result set size of " + totalRowsRequested + ", which exceeds the limit - "
-        + PhoenixHBaseAccessor.RESULTSET_LIMIT + ". Please request higher precision.");
+      throw new PrecisionLimitExceededException("Requested " +  condition.getMetricNames().size()
+ " metrics for "
+        + numHosts + " hosts in " + precision +  " precision for the time range of " + range/1000
+        + " seconds. Estimated resultset size of " + totalRowsRequested + " is greater than
the limit of "
+        + PhoenixHBaseAccessor.RESULTSET_LIMIT + ". Request lower precision or fewer number
of metrics or hosts." +
+        " Alternatively, increase the limit value through ams-site:timeline.metrics.service.default.result.limit
config");
     }
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/602fb5d2/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
index 6bf15c7..9c6617c 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
 
 import org.apache.hadoop.metrics2.sink.timeline.Precision;
+import org.apache.hadoop.metrics2.sink.timeline.PrecisionLimitExceededException;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.DefaultCondition;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL;
@@ -29,8 +30,11 @@ import java.sql.Connection;
 import java.sql.ParameterMetaData;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
+
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
@@ -422,4 +426,132 @@ public class TestPhoenixTransactSQL {
     Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_HOURLY"));
     verify(connection, preparedStatement);
   }
+
+  @Test
+  public void testResultSetLimitCheck() throws SQLException {
+
+    List<String> metrics = new ArrayList<String>();
+    List<String> hosts = new ArrayList<String>();
+    int numMetrics = 0;
+    int numHosts = 0;
+    int limit = PhoenixHBaseAccessor.RESULTSET_LIMIT;
+
+    // 22 Metrics x 2 Hosts x 1 hour with requested SECONDS precision = 15840 points. Should
be OK!
+    numMetrics = 22;
+    numHosts = 2;
+    for (int i = 0; i < numMetrics; i++) {
+      metrics.add("TestMetric"+i);
+    }
+    for (int i = 0; i < numHosts; i++) {
+      hosts.add("TestHost"+i);
+    }
+
+    Condition condition = new DefaultCondition(
+      metrics, hosts,
+      "a1", "i1", 1407950000L, 1407953600L, Precision.SECONDS, null, false);
+    Connection connection = createNiceMock(Connection.class);
+    PreparedStatement preparedStatement = createNiceMock(PreparedStatement.class);
+    Capture<String> stmtCapture = new Capture<String>();
+    expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+      .andReturn(preparedStatement);
+
+    replay(connection, preparedStatement);
+    PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+    String stmt = stmtCapture.getValue();
+    Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+    verify(connection, preparedStatement);
+
+    //Check without passing precision. Should be OK!
+    condition = new DefaultCondition(
+      metrics, hosts,
+      "a1", "i1", 1407950000L, 1407953600L, null, null, false);
+    connection = createNiceMock(Connection.class);
+    preparedStatement = createNiceMock(PreparedStatement.class);
+    stmtCapture = new Capture<String>();
+    expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+      .andReturn(preparedStatement);
+
+    replay(connection, preparedStatement);
+    PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+    stmt = stmtCapture.getValue();
+    Assert.assertTrue(stmt.contains("FROM METRIC_RECORD"));
+    verify(connection, preparedStatement);
+
+    //Check with more hosts and lesser metrics for 1 day data = 11520 points Should be OK!
+    metrics.clear();
+    hosts.clear();
+    numMetrics = 2;
+    numHosts = 20;
+    for (int i = 0; i < numMetrics; i++) {
+      metrics.add("TestMetric"+i);
+    }
+    for (int i = 0; i < numHosts; i++) {
+      hosts.add("TestHost"+i);
+    }
+    condition = new DefaultCondition(
+      metrics, hosts,
+      "a1", "i1", 1407867200L, 1407953600L, null, null, false);
+    connection = createNiceMock(Connection.class);
+    preparedStatement = createNiceMock(PreparedStatement.class);
+    stmtCapture = new Capture<String>();
+    expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+      .andReturn(preparedStatement);
+
+    replay(connection, preparedStatement);
+    PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+    stmt = stmtCapture.getValue();
+    Assert.assertTrue(stmt.contains("FROM METRIC_RECORD_MINUTE"));
+    verify(connection, preparedStatement);
+
+    //Check with 20 metrics, NO hosts and 1 day data = 5760 points. Should be OK!
+    metrics.clear();
+    hosts.clear();
+    numMetrics = 20;
+    for (int i = 0; i < numMetrics; i++) {
+      metrics.add("TestMetric"+i);
+    }
+    condition = new DefaultCondition(
+      metrics, hosts,
+      "a1", "i1", 1407867200L, 1407953600L, null, null, false);
+    connection = createNiceMock(Connection.class);
+    preparedStatement = createNiceMock(PreparedStatement.class);
+    stmtCapture = new Capture<String>();
+    expect(connection.prepareStatement(EasyMock.and(EasyMock.anyString(), EasyMock.capture(stmtCapture))))
+      .andReturn(preparedStatement);
+
+    replay(connection, preparedStatement);
+    PhoenixTransactSQL.prepareGetAggregateSqlStmt(connection, condition);
+    stmt = stmtCapture.getValue();
+    Assert.assertTrue(stmt.contains("FROM METRIC_AGGREGATE_MINUTE"));
+    verify(connection, preparedStatement);
+
+    //Check with 5 hosts and 10 metrics for 1 hour data = 18000 points. Should throw out
Exception!
+    metrics.clear();
+    hosts.clear();
+    numMetrics = 10;
+    numHosts = 5;
+    for (int i = 0; i < numMetrics; i++) {
+      metrics.add("TestMetric"+i);
+    }
+    for (int i = 0; i < numHosts; i++) {
+      hosts.add("TestHost"+i);
+    }
+    condition = new DefaultCondition(
+      metrics, hosts,
+      "a1", "i1", 1407950000L, 1407953600L, null, null, false);
+    boolean exceptionThrown = false;
+    boolean requestedSizeFoundInMessage = false;
+
+    try {
+      PhoenixTransactSQL.prepareGetMetricsSqlStmt(connection, condition);
+    } catch (PrecisionLimitExceededException pe) {
+      exceptionThrown = true;
+      String message = pe.getMessage();
+      if (message !=null && message.contains("18000")) {
+        requestedSizeFoundInMessage = true;
+      }
+    }
+    Assert.assertTrue(exceptionThrown);
+    Assert.assertTrue(requestedSizeFoundInMessage);
+  }
 }


Mime
View raw message