ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject ambari git commit: AMBARI-20056 : FIFO compaction resulting in too many store files on large cluster. (avijayan)
Date Thu, 23 Feb 2017 23:39:42 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk fc37b7376 -> c2c72b9da


AMBARI-20056 : FIFO compaction resulting in too many store files on large cluster. (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c2c72b9d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c2c72b9d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c2c72b9d

Branch: refs/heads/trunk
Commit: c2c72b9da289bf0d172ee103eb0c337a9ffda1df
Parents: fc37b73
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Thu Feb 23 15:39:36 2017 -0800
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Thu Feb 23 15:39:36 2017 -0800

----------------------------------------------------------------------
 .../timeline/HBaseTimelineMetricStore.java      |  18 +-
 .../metrics/timeline/PhoenixHBaseAccessor.java  | 199 +++++++++++--------
 .../timeline/TimelineMetricConfiguration.java   |  25 ++-
 .../timeline/ITPhoenixHBaseAccessor.java        |  21 +-
 .../server/upgrade/UpgradeCatalog250.java       |   7 +
 .../0.1.0/configuration/ams-site.xml            |   8 -
 .../server/upgrade/UpgradeCatalog250Test.java   |  20 +-
 7 files changed, 189 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index 575b8ea..17c58f0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -104,13 +104,17 @@ public class HBaseTimelineMetricStore extends AbstractService implements
Timelin
       hBaseAccessor.initPoliciesAndTTL();
       // Start HA service
       // Start the controller
-      haController = new MetricCollectorHAController(configuration);
-      try {
-        haController.initializeHAController();
-      } catch (Exception e) {
-        LOG.error(e);
-        throw new MetricsSystemInitializationException("Unable to " +
-          "initialize HA controller", e);
+      if (!configuration.isDistributedCollectorModeDisabled()) {
+        haController = new MetricCollectorHAController(configuration);
+        try {
+          haController.initializeHAController();
+        } catch (Exception e) {
+          LOG.error(e);
+          throw new MetricsSystemInitializationException("Unable to " +
+            "initialize HA controller", e);
+        }
+      } else {
+        LOG.info("Distributed collector mode disabled");
       }
 
       //Initialize whitelisting & blacklisting if needed

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index 04405f7..8b0d84b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
 
+import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -81,6 +82,12 @@ import java.util.concurrent.TimeUnit;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
@@ -177,10 +184,12 @@ public class PhoenixHBaseAccessor {
 
   static final String HSTORE_COMPACTION_CLASS_KEY =
     "hbase.hstore.defaultengine.compactionpolicy.class";
+  static final String HSTORE_ENGINE_CLASS =
+    "hbase.hstore.engine.class";
   static final String FIFO_COMPACTION_POLICY_CLASS =
     "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
-  static final String DEFAULT_COMPACTION_POLICY_CLASS =
-    "org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy";
+  static final String DATE_TIERED_COMPACTION_POLICY =
+    "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
   static final String BLOCKING_STORE_FILES_KEY =
     "hbase.hstore.blockingStoreFiles";
 
@@ -503,8 +512,6 @@ public class PhoenixHBaseAccessor {
   }
 
   protected void initPoliciesAndTTL() {
-    boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
-    boolean enableFifoCompaction = metricsConf.getBoolean("timeline.metrics.hbase.fifo.compaction.enabled",
true);
 
     HBaseAdmin hBaseAdmin = null;
     try {
@@ -519,90 +526,23 @@ public class PhoenixHBaseAccessor {
           boolean modifyTable = false;
           HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
 
+          //Set normalizer preferences
+          boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
           if (enableNormalizer ^ tableDescriptor.isNormalizationEnabled()) {
             tableDescriptor.setNormalizationEnabled(enableNormalizer);
             LOG.info("Normalizer set to " + enableNormalizer + " for " + tableName);
             modifyTable = true;
           }
 
-          if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
-            if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
-              LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability +
" for table : " + tableName);
-              boolean validDurability = true;
-              if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
-                tableDescriptor.setDurability(Durability.SKIP_WAL);
-              } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
-                tableDescriptor.setDurability(Durability.SYNC_WAL);
-              } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
-                tableDescriptor.setDurability(Durability.ASYNC_WAL);
-              } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
-                tableDescriptor.setDurability(Durability.FSYNC_WAL);
-              } else {
-                LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY
+ " : " + timelineMetricsPrecisionTableDurability);
-                validDurability = false;
-              }
-              if (validDurability) {
-                modifyTable = true;
-              }
-            }
-          } else {
-            if (!timelineMetricsTablesDurability.isEmpty()) {
-              LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table
: " + tableName);
-              boolean validDurability = true;
-              if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
-                tableDescriptor.setDurability(Durability.SKIP_WAL);
-              } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
-                tableDescriptor.setDurability(Durability.SYNC_WAL);
-              } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
-                tableDescriptor.setDurability(Durability.ASYNC_WAL);
-              } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
-                tableDescriptor.setDurability(Durability.FSYNC_WAL);
-              } else {
-                LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY
+ " : " + timelineMetricsTablesDurability);
-                validDurability = false;
-              }
-              if (validDurability) {
-                modifyTable = true;
-              }
-            }
-          }
+          //Set durability preferences
+          boolean durabilitySettingsModified = setDurabilityForTable(tableName, tableDescriptor);
+          modifyTable = modifyTable || durabilitySettingsModified;
+
+          //Set compaction policy preferences
+          boolean compactionPolicyModified = false;
+          compactionPolicyModified = setCompactionPolicyForTable(tableName, tableDescriptor);
+          modifyTable = modifyTable || compactionPolicyModified;
 
-          Map<String, String> config = tableDescriptor.getConfiguration();
-          if (enableFifoCompaction &&
-             !FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY)))
{
-            tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY,
-              FIFO_COMPACTION_POLICY_CLASS);
-            LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
-              " = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName);
-            // Need to set blockingStoreFiles to 1000 for FIFO
-            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000);
-            if (blockingStoreFiles < 1000) {
-              blockingStoreFiles = 1000;
-            }
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
-            LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + blockingStoreFiles + " for " + tableName);
-            modifyTable = true;
-          }
-          // Set back original policy if fifo disabled
-          if (!enableFifoCompaction &&
-             FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY)))
{
-            tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY,
-              DEFAULT_COMPACTION_POLICY_CLASS);
-            LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
-              " = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName);
-
-            int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300);
-            if (blockingStoreFiles > 300) {
-              LOG.warn("HBase blocking store file set too high without FIFO " +
-                "Compaction policy enabled, restoring low value = 300.");
-              blockingStoreFiles = 300;
-            }
-            tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
-            LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
-              " = " + blockingStoreFiles + " for " + tableName);
-            modifyTable = true;
-          }
           // Change TTL setting to match user configuration
           HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
           if (columnFamilies != null) {
@@ -635,6 +575,103 @@ public class PhoenixHBaseAccessor {
     }
   }
 
+  private boolean setDurabilityForTable(String tableName, HTableDescriptor tableDescriptor)
{
+
+    boolean modifyTable = false;
+    //Set WAL preferences
+    if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
+      if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
+        LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for
table : " + tableName);
+        boolean validDurability = true;
+        if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+          tableDescriptor.setDurability(Durability.SKIP_WAL);
+        } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+          tableDescriptor.setDurability(Durability.SYNC_WAL);
+        } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+          tableDescriptor.setDurability(Durability.ASYNC_WAL);
+        } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+          tableDescriptor.setDurability(Durability.FSYNC_WAL);
+        } else {
+          LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + "
: " + timelineMetricsPrecisionTableDurability);
+          validDurability = false;
+        }
+        if (validDurability) {
+          modifyTable = true;
+        }
+      }
+    } else {
+      if (!timelineMetricsTablesDurability.isEmpty()) {
+        LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table :
" + tableName);
+        boolean validDurability = true;
+        if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
+          tableDescriptor.setDurability(Durability.SKIP_WAL);
+        } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
+          tableDescriptor.setDurability(Durability.SYNC_WAL);
+        } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
+          tableDescriptor.setDurability(Durability.ASYNC_WAL);
+        } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
+          tableDescriptor.setDurability(Durability.FSYNC_WAL);
+        } else {
+          LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY +
" : " + timelineMetricsTablesDurability);
+          validDurability = false;
+        }
+        if (validDurability) {
+          modifyTable = true;
+        }
+      }
+    }
+    return modifyTable;
+  }
+
+  private boolean setCompactionPolicyForTable(String tableName, HTableDescriptor tableDescriptor)
{
+
+    String compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY,
+      HSTORE_ENGINE_CLASS);
+    String compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS,
+      DATE_TIERED_COMPACTION_POLICY);
+    int blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES,
60);
+
+    if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+      compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY,
+        HSTORE_COMPACTION_CLASS_KEY);
+      compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS,
+        FIFO_COMPACTION_POLICY_CLASS);
+      blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES,
1000);
+    }
+
+    Map<String, String> config = new HashMap(tableDescriptor.getConfiguration());
+
+    if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass))
{
+      config.remove(HSTORE_COMPACTION_CLASS_KEY);
+      config.remove(HSTORE_ENGINE_CLASS);
+      //Default blockingStoreFiles = 300
+      setHbaseBlockingStoreFiles(tableDescriptor, tableName, 300);
+    } else {
+      tableDescriptor.setConfiguration(compactionPolicyKey, compactionPolicyClass);
+      setHbaseBlockingStoreFiles(tableDescriptor, tableName, blockingStoreFiles);
+    }
+
+    if (!compactionPolicyKey.equals(HSTORE_ENGINE_CLASS)) {
+      tableDescriptor.removeConfiguration(HSTORE_ENGINE_CLASS);
+    }
+    if (!compactionPolicyKey.equals(HSTORE_COMPACTION_CLASS_KEY)) {
+      tableDescriptor.removeConfiguration(HSTORE_COMPACTION_CLASS_KEY);
+    }
+
+    Map<String, String> newConfig = tableDescriptor.getConfiguration();
+    return !Maps.difference(config, newConfig).areEqual();
+  }
+
+  private void setHbaseBlockingStoreFiles(HTableDescriptor tableDescriptor, String tableName,
int value) {
+    int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, value);
+    if (blockingStoreFiles != value) {
+      blockingStoreFiles = value;
+    }
+    tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
+    LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
+      " = " + blockingStoreFiles + " for " + tableName);
+  }
+
   protected String getSplitPointsStr(String splitPoints) {
     if (StringUtils.isEmpty(splitPoints.trim())) {
       return "";

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index b7b9968..72e895d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -269,6 +269,24 @@ public class TimelineMetricConfiguration {
   public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
     "timeline.metrics.cluster.aggregation.sql.filters";
 
+  public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY
=
+    "timeline.metrics.hbase.aggregate.table.compaction.policy.key";
+
+  public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS
=
+    "timeline.metrics.hbase.aggregate.table.compaction.policy.class";
+
+  public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES
=
+    "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles";
+
+  public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY
=
+    "timeline.metrics.hbase.precision.table.compaction.policy.key";
+
+  public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS
=
+    "timeline.metrics.hbase.precision.table.compaction.policy.class";
+
+  public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES
=
+    "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+
   public static final String HOST_APP_ID = "HOST";
 
   public static final String DEFAULT_INSTANCE_PORT = "12001";
@@ -425,9 +443,12 @@ public class TimelineMetricConfiguration {
     return defaultRpcAddress;
   }
 
-  public boolean isDistributedOperationModeEnabled() {
+  public boolean isDistributedCollectorModeDisabled() {
     try {
-      return getMetricsConf().get("timeline.metrics.service.operation.mode").equals("distributed");
+      if (metricsConf != null) {
+        return Boolean.parseBoolean(metricsConf.get("timeline.metrics.service.distributed.collector.mode.disabled",
"false"));
+      }
+      return false;
     } catch (Exception e) {
       return false;
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
index 3bc1929..0087fd9 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
@@ -62,8 +62,10 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createMetricHostAggregate;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
 import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
@@ -327,8 +329,6 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest
{
       HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
       tableDescriptor.setNormalizationEnabled(true);
       Assert.assertTrue("Normalizer enabled.", tableDescriptor.isNormalizationEnabled());
-      Assert.assertNull("Default compaction policy is null.",
-        tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY));
 
       for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) {
         if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
@@ -352,7 +352,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest
{
 
     // Verify expected policies are set
     boolean normalizerEnabled = false;
-    String compactionPolicy = null;
+    String precisionTableCompactionPolicy = null;
+    String aggregateTableCompactionPolicy = null;
     boolean tableDurabilitySet  = false;
     for (int i = 0; i < 10; i++) {
       LOG.warn("Policy check retry : " + i);
@@ -360,12 +361,15 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest
{
         HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
         normalizerEnabled = tableDescriptor.isNormalizationEnabled();
         tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
-        compactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY);
+        if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+          precisionTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_ENGINE_CLASS);
+        } else {
+          aggregateTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY);
+        }
         LOG.debug("Table: " + tableName + ", normalizerEnabled = " + normalizerEnabled);
-        LOG.debug("Table: " + tableName + ", compactionPolicy = " + compactionPolicy);
         // Best effort for 20 seconds
-        if (normalizerEnabled || compactionPolicy == null) {
-          Thread.sleep(2000l);
+        if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy
==null)) {
+          Thread.sleep(20000l);
         }
         if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
           for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) {
@@ -377,7 +381,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest
{
 
     Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
     Assert.assertTrue("Durability Set.", tableDurabilitySet);
-    Assert.assertEquals("FIFO compaction policy is set.", FIFO_COMPACTION_POLICY_CLASS, compactionPolicy);
+    Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS,
precisionTableCompactionPolicy);
+    Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY,
aggregateTableCompactionPolicy);
     Assert.assertEquals("Precision TTL value not changed.", String.valueOf(2 * 86400), precisionTtl);
 
     hBaseAdmin.close();

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index bfab0fe..b0243b7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -381,6 +381,13 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
             if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) {
               isDistributed = true;
             }
+
+            Map<String, String> amsSiteProperties = amsSite.getProperties();
+
+            if (amsSiteProperties != null && amsSiteProperties.containsKey("timeline.metrics.hbase.fifo.compaction.enabled"))
{
+              LOG.info("Removing timeline.metrics.hbase.fifo.compaction.enabled from ams-site");
+              removeConfigurationPropertiesFromCluster(cluster, AMS_SITE, Collections.singleton("timeline.metrics.hbase.fifo.compaction.enabled"));
+            }
           }
 
           if (isDistributed) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index 45fe1a7..8e1671e 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -651,14 +651,6 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>timeline.metrics.hbase.fifo.compaction.enabled</name>
-    <value>true</value>
-    <description>
-      Enable Compaction policy for lower for Precision and Minute aggregate tables.
-    </description>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
     <name>timeline.metrics.aggregators.skip.blockcache.enabled</name>
     <value>false</value>
     <description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index a08b38b..274d7eb 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -67,6 +67,7 @@ import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.easymock.Capture;
+import org.easymock.CaptureType;
 import org.easymock.EasyMock;
 import org.easymock.EasyMockRunner;
 import org.easymock.EasyMockSupport;
@@ -643,8 +644,16 @@ public class UpgradeCatalog250Test {
     Map<String, String> amsSite = new HashMap<String, String>() {
       {
         put("timeline.metrics.service.operation.mode", "distributed");
+        put("timeline.metrics.hbase.fifo.compaction.enabled", "true");
       }
     };
+
+    Map<String, String> newAmsSite = new HashMap<String, String>() {
+      {
+        put("timeline.metrics.service.operation.mode", "distributed");
+      }
+    };
+
     EasyMockSupport easyMockSupport = new EasyMockSupport();
 
     Config mockAmsHbaseSite = easyMockSupport.createNiceMock(Config.class);
@@ -670,18 +679,23 @@ public class UpgradeCatalog250Test {
       .createNiceMock();
 
     Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
-    Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture();
+    Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(CaptureType.ALL);
 
     expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
     expect(controller.getClusters()).andReturn(clusters).anyTimes();
     expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture),
anyString(),
-        EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+        EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).times(2);
 
     replay(controller, injector2);
     new UpgradeCatalog250(injector2).updateAMSConfigs();
     easyMockSupport.verifyAll();
 
-    Map<String, String> updatedProperties = propertiesCapture.getValue();
+    assertTrue(propertiesCapture.getValues().size() == 2);
+
+    Map<String, String> updatedProperties = propertiesCapture.getValues().get(0);
+    assertTrue(Maps.difference(newAmsSite, updatedProperties).areEqual());
+
+    updatedProperties = propertiesCapture.getValues().get(1);
     assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
   }
 


Mime
View raw message