hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sj...@apache.org
Subject [2/2] hadoop git commit: YARN-6094. Update the coprocessor to be a dynamically loaded one. Contributed by Vrushali C.
Date Thu, 19 Jan 2017 23:05:58 GMT
YARN-6094. Update the coprocessor to be a dynamically loaded one. Contributed by Vrushali C.

(cherry picked from commit 0327a79d79a4d56d9c7cb6889886afd2272b07d3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0177c95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0177c95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0177c95

Branch: refs/heads/YARN-5355-branch-2
Commit: e0177c952c3b64c8dcf0408562faa98f725280e0
Parents: c11078f
Author: Sangjin Lee <sjlee@apache.org>
Authored: Thu Jan 19 14:52:47 2017 -0800
Committer: Sangjin Lee <sjlee@apache.org>
Committed: Thu Jan 19 14:55:55 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  12 +
 .../src/main/resources/yarn-default.xml         |   9 +
 ...stTimelineReaderWebServicesHBaseStorage.java |   4 +-
 .../storage/DataGeneratorForTest.java           | 364 ++++++++++++-------
 .../storage/TestHBaseTimelineStorageApps.java   |   6 +-
 .../TestHBaseTimelineStorageEntities.java       |   6 +-
 .../storage/TestHBaseTimelineStorageSchema.java |  12 +-
 .../storage/flow/TestFlowDataGenerator.java     |  28 +-
 .../flow/TestHBaseStorageFlowActivity.java      |   8 +-
 .../storage/flow/TestHBaseStorageFlowRun.java   |  46 ++-
 .../flow/TestHBaseStorageFlowRunCompaction.java |   8 +-
 .../storage/flow/FlowRunCoprocessor.java        |  36 +-
 .../storage/flow/FlowRunTable.java              |  33 +-
 .../src/site/markdown/TimelineServiceV2.md      |  26 +-
 14 files changed, 322 insertions(+), 276 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 659b5eb..e1ecaf9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2058,6 +2058,18 @@ public class YarnConfiguration extends Configuration {
       + "hbase.coprocessor.app-final-value-retention-milliseconds";
 
   /**
+   * The name of the setting for the location of the coprocessor
+   * jar on hdfs.
+   */
+  public static final String FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION =
+      TIMELINE_SERVICE_PREFIX
+      + "hbase.coprocessor.jar.hdfs.location";
+
+  /** default hdfs location for flowrun coprocessor jar. */
+  public static final String DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR =
+      "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar";
+
+    /**
    * The name for setting that points to an optional HBase configuration
    * (hbase-site.xml file) with settings that will override the ones found on
    * the classpath.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 416d705..a21ee2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2272,6 +2272,15 @@
 
   <property>
     <description>
+    The default hdfs location for flowrun coprocessor jar.
+    </description>
+    <name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location
+    </name>
+    <value>/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar</value>
+  </property>
+
+  <property>
+    <description>
     The value of this parameter sets the prefix for all tables that are part of
     timeline service in the hbase storage schema. It can be set to "dev."
     or "staging." if it is to be used for development or staging instances.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 485c191..d9a7078 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
@@ -89,7 +89,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
     Configuration conf = util.getConfiguration();
     conf.setInt("hfile.format.version", 3);
     util.startMiniCluster();
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+    DataGeneratorForTest.createSchema(conf);
     loadData();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
index cafacab..1a522fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/DataGeneratorForTest.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -31,10 +32,33 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 
-final class DataGeneratorForTest {
-  static void loadApps(HBaseTestingUtility util) throws IOException {
+/**
+ * Utility class that creates the schema and generates test data.
+ */
+public final class DataGeneratorForTest {
+
+  // private constructor for utility class
+  private DataGeneratorForTest() {
+  }
+
+   /**
+   * Creates the schema for timeline service.
+   * @param conf
+   * @throws IOException
+   */
+  public static void createSchema(final Configuration conf)
+      throws IOException {
+    // set the jar location to null so that
+    // the coprocessor class is loaded from classpath
+    conf.set(YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION, " ");
+    // now create all tables
+    TimelineSchemaCreator.createAllTables(conf, false);
+  }
+
+  public static void loadApps(HBaseTestingUtility util) throws IOException {
     TimelineEntities te = new TimelineEntities();
     TimelineEntity entity = new TimelineEntity();
     String id = "application_1111111111_2222";
@@ -43,11 +67,7 @@ final class DataGeneratorForTest {
     Long cTime = 1425016502000L;
     entity.setCreatedTime(cTime);
     // add the info map in Timeline Entity
-    Map<String, Object> infoMap = new HashMap<>();
-    infoMap.put("infoMapKey1", "infoMapValue2");
-    infoMap.put("infoMapKey2", 20);
-    infoMap.put("infoMapKey3", 85.85);
-    entity.addInfo(infoMap);
+    entity.addInfo(getInfoMap3());
     // add the isRelatedToEntity info
     Set<String> isRelatedToSet = new HashSet<>();
     isRelatedToSet.add("relatedto1");
@@ -72,29 +92,15 @@ final class DataGeneratorForTest {
     entity.addConfigs(conf);
     // add metrics
     Set<TimelineMetric> metrics = new HashSet<>();
-    TimelineMetric m1 = new TimelineMetric();
-    m1.setId("MAP_SLOT_MILLIS");
-    Map<Long, Number> metricValues = new HashMap<>();
     long ts = System.currentTimeMillis();
-    metricValues.put(ts - 120000, 100000000);
-    metricValues.put(ts - 100000, 200000000);
-    metricValues.put(ts - 80000, 300000000);
-    metricValues.put(ts - 60000, 400000000);
-    metricValues.put(ts - 40000, 50000000000L);
-    metricValues.put(ts - 20000, 60000000000L);
-    m1.setType(Type.TIME_SERIES);
-    m1.setValues(metricValues);
-    metrics.add(m1);
+    metrics.add(getMetric4(ts));
 
     TimelineMetric m12 = new TimelineMetric();
     m12.setId("MAP1_BYTES");
     m12.addValue(ts, 50);
     metrics.add(m12);
     entity.addMetrics(metrics);
-    TimelineEvent event = new TimelineEvent();
-    event.setId("start_event");
-    event.setTimestamp(ts);
-    entity.addEvent(event);
+    entity.addEvent(addStartEvent(ts));
     te.addEntity(entity);
     TimelineEntities te1 = new TimelineEntities();
     TimelineEntity entity1 = new TimelineEntity();
@@ -103,10 +109,7 @@ final class DataGeneratorForTest {
     entity1.setType(TimelineEntityType.YARN_APPLICATION.toString());
     entity1.setCreatedTime(cTime + 20L);
     // add the info map in Timeline Entity
-    Map<String, Object> infoMap1 = new HashMap<>();
-    infoMap1.put("infoMapKey1", "infoMapValue1");
-    infoMap1.put("infoMapKey2", 10);
-    entity1.addInfo(infoMap1);
+    entity1.addInfo(getInfoMap4());
 
     // add the isRelatedToEntity info
     Set<String> isRelatedToSet1 = new HashSet<>();
@@ -134,21 +137,7 @@ final class DataGeneratorForTest {
     entity1.addConfigs(conf1);
 
     // add metrics
-    Set<TimelineMetric> metrics1 = new HashSet<>();
-    TimelineMetric m2 = new TimelineMetric();
-    m2.setId("MAP1_SLOT_MILLIS");
-    Map<Long, Number> metricValues1 = new HashMap<>();
-    long ts1 = System.currentTimeMillis();
-    metricValues1.put(ts1 - 120000, 100000000);
-    metricValues1.put(ts1 - 100000, 200000000);
-    metricValues1.put(ts1 - 80000, 300000000);
-    metricValues1.put(ts1 - 60000, 400000000);
-    metricValues1.put(ts1 - 40000, 50000000000L);
-    metricValues1.put(ts1 - 20000, 60000000000L);
-    m2.setType(Type.TIME_SERIES);
-    m2.setValues(metricValues1);
-    metrics1.add(m2);
-    entity1.addMetrics(metrics1);
+    entity1.addMetrics(getMetrics4());
     TimelineEvent event11 = new TimelineEvent();
     event11.setId("end_event");
     event11.setTimestamp(ts);
@@ -160,27 +149,7 @@ final class DataGeneratorForTest {
     te1.addEntity(entity1);
 
     TimelineEntities te2 = new TimelineEntities();
-    TimelineEntity entity2 = new TimelineEntity();
-    String id2 = "application_1111111111_4444";
-    entity2.setId(id2);
-    entity2.setType(TimelineEntityType.YARN_APPLICATION.toString());
-    entity2.setCreatedTime(cTime + 40L);
-    TimelineEvent event21 = new TimelineEvent();
-    event21.setId("update_event");
-    event21.setTimestamp(ts - 20);
-    entity2.addEvent(event21);
-    Set<String> isRelatedToSet2 = new HashSet<String>();
-    isRelatedToSet2.add("relatedto3");
-    Map<String, Set<String>> isRelatedTo2 = new HashMap<>();
-    isRelatedTo2.put("task1", isRelatedToSet2);
-    entity2.setIsRelatedToEntities(isRelatedTo2);
-    Map<String, Set<String>> relatesTo3 = new HashMap<>();
-    Set<String> relatesToSet14 = new HashSet<String>();
-    relatesToSet14.add("relatesto7");
-    relatesTo3.put("container2", relatesToSet14);
-    entity2.setRelatesToEntities(relatesTo3);
-
-    te2.addEntity(entity2);
+    te2.addEntity(getEntity4(cTime, ts));
     HBaseTimelineWriterImpl hbi = null;
     try {
       hbi = new HBaseTimelineWriterImpl();
@@ -206,29 +175,86 @@ final class DataGeneratorForTest {
     }
   }
 
-  static void loadEntities(HBaseTestingUtility util) throws IOException {
-    TimelineEntities te = new TimelineEntities();
-    TimelineEntity entity = new TimelineEntity();
-    String id = "hello";
-    String type = "world";
-    entity.setId(id);
-    entity.setType(type);
-    Long cTime = 1425016502000L;
-    entity.setCreatedTime(cTime);
-    // add the info map in Timeline Entity
+  private static Set<TimelineMetric> getMetrics4() {
+    Set<TimelineMetric> metrics1 = new HashSet<>();
+    TimelineMetric m2 = new TimelineMetric();
+    m2.setId("MAP1_SLOT_MILLIS");
+    long ts1 = System.currentTimeMillis();
+    Map<Long, Number> metricValues1 = new HashMap<>();
+    metricValues1.put(ts1 - 120000, 100000000);
+    metricValues1.put(ts1 - 100000, 200000000);
+    metricValues1.put(ts1 - 80000, 300000000);
+    metricValues1.put(ts1 - 60000, 400000000);
+    metricValues1.put(ts1 - 40000, 50000000000L);
+    metricValues1.put(ts1 - 20000, 60000000000L);
+    m2.setType(Type.TIME_SERIES);
+    m2.setValues(metricValues1);
+    metrics1.add(m2);
+    return metrics1;
+  }
+
+  private static TimelineEntity getEntity4(long cTime, long ts) {
+    TimelineEntity entity2 = new TimelineEntity();
+    String id2 = "application_1111111111_4444";
+    entity2.setId(id2);
+    entity2.setType(TimelineEntityType.YARN_APPLICATION.toString());
+    entity2.setCreatedTime(cTime + 40L);
+    TimelineEvent event21 = new TimelineEvent();
+    event21.setId("update_event");
+    event21.setTimestamp(ts - 20);
+    entity2.addEvent(event21);
+    Set<String> isRelatedToSet2 = new HashSet<String>();
+    isRelatedToSet2.add("relatedto3");
+    Map<String, Set<String>> isRelatedTo2 = new HashMap<>();
+    isRelatedTo2.put("task1", isRelatedToSet2);
+    entity2.setIsRelatedToEntities(isRelatedTo2);
+    Map<String, Set<String>> relatesTo3 = new HashMap<>();
+    Set<String> relatesToSet14 = new HashSet<String>();
+    relatesToSet14.add("relatesto7");
+    relatesTo3.put("container2", relatesToSet14);
+    entity2.setRelatesToEntities(relatesTo3);
+    return entity2;
+  }
+
+  private static Map<String, Object> getInfoMap4() {
+    Map<String, Object> infoMap1 = new HashMap<>();
+    infoMap1.put("infoMapKey1", "infoMapValue1");
+    infoMap1.put("infoMapKey2", 10);
+    return infoMap1;
+  }
+
+  private static TimelineMetric getMetric4(long ts) {
+    TimelineMetric m1 = new TimelineMetric();
+    m1.setId("MAP_SLOT_MILLIS");
+    Map<Long, Number> metricValues = new HashMap<>();
+    metricValues.put(ts - 120000, 100000000);
+    metricValues.put(ts - 100000, 200000000);
+    metricValues.put(ts - 80000, 300000000);
+    metricValues.put(ts - 60000, 400000000);
+    metricValues.put(ts - 40000, 50000000000L);
+    metricValues.put(ts - 20000, 60000000000L);
+    m1.setType(Type.TIME_SERIES);
+    m1.setValues(metricValues);
+    return m1;
+  }
+
+  private static Map<String, Object> getInfoMap3() {
+    Map<String, Object> infoMap = new HashMap<>();
+    infoMap.put("infoMapKey1", "infoMapValue2");
+    infoMap.put("infoMapKey2", 20);
+    infoMap.put("infoMapKey3", 85.85);
+    return infoMap;
+  }
+
+  private static Map<String, Object> getInfoMap1() {
     Map<String, Object> infoMap = new HashMap<>();
     infoMap.put("infoMapKey1", "infoMapValue2");
     infoMap.put("infoMapKey2", 20);
     infoMap.put("infoMapKey3", 71.4);
-    entity.addInfo(infoMap);
-    // add the isRelatedToEntity info
-    Set<String> isRelatedToSet = new HashSet<>();
-    isRelatedToSet.add("relatedto1");
-    Map<String, Set<String>> isRelatedTo = new HashMap<>();
-    isRelatedTo.put("task", isRelatedToSet);
-    entity.setIsRelatedToEntities(isRelatedTo);
+    return infoMap;
+  }
 
-    // add the relatesTo info
+  private static Map<String, Set<String>> getRelatesTo1() {
     Set<String> relatesToSet = new HashSet<String>();
     relatesToSet.add("relatesto1");
     relatesToSet.add("relatesto3");
@@ -237,29 +263,78 @@ final class DataGeneratorForTest {
     Set<String> relatesToSet11 = new HashSet<>();
     relatesToSet11.add("relatesto4");
     relatesTo.put("container1", relatesToSet11);
-    entity.setRelatesToEntities(relatesTo);
+    return relatesTo;
+  }
 
-    // add some config entries
+  private static Map<String, String> getConfig1() {
     Map<String, String> conf = new HashMap<>();
     conf.put("config_param1", "value1");
     conf.put("config_param2", "value2");
     conf.put("cfg_param1", "value3");
-    entity.addConfigs(conf);
+    return conf;
+  }
 
-    // add metrics
-    Set<TimelineMetric> metrics = new HashSet<>();
-    TimelineMetric m1 = new TimelineMetric();
-    m1.setId("MAP_SLOT_MILLIS");
+  private static Map<String, String> getConfig2() {
+    Map<String, String> conf1 = new HashMap<>();
+    conf1.put("cfg_param1", "value1");
+    conf1.put("cfg_param2", "value2");
+    return conf1;
+  }
+
+  private static Map<String, Object> getInfoMap2() {
+    Map<String, Object> infoMap1 = new HashMap<>();
+    infoMap1.put("infoMapKey1", "infoMapValue1");
+    infoMap1.put("infoMapKey2", 10);
+    return infoMap1;
+  }
+
+  private static Map<String, Set<String>> getIsRelatedTo1() {
+    Set<String> isRelatedToSet = new HashSet<>();
+    isRelatedToSet.add("relatedto1");
+    Map<String, Set<String>> isRelatedTo = new HashMap<>();
+    isRelatedTo.put("task", isRelatedToSet);
+    return isRelatedTo;
+  }
+
+  private static Map<Long, Number> getMetricValues1(long ts) {
     Map<Long, Number> metricValues = new HashMap<>();
-    long ts = System.currentTimeMillis();
     metricValues.put(ts - 120000, 100000000);
     metricValues.put(ts - 100000, 200000000);
     metricValues.put(ts - 80000, 300000000);
     metricValues.put(ts - 60000, 400000000);
     metricValues.put(ts - 40000, 50000000000L);
     metricValues.put(ts - 20000, 70000000000L);
+    return metricValues;
+  }
+
+  public static void loadEntities(HBaseTestingUtility util)
+      throws IOException {
+    TimelineEntities te = new TimelineEntities();
+    TimelineEntity entity = new TimelineEntity();
+    String id = "hello";
+    String type = "world";
+    entity.setId(id);
+    entity.setType(type);
+    Long cTime = 1425016502000L;
+    entity.setCreatedTime(cTime);
+    // add the info map in Timeline Entity
+    entity.addInfo(getInfoMap1());
+    // add the isRelatedToEntity info
+    entity.setIsRelatedToEntities(getIsRelatedTo1());
+
+    // add the relatesTo info
+    entity.setRelatesToEntities(getRelatesTo1());
+
+    // add some config entries
+    entity.addConfigs(getConfig1());
+
+    // add metrics
+    Set<TimelineMetric> metrics = new HashSet<>();
+    TimelineMetric m1 = new TimelineMetric();
+    m1.setId("MAP_SLOT_MILLIS");
+    long ts = System.currentTimeMillis();
     m1.setType(Type.TIME_SERIES);
-    m1.setValues(metricValues);
+    m1.setValues(getMetricValues1(ts));
     metrics.add(m1);
 
     TimelineMetric m12 = new TimelineMetric();
@@ -267,10 +342,7 @@ final class DataGeneratorForTest {
     m12.addValue(ts, 50);
     metrics.add(m12);
     entity.addMetrics(metrics);
-    TimelineEvent event = new TimelineEvent();
-    event.setId("start_event");
-    event.setTimestamp(ts);
-    entity.addEvent(event);
+    entity.addEvent(addStartEvent(ts));
     te.addEntity(entity);
 
     TimelineEntity entity1 = new TimelineEntity();
@@ -280,10 +352,7 @@ final class DataGeneratorForTest {
     entity1.setCreatedTime(cTime + 20L);
 
     // add the info map in Timeline Entity
-    Map<String, Object> infoMap1 = new HashMap<>();
-    infoMap1.put("infoMapKey1", "infoMapValue1");
-    infoMap1.put("infoMapKey2", 10);
-    entity1.addInfo(infoMap1);
+    entity1.addInfo(getInfoMap2());
 
     // add event.
     TimelineEvent event11 = new TimelineEvent();
@@ -297,15 +366,7 @@ final class DataGeneratorForTest {
 
 
     // add the isRelatedToEntity info
-    Set<String> isRelatedToSet1 = new HashSet<>();
-    isRelatedToSet1.add("relatedto3");
-    isRelatedToSet1.add("relatedto5");
-    Map<String, Set<String>> isRelatedTo1 = new HashMap<>();
-    isRelatedTo1.put("task1", isRelatedToSet1);
-    Set<String> isRelatedToSet11 = new HashSet<>();
-    isRelatedToSet11.add("relatedto4");
-    isRelatedTo1.put("task2", isRelatedToSet11);
-    entity1.setIsRelatedToEntities(isRelatedTo1);
+    entity1.setIsRelatedToEntities(getIsRelatedTo2());
 
     // add the relatesTo info
     Set<String> relatesToSet1 = new HashSet<String>();
@@ -316,49 +377,20 @@ final class DataGeneratorForTest {
     entity1.setRelatesToEntities(relatesTo1);
 
     // add some config entries
-    Map<String, String> conf1 = new HashMap<>();
-    conf1.put("cfg_param1", "value1");
-    conf1.put("cfg_param2", "value2");
-    entity1.addConfigs(conf1);
+    entity1.addConfigs(getConfig2());
 
     // add metrics
     Set<TimelineMetric> metrics1 = new HashSet<>();
     TimelineMetric m2 = new TimelineMetric();
     m2.setId("MAP1_SLOT_MILLIS");
-    Map<Long, Number> metricValues1 = new HashMap<>();
     long ts1 = System.currentTimeMillis();
-    metricValues1.put(ts1 - 120000, 100000000);
-    metricValues1.put(ts1 - 100000, 200000000);
-    metricValues1.put(ts1 - 80000, 300000000);
-    metricValues1.put(ts1 - 60000, 400000000);
-    metricValues1.put(ts1 - 40000, 50000000000L);
-    metricValues1.put(ts1 - 20000, 60000000000L);
     m2.setType(Type.TIME_SERIES);
-    m2.setValues(metricValues1);
+    m2.setValues(getMetricValues2(ts1));
     metrics1.add(m2);
     entity1.addMetrics(metrics1);
     te.addEntity(entity1);
 
-    TimelineEntity entity2 = new TimelineEntity();
-    String id2 = "hello2";
-    entity2.setId(id2);
-    entity2.setType(type);
-    entity2.setCreatedTime(cTime + 40L);
-    TimelineEvent event21 = new TimelineEvent();
-    event21.setId("update_event");
-    event21.setTimestamp(ts - 20);
-    entity2.addEvent(event21);
-    Set<String> isRelatedToSet2 = new HashSet<>();
-    isRelatedToSet2.add("relatedto3");
-    Map<String, Set<String>> isRelatedTo2 = new HashMap<>();
-    isRelatedTo2.put("task1", isRelatedToSet2);
-    entity2.setIsRelatedToEntities(isRelatedTo2);
-    Map<String, Set<String>> relatesTo3 = new HashMap<>();
-    Set<String> relatesToSet14 = new HashSet<>();
-    relatesToSet14.add("relatesto7");
-    relatesTo3.put("container2", relatesToSet14);
-    entity2.setRelatesToEntities(relatesTo3);
-    te.addEntity(entity2);
+    te.addEntity(getEntity2(type, cTime, ts));
 
     // For listing types
     for (int i = 0; i < 10; i++) {
@@ -421,4 +453,58 @@ final class DataGeneratorForTest {
       }
     }
   }
+
+  private static TimelineEntity getEntity2(String type, long cTime,
+      long ts) {
+    TimelineEntity entity2 = new TimelineEntity();
+    String id2 = "hello2";
+    entity2.setId(id2);
+    entity2.setType(type);
+    entity2.setCreatedTime(cTime + 40L);
+    TimelineEvent event21 = new TimelineEvent();
+    event21.setId("update_event");
+    event21.setTimestamp(ts - 20);
+    entity2.addEvent(event21);
+    Set<String> isRelatedToSet2 = new HashSet<>();
+    isRelatedToSet2.add("relatedto3");
+    Map<String, Set<String>> isRelatedTo2 = new HashMap<>();
+    isRelatedTo2.put("task1", isRelatedToSet2);
+    entity2.setIsRelatedToEntities(isRelatedTo2);
+    Map<String, Set<String>> relatesTo3 = new HashMap<>();
+    Set<String> relatesToSet14 = new HashSet<>();
+    relatesToSet14.add("relatesto7");
+    relatesTo3.put("container2", relatesToSet14);
+    entity2.setRelatesToEntities(relatesTo3);
+    return entity2;
+  }
+
+  private static TimelineEvent addStartEvent(long ts) {
+    TimelineEvent event = new TimelineEvent();
+    event.setId("start_event");
+    event.setTimestamp(ts);
+    return event;
+  }
+
+  private static Map<Long, Number> getMetricValues2(long ts1) {
+    Map<Long, Number> metricValues1 = new HashMap<>();
+    metricValues1.put(ts1 - 120000, 100000000);
+    metricValues1.put(ts1 - 100000, 200000000);
+    metricValues1.put(ts1 - 80000, 300000000);
+    metricValues1.put(ts1 - 60000, 400000000);
+    metricValues1.put(ts1 - 40000, 50000000000L);
+    metricValues1.put(ts1 - 20000, 60000000000L);
+    return metricValues1;
+  }
+
+  private static Map<String, Set<String>> getIsRelatedTo2() {
+    Set<String> isRelatedToSet1 = new HashSet<>();
+    isRelatedToSet1.add("relatedto3");
+    isRelatedToSet1.add("relatedto5");
+    Map<String, Set<String>> isRelatedTo1 = new HashMap<>();
+    isRelatedTo1.put("task1", isRelatedToSet1);
+    Set<String> isRelatedToSet11 = new HashSet<>();
+    isRelatedToSet11.add("relatedto4");
+    isRelatedTo1.put("task2", isRelatedToSet11);
+    return isRelatedTo1;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index b3e5197..31d56b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -91,14 +91,10 @@ public class TestHBaseTimelineStorageApps {
   public static void setupBeforeClass() throws Exception {
     util = new HBaseTestingUtility();
     util.startMiniCluster();
-    createSchema();
+    DataGeneratorForTest.createSchema(util.getConfiguration());
     DataGeneratorForTest.loadApps(util);
   }
 
-  private static void createSchema() throws IOException {
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
-  }
-
   @Before
   public void init() throws Exception {
     reader = new HBaseTimelineReaderImpl();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index d9ec5b4..b188802 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -98,14 +98,10 @@ public class TestHBaseTimelineStorageEntities {
   public static void setupBeforeClass() throws Exception {
     util = new HBaseTestingUtility();
     util.startMiniCluster();
-    createSchema();
+    DataGeneratorForTest.createSchema(util.getConfiguration());
     DataGeneratorForTest.loadEntities(util);
   }
 
-  private static void createSchema() throws IOException {
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
-  }
-
   @Before
   public void init() throws Exception {
     reader = new HBaseTimelineReaderImpl();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
index 53045e5..0dcd171 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
@@ -48,17 +48,15 @@ public class TestHBaseTimelineStorageSchema {
   @BeforeClass
   public static void setupBeforeClass() throws Exception {
     util = new HBaseTestingUtility();
+    Configuration conf = util.getConfiguration();
+    conf.setInt("hfile.format.version", 3);
     util.startMiniCluster();
   }
 
-  private static void createSchema(Configuration conf) throws IOException {
-    TimelineSchemaCreator.createAllTables(conf, false);
-  }
-
   @Test
   public void createWithDefaultPrefix() throws IOException {
     Configuration hbaseConf = util.getConfiguration();
-    createSchema(hbaseConf);
+    DataGeneratorForTest.createSchema(hbaseConf);
     Connection conn = null;
     conn = ConnectionFactory.createConnection(hbaseConf);
     Admin admin = conn.getAdmin();
@@ -88,7 +86,7 @@ public class TestHBaseTimelineStorageSchema {
     String prefix = "unit-test.";
     hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME,
         prefix);
-    createSchema(hbaseConf);
+    DataGeneratorForTest.createSchema(hbaseConf);
     Connection conn = null;
     conn = ConnectionFactory.createConnection(hbaseConf);
     Admin admin = conn.getAdmin();
@@ -115,7 +113,7 @@ public class TestHBaseTimelineStorageSchema {
     prefix = "yet-another-unit-test.";
     hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME,
         prefix);
-    createSchema(hbaseConf);
+    DataGeneratorForTest.createSchema(hbaseConf);
     entityTableName = BaseTable.getTableName(hbaseConf,
         EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME);
     assertTrue(admin.tableExists(entityTableName));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index b608987..8ab32df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.Configuration;
 /**
  * Generates the data/entities for the FlowRun and FlowActivity Tables.
  */
-final class TestFlowDataGenerator {
+public final class TestFlowDataGenerator {
   private TestFlowDataGenerator() {
   }
 
@@ -41,7 +41,8 @@ final class TestFlowDataGenerator {
   private static final String METRIC_2 = "HDFS_BYTES_READ";
   public static final long END_TS_INCR = 10000L;
 
-  static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
+  public static TimelineEntity getEntityMetricsApp1(long insertTs,
+      Configuration c1) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunMetrics_test";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -83,7 +84,7 @@ final class TestFlowDataGenerator {
   }
 
 
-  static TimelineEntity getEntityMetricsApp1Complete(long insertTs,
+  public static TimelineEntity getEntityMetricsApp1Complete(long insertTs,
       Configuration c1) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunMetrics_test";
@@ -125,7 +126,7 @@ final class TestFlowDataGenerator {
   }
 
 
-  static TimelineEntity getEntityMetricsApp1(long insertTs) {
+  public static TimelineEntity getEntityMetricsApp1(long insertTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunMetrics_test";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -168,8 +169,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-
-  static TimelineEntity getEntityMetricsApp2(long insertTs) {
+  public static TimelineEntity getEntityMetricsApp2(long insertTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunMetrics_test";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -200,7 +200,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getEntity1() {
+  public static TimelineEntity getEntity1() {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunHello";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -243,7 +243,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getAFullEntity(long ts, long endTs) {
+  public static TimelineEntity getAFullEntity(long ts, long endTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunFullEntity";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -292,7 +292,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getEntityGreaterStartTime(long startTs) {
+  public static TimelineEntity getEntityGreaterStartTime(long startTs) {
     TimelineEntity entity = new TimelineEntity();
     entity.setCreatedTime(startTs);
     entity.setId("flowRunHello with greater start time");
@@ -308,7 +308,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getEntityMaxEndTime(long endTs) {
+  public static TimelineEntity getEntityMaxEndTime(long endTs) {
     TimelineEntity entity = new TimelineEntity();
     entity.setId("flowRunHello Max End time");
     entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
@@ -322,7 +322,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getEntityMinStartTime(long startTs) {
+  public static TimelineEntity getEntityMinStartTime(long startTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunHelloMInStartTime";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -336,7 +336,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getMinFlushEntity(long startTs) {
+  public static TimelineEntity getMinFlushEntity(long startTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunHelloFlushEntityMin";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -350,7 +350,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getMaxFlushEntity(long startTs) {
+  public static TimelineEntity getMaxFlushEntity(long startTs) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowRunHelloFlushEntityMax";
     String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -365,7 +365,7 @@ final class TestFlowDataGenerator {
     return entity;
   }
 
-  static TimelineEntity getFlowApp1(long appCreatedTime) {
+  public static TimelineEntity getFlowApp1(long appCreatedTime) {
     TimelineEntity entity = new TimelineEntity();
     String id = "flowActivity_test";
     String type = TimelineEntityType.YARN_APPLICATION.toString();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 97d40fd..1af47a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -47,9 +47,9 @@ import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
 import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
 import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
 import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
@@ -70,11 +70,7 @@ public class TestHBaseStorageFlowActivity {
     Configuration conf = util.getConfiguration();
     conf.setInt("hfile.format.version", 3);
     util.startMiniCluster();
-    createSchema();
-  }
-
-  private static void createSchema() throws IOException {
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+    DataGeneratorForTest.createSchema(util.getConfiguration());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 00fee69..e376c6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -41,8 +40,8 @@ import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
@@ -57,10 +56,10 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompa
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
 import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
@@ -84,11 +83,7 @@ public class TestHBaseStorageFlowRun {
     Configuration conf = util.getConfiguration();
     conf.setInt("hfile.format.version", 3);
     util.startMiniCluster();
-    createSchema();
-  }
-
-  private static void createSchema() throws IOException {
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+    DataGeneratorForTest.createSchema(util.getConfiguration());
   }
 
   @Test
@@ -106,12 +101,7 @@ public class TestHBaseStorageFlowRun {
       // check the regions.
       // check in flow run table
       util.waitUntilAllRegionsAssigned(table);
-      HRegionServer server = util.getRSForFirstRegionInTable(table);
-      List<Region> regions = server.getOnlineRegions(table);
-      for (Region region : regions) {
-        assertTrue(FlowRunTable.isFlowRunTable(region.getRegionInfo(),
-            hbaseConf));
-      }
+      checkCoprocessorExists(table, true);
     }
 
     table = BaseTable.getTableName(hbaseConf,
@@ -121,12 +111,7 @@ public class TestHBaseStorageFlowRun {
       // check the regions.
       // check in flow activity table
       util.waitUntilAllRegionsAssigned(table);
-      HRegionServer server = util.getRSForFirstRegionInTable(table);
-      List<Region> regions = server.getOnlineRegions(table);
-      for (Region region : regions) {
-        assertFalse(FlowRunTable.isFlowRunTable(region.getRegionInfo(),
-            hbaseConf));
-      }
+      checkCoprocessorExists(table, false);
     }
 
     table = BaseTable.getTableName(hbaseConf, EntityTable.TABLE_NAME_CONF_NAME,
@@ -135,12 +120,23 @@ public class TestHBaseStorageFlowRun {
       // check the regions.
       // check in entity run table
       util.waitUntilAllRegionsAssigned(table);
-      HRegionServer server = util.getRSForFirstRegionInTable(table);
-      List<Region> regions = server.getOnlineRegions(table);
-      for (Region region : regions) {
-        assertFalse(FlowRunTable.isFlowRunTable(region.getRegionInfo(),
-            hbaseConf));
+      checkCoprocessorExists(table, false);
+    }
+  }
+
+  private void checkCoprocessorExists(TableName table, boolean exists)
+      throws IOException, InterruptedException {
+    HRegionServer server = util.getRSForFirstRegionInTable(table);
+    List<Region> regions = server.getOnlineRegions(table);
+    for (Region region : regions) {
+      boolean found = false;
+      Set<String> coprocs = region.getCoprocessorHost().getCoprocessors();
+      for (String coprocName : coprocs) {
+        if (coprocName.contains("FlowRunCoprocessor")) {
+          found = true;
+        }
       }
+      assertEquals(found, exists);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index a4c0e44..fa9d029 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
@@ -82,11 +82,7 @@ public class TestHBaseStorageFlowRunCompaction {
     Configuration conf = util.getConfiguration();
     conf.setInt("hfile.format.version", 3);
     util.startMiniCluster();
-    createSchema();
-  }
-
-  private static void createSchema() throws IOException {
-    TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+    DataGeneratorForTest.createSchema(util.getConfiguration());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
index 122d399..24101c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGen
 public class FlowRunCoprocessor extends BaseRegionObserver {
 
   private static final Log LOG = LogFactory.getLog(FlowRunCoprocessor.class);
-  private boolean isFlowRunRegion = false;
 
   private Region region;
   /**
@@ -71,15 +70,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
     if (e instanceof RegionCoprocessorEnvironment) {
       RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
       this.region = env.getRegion();
-      isFlowRunRegion = FlowRunTable.isFlowRunTable(
-          region.getRegionInfo(), env.getConfiguration());
     }
   }
 
-  public boolean isFlowRunRegion() {
-    return isFlowRunRegion;
-  }
-
   /*
    * (non-Javadoc)
    *
@@ -99,10 +92,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put,
       WALEdit edit, Durability durability) throws IOException {
     Map<String, byte[]> attributes = put.getAttributesMap();
-
-    if (!isFlowRunRegion) {
-      return;
-    }
     // Assumption is that all the cells in a put are the same operation.
     List<Tag> tags = new ArrayList<>();
     if ((attributes != null) && (attributes.size() > 0)) {
@@ -170,10 +159,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   @Override
   public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
       Get get, List<Cell> results) throws IOException {
-    if (!isFlowRunRegion) {
-      return;
-    }
-
     Scan scan = new Scan(get);
     scan.setMaxVersions();
     RegionScanner scanner = null;
@@ -205,12 +190,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   public RegionScanner preScannerOpen(
       ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
       RegionScanner scanner) throws IOException {
-
-    if (isFlowRunRegion) {
-      // set max versions for scan to see all
-      // versions to aggregate for metrics
-      scan.setMaxVersions();
-    }
+    // set max versions for scan to see all
+    // versions to aggregate for metrics
+    scan.setMaxVersions();
     return scanner;
   }
 
@@ -230,9 +212,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   public RegionScanner postScannerOpen(
       ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
       RegionScanner scanner) throws IOException {
-    if (!isFlowRunRegion) {
-      return scanner;
-    }
     return new FlowScanner(e.getEnvironment(), scan,
         scanner, FlowScannerOperation.READ);
   }
@@ -241,9 +220,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   public InternalScanner preFlush(
       ObserverContext<RegionCoprocessorEnvironment> c, Store store,
       InternalScanner scanner) throws IOException {
-    if (!isFlowRunRegion) {
-      return scanner;
-    }
     if (LOG.isDebugEnabled()) {
       if (store != null) {
         LOG.debug("preFlush store = " + store.getColumnFamilyName()
@@ -264,9 +240,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
   @Override
   public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
       Store store, StoreFile resultFile) {
-    if (!isFlowRunRegion) {
-      return;
-    }
     if (LOG.isDebugEnabled()) {
       if (store != null) {
         LOG.debug("postFlush store = " + store.getColumnFamilyName()
@@ -288,9 +261,6 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
       InternalScanner scanner, ScanType scanType, CompactionRequest request)
       throws IOException {
 
-    if (!isFlowRunRegion) {
-      return scanner;
-    }
     FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION;
     if (request != null) {
       requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
index 4cd581b..ec973cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -23,13 +23,14 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
 
 /**
  * The flow run table has column family info
@@ -133,29 +134,17 @@ public class FlowRunTable extends BaseTable<FlowRunTable> {
     infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
 
     // TODO: figure the split policy
-    flowRunTableDescp.addCoprocessor(FlowRunCoprocessor.class
-        .getCanonicalName());
+    String coprocessorJarPathStr = hbaseConf.get(
+        YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION,
+        YarnConfiguration.DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR);
+
+    Path coprocessorJarPath = new Path(coprocessorJarPathStr);
+    LOG.info("CoprocessorJarPath=" + coprocessorJarPath.toString());
+    flowRunTableDescp.addCoprocessor(
+        FlowRunCoprocessor.class.getCanonicalName(), coprocessorJarPath,
+        Coprocessor.PRIORITY_USER, null);
     admin.createTable(flowRunTableDescp);
     LOG.info("Status of table creation for " + table.getNameAsString() + "="
         + admin.tableExists(table));
   }
-
-  public static boolean isFlowRunTable(HRegionInfo hRegionInfo,
-      Configuration conf) {
-    String regionTableName = hRegionInfo.getTable().getNameAsString();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("regionTableName=" + regionTableName);
-    }
-    String flowRunTableName = BaseTable.getTableName(conf,
-        FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME)
-        .getNameAsString();
-    if (flowRunTableName.equalsIgnoreCase(regionTableName)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(" table is the flow run table!! "
-            + flowRunTableName);
-      }
-      return true;
-    }
-    return false;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0177c95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index 7a0c477..e53b05d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -216,30 +216,32 @@ http://hbase.apache.org/book.html#standalone.over.hdfs .
 Once you have an Apache HBase cluster ready to use, perform the following steps.
 
 ##### <a name="Enable_the_coprocessor"> </a>Step 2) Enable the coprocessor
+In this version, the coprocessor is loaded dynamically (table coprocessor for the `flowrun` table).
 
-Step 2.1) Add the timeline service jar to the HBase classpath in all HBase machines in the cluster. It
-is needed for the coprocessor as well as the schema creator. For example,
+Copy the timeline service jar to HDFS from where HBase can load it. It
+is needed for the `flowrun` table creation in the schema creator. The default HDFS location is `/hbase/coprocessor`.
+For example,
 
-    cp hadoop-yarn-server-timelineservice-3.0.0-alpha1-SNAPSHOT.jar /usr/hbase/lib/
+    hadoop fs -mkdir /hbase/coprocessor
+    hadoop fs -put hadoop-yarn-server-timelineservice-3.0.0-alpha1-SNAPSHOT.jar
+           /hbase/coprocessor/hadoop-yarn-server-timelineservice.jar
 
-Step 2.2) Enable the coprocessor that handles the aggregation. To enable it, add the following entry in
-region servers' `hbase-site.xml` file (generally located in the `conf` directory) as follows:
+
+If you want to place the jar at a different location on hdfs, there also exists a yarn
+configuration setting called `yarn.timeline-service.hbase.coprocessor.jar.hdfs.location`.
+For example,
 
 ```
 <property>
-  <name>hbase.coprocessor.region.classes</name>
-  <value>org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunCoprocessor</value>
+  <name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location</name>
+  <value>/custom/hdfs/path/jarName</value>
 </property>
 ```
 
-Step 2.3) Restart the region servers and the master to pick up the timeline service jar as well
-as the config change. In this version, the coprocessor is loaded statically
-(i.e. system coprocessor) as opposed to a dynamically (table coprocessor).
-
 ##### <a name="Create_schema"> </a>Step 3) Create the timeline service schema
 Finally, run the schema creator tool to create the necessary tables:
 
-    bin/hbase org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator
+    bin/hadoop org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator
 
 The `TimelineSchemaCreator` tool supports a few options that may come handy especially when you
 are testing. For example, you can use `-skipExistingTable` (`-s` for short) to skip existing tables


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message