ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject [11/30] ambari git commit: AMBARI-5707. Replace Ganglia with high performant and pluggable Metrics System. (swagle)
Date Tue, 02 Dec 2014 17:28:27 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/ams_query.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/ams_query.py b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/ams_query.py
new file mode 100644
index 0000000..d51357a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/ams_query.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import urllib2
+import signal
+import sys
+import optparse
+import time
+
+# http://162.216.148.45:8188/ws/v1/timeline/metrics?
+# metricNames=rpc.rpc.RpcAuthenticationSuccesses
+# &appId=nodemanager&hostname=local.0&startTime=1414152029&endTime=1414155629
+
+AMS_URL = "http://{0}:8188/ws/v1/timeline/metrics?metricNames={1}&appid={" \
+          "2}&hostname={3}"
+
+# in fact it can be list automatically generated from ambari
+# UI queries
+host_metrics = {
+  'cpu': ['cpu_user', 'cpu_wio', 'cpu_nice', 'cpu_aidle', 'cpu_system', 'cpu_idle'],
+  'disk': ['disk_total', 'disk_free'],
+  'load': ['load_one', 'load_fifteen', 'load_five'],
+  'mem': ['swap_free', 'mem_shared', 'mem_free', 'mem_cached', 'mem_buffers'],
+  'network': ['bytes_in', 'bytes_out', 'pkts_in', 'pkts_out'],
+  'process': ['proc_total', 'proc_run']
+}
+
+# HDFS_SERVICE
+namenode_metrics = {
+  'dfs.Capacity': ['dfs.FSNamesystem.CapacityRemainingGB',
+                   'dfs.FSNamesystem.CapacityUsedGB',
+                   'dfs.FSNamesystem.CapacityTotalGB'],
+  'dfs.Replication': ['dfs.FSNamesystem.PendingReplicationBlocks',
+                      'dfs.FSNamesystem.UnderReplicatedBlocks'],
+  'dfs.File': ['dfs.namenode.FileInfoOps', 'dfs.namenode.CreateFileOps'],
+  'jvm.gc': ['jvm.JvmMetrics.GcTimeMillis'],
+  'jvm.mem': ['jvm.JvmMetrics.MemNonHeapUsedM',
+              'jvm.JvmMetrics.MemNonHeapCommittedM',
+              'jvm.JvmMetrics.MemHeapUsedM',
+              'jvm.JvmMetrics.MemHeapCommittedM'],
+  'jvm.thread': ['jvm.JvmMetrics.ThreadsRunnable',
+                 'jvm.JvmMetrics.ThreadsBlocked',
+                 'jvm.JvmMetrics.ThreadsWaiting',
+                 'jvm.JvmMetrics.ThreadsTimedWaiting'],
+  'rpc': ['rpc.rpc.RpcQueueTimeAvgTime']
+}
+
+all_metrics = {
+  'HOST': host_metrics,
+  'namenode': namenode_metrics
+}
+
+all_metrics_times = {}
+
+
+# hostnames = ['EPPLKRAW0101.0']  # 'local.0'
+# metrics_test_host = '162.216.150.247' # metricstest-100
+# metrics_test_host = '162.216.148.45'    # br-3
+# start_time = int(time.time())           # 1414425208
+
+
+def main(argv=None):
+  # Allow Ctrl-C
+  signal.signal(signal.SIGINT, signal_handler)
+
+  parser = optparse.OptionParser()
+
+  parser.add_option("-H", "--host", dest="host",
+                    help="AMS host")
+  parser.add_option("-t", "--starttime", dest="start_time_secs",
+                    default=int(time.time()),
+                    help="start time in seconds, default value is current time")
+  parser.add_option("-n", "--nodes", dest="node_names",
+                    help="nodes from cluster, used as a param to query for")
+  (options, args) = parser.parse_args()
+
+  if options.host is None:
+    print "AMS host name is required (--host or -h)"
+    exit(-1)
+
+  if options.node_names is None:
+    print "cluster nodes are required (--nodes or -n)"
+    exit(3)
+
+  global start_time_secs, metrics_test_host, hostnames
+
+  metrics_test_host = options.host
+  start_time_secs = int(options.start_time_secs)
+  hostnames = [options.node_names]
+
+  while True:
+    run()
+    time.sleep(15)
+    start_time_secs += 15
+
+
+def signal_handler(signal, frame):
+  print('Exiting, Ctrl+C press detected!')
+  print_all_metrics(all_metrics_times)
+  sys.exit(0)
+
+
+def run():
+  hostname = ','.join(hostnames)
+  qs = QuerySender(metrics_test_host, True)
+  for metric_name in all_metrics:
+    print
+    print 'Querying for ' + metric_name + ' metrics'
+    current_time_secs = start_time_secs
+    qs.query_all_app_metrics(hostname, metric_name,
+                             all_metrics[metric_name],
+                             current_time_secs)
+
+
+def add_query_metrics_for_app_id(app_id, metric_timing):
+  if not app_id in all_metrics_times:
+    all_metrics_times[app_id] = []
+  all_metrics_times[app_id].append(metric_timing)
+
+
+def print_all_metrics(metrics):
+  print 'Metrics Summary'
+  for app_id in sorted(metrics):
+    first = True
+    for single_query_metrics in metrics[app_id]:
+      print_app_metrics(app_id, single_query_metrics, first)
+      first = False
+
+
+def print_app_metrics(app_id, metric_timing, header=False):
+  #header
+  if header:
+    print app_id + ': ' + ','.join(sorted(metric_timing.keys()))
+  #vals
+  print app_id + ':',
+  for key in sorted(metric_timing):
+    print '%.3f,' % metric_timing[key],
+  print
+
+
+class QuerySender:
+  def __init__(self, metrics_address, print_responses=False):
+    self.metrics_address = metrics_address
+    self.print_responses = print_responses
+
+  def query_all_app_metrics(self, hostname, app_id, metrics, current_time_secs):
+    metric_timing = {}
+    for key in metrics:
+      print 'Getting metrics for', key
+      query_time = time.time()
+
+      metric_names = ','.join(metrics[key])
+      self.query(hostname, app_id, metric_names, current_time_secs)
+      query_time_elapsed = time.time() - query_time
+
+      print 'Query for "%s" took %s' % (key, query_time_elapsed)
+      metric_timing[key] = query_time_elapsed
+
+    add_query_metrics_for_app_id(app_id, metric_timing)
+    if self.print_responses:
+      print_app_metrics(app_id, metric_timing)
+
+  def query(self, hostname, app_id, metric_names, current_time_secs):
+    url = self.create_url(hostname, metric_names, app_id, current_time_secs)
+    print url
+    response = self.send(url)
+    if self.print_responses:
+      print response
+    pass
+
+  def send(self, url):
+    request = urllib2.Request(url)
+    try:
+      response = urllib2.urlopen(request, timeout=int(30))
+      response = response.read()
+      return response
+
+    except urllib2.URLError as e:
+      print e.reason
+
+  def create_url(self, hostname, metric_names, app_id, current_time_secs):
+    server = AMS_URL.format(self.metrics_address,
+                            metric_names,
+                            app_id,
+                            hostname)
+    t = current_time_secs
+    server += '&startTime=%s&endTime=%s' % (t, t + 3600)
+    return server
+
+
+if __name__ == '__main__':
+  main()

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start.sh b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start.sh
new file mode 100644
index 0000000..1d6fff5
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# set -x
+cd "$(dirname "$0")"
+
+METRICS_HOST=$1
+HOST_COUNT=$2
+
+echo $$ > sim_pid
+cat sim_pid
+#HOMEDIR
+exec  java -jar ../lib/ambari-metrics/ambari-metrics-timelineservice-simulator*.jar  -h `hostname -f` -n ${HOST_COUNT} -m ${METRICS_HOST} -c 15000 -s 60000

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start_slaves.sh b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start_slaves.sh
new file mode 100644
index 0000000..e1e51c8
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/start_slaves.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+## set -x
+
+SLAVES=$1
+METRICS_HOST=$2
+HOST_COUNT=$3
+LOADSIM_HOMEDIR="LoadSimulator-1.0-SNAPSHOT"
+
+pdsh -w ${SLAVES} "$LOADSIM_HOMEDIR/start.sh ${METRICS_HOST} ${HOST_COUNT}'"

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/status_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/status_slaves.sh b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/status_slaves.sh
new file mode 100644
index 0000000..79787fd
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/status_slaves.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+## set -x
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop.sh b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop.sh
new file mode 100644
index 0000000..2220861
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# set -x
+cd "$(dirname "$0")"
+read PID <./sim_pid
+
+FOUND_PID=`ps aux | grep simulator | grep $PID`
+if [ -z "${FOUND_PID}" ]
+then
+   echo "process simulator not fund under pid $PID"
+else
+   echo "process simulator running as $PID, killing"
+   kill ${PID}
+fi

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop_slaves.sh
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop_slaves.sh b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop_slaves.sh
new file mode 100644
index 0000000..7cb567a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/scripts/stop_slaves.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# set -x
+
+SLAVES=$1
+LOADSIM_HOMEDIR="LoadSimulator-1.0-SNAPSHOT"
+
+pdsh -w ${SLAVES} "$LOADSIM_HOMEDIR/stop.sh"

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/ams-site.xml b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/ams-site.xml
new file mode 100644
index 0000000..b08609d
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/ams-site.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- Empty site, all defaults should be initialized in the code -->
+<configuration>
+  <property>
+    <name>test</name>
+    <value>testReady</value>
+  </property>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hadoop-policy.xml b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hadoop-policy.xml
new file mode 100644
index 0000000..41bde16
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hadoop-policy.xml
@@ -0,0 +1,134 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration supports_final="true">
+  <property>
+    <name>security.client.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for ClientProtocol, which is used by user code
+    via the DistributedFileSystem.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.client.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
+    for block recovery.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for DatanodeProtocol, which is used by datanodes to
+    communicate with the namenode.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.inter.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
+    for updating generation timestamp.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.namenode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for NamenodeProtocol, the protocol used by the secondary
+    namenode to communicate with the namenode.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.inter.tracker.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for InterTrackerProtocol, used by the tasktrackers to
+    communicate with the jobtracker.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.job.client.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for JobSubmissionProtocol, used by job clients to
+    communciate with the jobtracker for job submission, querying job status etc.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.job.task.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
+    tasks to communicate with the parent tasktracker.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+ <property>
+    <name>security.admin.operations.protocol.acl</name>
+    <value>hadoop</value>
+    <description>ACL for AdminOperationsProtocol. Used for admin commands.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+    <value>hadoop</value>
+    <description>ACL for RefreshUserMappingsProtocol. Used to refresh
+    users mappings. The ACL is a comma-separated list of user and
+    group names. The user and group list is separated by a blank. For
+    e.g. "alice,bob users,wheel".  A special value of "*" means all
+    users are allowed.</description>
+  </property>
+
+<property>
+    <name>security.refresh.policy.protocol.acl</name>
+    <value>hadoop</value>
+    <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
+    dfsadmin and mradmin commands to refresh the security policy in-effect.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hbase-site.xml b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hbase-site.xml
new file mode 100644
index 0000000..c453900
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/conf/hbase-site.xml
@@ -0,0 +1,230 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>file:///var/lib/hbase</value>
+    <description>
+      AMS service uses HBase as default storage backend. Set the rootdir for
+      HBase to either local filesystem path if using AMS in embedded mode or
+      to a HDFS dir, example: hdfs://namenode.example.org:9000/hbase.  By
+      default HBase writes into /tmp. Change this configuration else all data
+      will be lost on machine restart.
+    </description>
+  </property>
+  <property>
+    <name>hbase.tmp.dir</name>
+    <value>/tmp</value>
+    <description>
+      Temporary directory on the local filesystem.
+      Change this setting to point to a location more permanent
+      than '/tmp' (The '/tmp' directory is often cleared on
+      machine restart).
+    </description>
+  </property>
+  <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+    <description>
+      The mode the cluster will be in. Possible values are false for
+      standalone mode and true for distributed mode. If false, startup will run
+      all HBase and ZooKeeper daemons together in the one JVM.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.wait.on.regionservers.mintostart</name>
+    <value>1</value>
+    <description>
+      Ensure that HBase Master waits for # many region server to start.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>localhost</value>
+    <description>Comma separated list of servers in the ZooKeeper Quorum.
+      For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+      By default this is set to localhost for local and pseudo-distributed modes
+      of operation. For a fully-distributed setup, this should be set to a full
+      list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+      this is the list of servers which we will start/stop ZooKeeper on.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.info.bindAddress</name>
+    <value>0.0.0.0</value>
+    <description>The bind address for the HBase Master web UI</description>
+  </property>
+  <property>
+    <name>hbase.master.info.port</name>
+    <value>90010</value>
+    <description>The port for the HBase Master web UI.</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.info.port</name>
+    <value>90030</value>
+    <description>The port for the HBase RegionServer web UI.</description>
+  </property>
+  <property>
+    <name>hbase.hregion.majorcompaction</name>
+    <value>0</value>
+    <description>
+      The time (in milliseconds) between 'major' compactions of all
+      HStoreFiles in a region.
+      0 to disable automated major compactions.
+    </description>
+  </property>
+  <property>
+    <name>phoenix.query.spoolThresholdBytes</name>
+    <value>12582912</value>
+    <description>
+      Threshold size in bytes after which results from parallelly executed
+      query results are spooled to disk. Default is 20 mb.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.dataDir</name>
+    <value>${hbase.tmp.dir}/zookeeper</value>
+    <description>
+      Property from ZooKeeper's config zoo.cfg.
+      The directory where the snapshot is stored.
+    </description>
+  </property>
+  <property>
+    <name>hbase.client.scanner.caching</name>
+    <value>10000</value>
+    <description>
+      Number of rows that will be fetched when calling next on a scanner
+      if it is not served from (local, client) memory.
+    </description>
+  </property>
+  <property>
+    <name>hfile.block.cache.size</name>
+    <value>0.3</value>
+    <description>
+      Percentage of maximum heap (-Xmx setting) to allocate to block cache
+      used by a StoreFile. Default of 0.4 means allocate 40%.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.global.memstore.upperLimit</name>
+    <value>0.5</value>
+    <description>
+      Maximum size of all memstores in a region server before new
+      updates are blocked and flushes are forced. Defaults to 40% of heap
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.global.memstore.lowerLimit</name>
+    <value>0.4</value>
+    <description>
+      When memstores are being forced to flush to make room in
+      memory, keep flushing until we hit this mark. Defaults to 35% of heap.
+      This value equal to hbase.regionserver.global.memstore.upperLimit causes
+      the minimum possible flushing to occur when updates are blocked due to
+      memstore limiting.
+    </description>
+  </property>
+  <property>
+    <name>phoenix.groupby.maxCacheSize</name>
+    <value>307200000</value>
+    <description>
+      Size in bytes of pages cached during GROUP BY spilling. Default is 100Mb.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.memstore.block.multiplier</name>
+    <value>4</value>
+    <description>
+      Block updates if memstore has hbase.hregion.memstore.block.multiplier
+      times hbase.hregion.memstore.flush.size bytes. Useful preventing runaway
+      memstore during spikes in update traffic.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hstore.flusher.count</name>
+    <value>2</value>
+    <description>
+      The number of flush threads. With fewer threads, the MemStore flushes
+      will be queued. With more threads, the flushes will be executed in parallel,
+      increasing the load on HDFS, and potentially causing more compactions.
+    </description>
+  </property>
+  <property>
+    <name>phoenix.query.timeoutMs</name>
+    <value>1200000</value>
+    <description>
+      Number of milliseconds after which a query will timeout on the client.
+      Default is 10 min.
+    </description>
+  </property>
+  <property>
+    <name>hbase.client.scanner.timeout.period</name>
+    <value>900000</value>
+    <description>
+      Client scanner lease period in milliseconds.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.thread.compaction.large</name>
+    <value>2</value>
+    <description>
+      Configuration key for the large compaction threads.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.thread.compaction.small</name>
+    <value>3</value>
+    <description>
+      Configuration key for the small compaction threads.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hstore.blockingStoreFiles</name>
+    <value>200</value>
+    <description>
+      If more than this number of StoreFiles exist in any one Store
+      (one StoreFile is written per flush of MemStore), updates are blocked for
+      this region until a compaction is completed, or until
+      hbase.hstore.blockingWaitTime has been exceeded.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.memstore.flush.size</name>
+    <value>134217728</value>
+    <description>
+      Memstore will be flushed to disk if size of the memstore exceeds this
+      number of bytes. Value is checked by a thread that runs every
+      hbase.server.thread.wakefrequency.
+    </description>
+  </property>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
new file mode 100644
index 0000000..c41b8a7
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
+
+public class ApplicationHistoryStoreTestUtils {
+
+  protected ApplicationHistoryStore store;
+
+  protected void writeApplicationStartData(ApplicationId appId)
+      throws IOException {
+    store.applicationStarted(ApplicationStartData.newInstance(appId,
+      appId.toString(), "test type", "test queue", "test user", 0, 0));
+  }
+
+  protected void writeApplicationFinishData(ApplicationId appId)
+      throws IOException {
+    store.applicationFinished(ApplicationFinishData.newInstance(appId, 0,
+      appId.toString(), FinalApplicationStatus.UNDEFINED,
+      YarnApplicationState.FINISHED));
+  }
+
+  protected void writeApplicationAttemptStartData(
+      ApplicationAttemptId appAttemptId) throws IOException {
+    store.applicationAttemptStarted(ApplicationAttemptStartData.newInstance(
+      appAttemptId, appAttemptId.toString(), 0,
+      ContainerId.newInstance(appAttemptId, 1)));
+  }
+
+  protected void writeApplicationAttemptFinishData(
+      ApplicationAttemptId appAttemptId) throws IOException {
+    store.applicationAttemptFinished(ApplicationAttemptFinishData.newInstance(
+      appAttemptId, appAttemptId.toString(), "test tracking url",
+      FinalApplicationStatus.UNDEFINED, YarnApplicationAttemptState.FINISHED));
+  }
+
+  protected void writeContainerStartData(ContainerId containerId)
+      throws IOException {
+    store.containerStarted(ContainerStartData.newInstance(containerId,
+      Resource.newInstance(0, 0), NodeId.newInstance("localhost", 0),
+      Priority.newInstance(containerId.getId()), 0));
+  }
+
+  protected void writeContainerFinishData(ContainerId containerId)
+      throws IOException {
+    store.containerFinished(ContainerFinishData.newInstance(containerId, 0,
+      containerId.toString(), 0, ContainerState.COMPLETE));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
new file mode 100644
index 0000000..6b06918
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+// Timeline service client support is not enabled for AMS
+@Ignore
+public class TestApplicationHistoryClientService extends
+    ApplicationHistoryStoreTestUtils {
+
+  ApplicationHistoryServer historyServer = null;
+  String expectedLogUrl = null;
+
+  @Before
+  public void setup() {
+    historyServer = new ApplicationHistoryServer();
+    Configuration config = new YarnConfiguration();
+    expectedLogUrl = WebAppUtils.getHttpSchemePrefix(config) +
+        WebAppUtils.getAHSWebAppURLWithoutScheme(config) +
+        "/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
+        "container_0_0001_01_000001/test user";
+    config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+      MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
+    historyServer.init(config);
+    historyServer.start();
+    store =
+        ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistory())
+          .getHistoryStore();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    historyServer.stop();
+  }
+
+  @Test
+  public void testApplicationReport() throws IOException, YarnException {
+    ApplicationId appId = null;
+    appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    writeApplicationFinishData(appId);
+    GetApplicationReportRequest request =
+        GetApplicationReportRequest.newInstance(appId);
+    GetApplicationReportResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getApplicationReport(request);
+    ApplicationReport appReport = response.getApplicationReport();
+    Assert.assertNotNull(appReport);
+    Assert.assertEquals("application_0_0001", appReport.getApplicationId()
+      .toString());
+    Assert.assertEquals("test type", appReport.getApplicationType().toString());
+    Assert.assertEquals("test queue", appReport.getQueue().toString());
+  }
+
+  @Test
+  public void testApplications() throws IOException, YarnException {
+    ApplicationId appId = null;
+    appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    writeApplicationFinishData(appId);
+    ApplicationId appId1 = ApplicationId.newInstance(0, 2);
+    writeApplicationStartData(appId1);
+    writeApplicationFinishData(appId1);
+    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+    GetApplicationsResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getApplications(request);
+    List<ApplicationReport> appReport = response.getApplicationList();
+    Assert.assertNotNull(appReport);
+    Assert.assertEquals(appId, appReport.get(0).getApplicationId());
+    Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+  }
+
+  @Test
+  public void testApplicationAttemptReport() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    writeApplicationAttemptStartData(appAttemptId);
+    writeApplicationAttemptFinishData(appAttemptId);
+    GetApplicationAttemptReportRequest request =
+        GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+    GetApplicationAttemptReportResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getApplicationAttemptReport(request);
+    ApplicationAttemptReport attemptReport =
+        response.getApplicationAttemptReport();
+    Assert.assertNotNull(attemptReport);
+    Assert.assertEquals("appattempt_0_0001_000001", attemptReport
+      .getApplicationAttemptId().toString());
+  }
+
+  @Test
+  public void testApplicationAttempts() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ApplicationAttemptId appAttemptId1 =
+        ApplicationAttemptId.newInstance(appId, 2);
+    writeApplicationAttemptStartData(appAttemptId);
+    writeApplicationAttemptFinishData(appAttemptId);
+    writeApplicationAttemptStartData(appAttemptId1);
+    writeApplicationAttemptFinishData(appAttemptId1);
+    GetApplicationAttemptsRequest request =
+        GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getApplicationAttempts(request);
+    List<ApplicationAttemptReport> attemptReports =
+        response.getApplicationAttemptList();
+    Assert.assertNotNull(attemptReports);
+    Assert.assertEquals(appAttemptId, attemptReports.get(0)
+      .getApplicationAttemptId());
+    Assert.assertEquals(appAttemptId1, attemptReports.get(1)
+      .getApplicationAttemptId());
+  }
+
+  @Test
+  public void testContainerReport() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    writeContainerStartData(containerId);
+    writeContainerFinishData(containerId);
+    writeApplicationFinishData(appId);
+    GetContainerReportRequest request =
+        GetContainerReportRequest.newInstance(containerId);
+    GetContainerReportResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getContainerReport(request);
+    ContainerReport container = response.getContainerReport();
+    Assert.assertNotNull(container);
+    Assert.assertEquals(containerId, container.getContainerId());
+    Assert.assertEquals(expectedLogUrl, container.getLogUrl());
+  }
+
+  @Test
+  public void testContainers() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    ContainerId containerId1 = ContainerId.newInstance(appAttemptId, 2);
+    writeContainerStartData(containerId);
+    writeContainerFinishData(containerId);
+    writeContainerStartData(containerId1);
+    writeContainerFinishData(containerId1);
+    writeApplicationFinishData(appId);
+    GetContainersRequest request =
+        GetContainersRequest.newInstance(appAttemptId);
+    GetContainersResponse response =
+        historyServer.getClientService().getClientHandler()
+          .getContainers(request);
+    List<ContainerReport> containers = response.getContainerList();
+    Assert.assertNotNull(containers);
+    Assert.assertEquals(containerId, containers.get(1).getContainerId());
+    Assert.assertEquals(containerId1, containers.get(0).getContainerId());
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
new file mode 100644
index 0000000..aad23d9
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestApplicationHistoryManagerImpl extends
+    ApplicationHistoryStoreTestUtils {
+  ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null;
+
+  @Before
+  public void setup() throws Exception {
+    Configuration config = new Configuration();
+    config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+      MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
+    applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl();
+    applicationHistoryManagerImpl.init(config);
+    applicationHistoryManagerImpl.start();
+    store = applicationHistoryManagerImpl.getHistoryStore();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    applicationHistoryManagerImpl.stop();
+  }
+
+  @Test
+  @Ignore
+  public void testApplicationReport() throws IOException, YarnException {
+    ApplicationId appId = null;
+    appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    writeApplicationFinishData(appId);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    writeApplicationAttemptStartData(appAttemptId);
+    writeApplicationAttemptFinishData(appAttemptId);
+    ApplicationReport appReport =
+        applicationHistoryManagerImpl.getApplication(appId);
+    Assert.assertNotNull(appReport);
+    Assert.assertEquals(appId, appReport.getApplicationId());
+    Assert.assertEquals(appAttemptId,
+      appReport.getCurrentApplicationAttemptId());
+    Assert.assertEquals(appAttemptId.toString(), appReport.getHost());
+    Assert.assertEquals("test type", appReport.getApplicationType().toString());
+    Assert.assertEquals("test queue", appReport.getQueue().toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
new file mode 100644
index 0000000..3720852
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
+  .timeline.DefaultPhoenixDataSource;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
+  .timeline.PhoenixHBaseAccessor;
+import org.apache.zookeeper.ClientCnxn;
+import org.easymock.EasyMock;
+import org.junit.*;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics
+  .timeline.TimelineMetricConfiguration.METRICS_SITE_CONFIGURATION_FILE;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.*;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.junit.Assert.*;
+import static org.powermock.api.easymock.PowerMock.*;
+import static org.powermock.api.support.membermodification.MemberMatcher.method;
+import static org.powermock.api.support.membermodification.MemberModifier
+  .suppress;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ PhoenixHBaseAccessor.class, UserGroupInformation.class,
+  ClientCnxn.class, DefaultPhoenixDataSource.class})
+@PowerMockIgnore( {"javax.management.*"})
+public class TestApplicationHistoryServer {
+
+  ApplicationHistoryServer historyServer = null;
+  Configuration metricsConf = null;
+
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
+  @Before
+  @SuppressWarnings("all")
+  public void setup() throws URISyntaxException, IOException {
+    folder.create();
+    File hbaseSite = folder.newFile("hbase-site.xml");
+    File amsSite = folder.newFile("ams-site.xml");
+
+    FileUtils.writeStringToFile(hbaseSite, "<configuration>\n" +
+      "  <property>\n" +
+      "    <name>hbase.defaults.for.version.skip</name>\n" +
+      "    <value>true</value>\n" +
+      "  </property>" +
+      "  <property> " +
+      "    <name>hbase.zookeeper.quorum</name>\n" +
+      "    <value>localhost</value>\n" +
+      "  </property>" +
+      "</configuration>");
+
+    FileUtils.writeStringToFile(amsSite, "<configuration>\n" +
+      "  <property>\n" +
+      "    <name>test</name>\n" +
+      "    <value>testReady</value>\n" +
+      "  </property>\n" +
+      "  <property>\n" +
+      "    <name>timeline.metrics.host.aggregator.hourly.disabled</name>\n" +
+      "    <value>true</value>\n" +
+      "    <description>\n" +
+      "      Disable host based hourly aggregations.\n" +
+      "    </description>\n" +
+      "  </property>\n" +
+      "  <property>\n" +
+      "    <name>timeline.metrics.host.aggregator.minute.disabled</name>\n" +
+      "    <value>true</value>\n" +
+      "    <description>\n" +
+      "      Disable host based minute aggregations.\n" +
+      "    </description>\n" +
+      "  </property>\n" +
+      "  <property>\n" +
+      "    <name>timeline.metrics.cluster.aggregator.hourly.disabled</name>\n" +
+      "    <value>true</value>\n" +
+      "    <description>\n" +
+      "      Disable cluster based hourly aggregations.\n" +
+      "    </description>\n" +
+      "  </property>\n" +
+      "  <property>\n" +
+      "    <name>timeline.metrics.cluster.aggregator.minute.disabled</name>\n" +
+      "    <value>true</value>\n" +
+      "    <description>\n" +
+      "      Disable cluster based minute aggregations.\n" +
+      "    </description>\n" +
+      "  </property>" +
+      "</configuration>");
+
+    ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+    // Add the conf dir to the classpath
+    // Chain the current thread classloader
+    URLClassLoader urlClassLoader = null;
+    try {
+      urlClassLoader = new URLClassLoader(new URL[] {
+        folder.getRoot().toURI().toURL() }, currentClassLoader);
+    } catch (MalformedURLException e) {
+      e.printStackTrace();
+    }
+
+    Thread.currentThread().setContextClassLoader(urlClassLoader);
+    metricsConf = new Configuration(false);
+    metricsConf.addResource(Thread.currentThread().getContextClassLoader()
+      .getResource(METRICS_SITE_CONFIGURATION_FILE).toURI().toURL());
+    assertNotNull(metricsConf.get("test"));
+  }
+
+  // simple test init/start/stop ApplicationHistoryServer. Status should change.
+  @Test(timeout = 50000)
+  public void testStartStopServer() throws Exception {
+    Configuration config = new YarnConfiguration();
+    UserGroupInformation ugi =
+      UserGroupInformation.createUserForTesting("ambari", new String[] {"ambari"});
+
+    mockStatic(UserGroupInformation.class);
+    expect(UserGroupInformation.getCurrentUser()).andReturn(ugi).anyTimes();
+    expect(UserGroupInformation.isSecurityEnabled()).andReturn(false).anyTimes();
+    config.set(YarnConfiguration.APPLICATION_HISTORY_STORE,
+      "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore");
+
+    Connection connection = createNiceMock(Connection.class);
+    Statement stmt = createNiceMock(Statement.class);
+    mockStatic(DriverManager.class);
+    expect(DriverManager.getConnection("jdbc:phoenix:localhost:2181:/hbase"))
+      .andReturn(connection).anyTimes();
+    expect(connection.createStatement()).andReturn(stmt).anyTimes();
+    suppress(method(Statement.class, "executeUpdate", String.class));
+    connection.close();
+    expectLastCall();
+
+    EasyMock.replay(connection, stmt);
+    replayAll();
+
+    historyServer = new ApplicationHistoryServer();
+    historyServer.init(config);
+
+    verifyAll();
+
+    assertEquals(STATE.INITED, historyServer.getServiceState());
+    assertEquals(4, historyServer.getServices().size());
+    ApplicationHistoryClientService historyService =
+      historyServer.getClientService();
+    assertNotNull(historyServer.getClientService());
+    assertEquals(STATE.INITED, historyService.getServiceState());
+
+    historyServer.start();
+    assertEquals(STATE.STARTED, historyServer.getServiceState());
+    assertEquals(STATE.STARTED, historyService.getServiceState());
+    historyServer.stop();
+    assertEquals(STATE.STOPPED, historyServer.getServiceState());
+  }
+
+  // test launch method
+  @Ignore
+  @Test(timeout = 60000)
+  public void testLaunch() throws Exception {
+
+    UserGroupInformation ugi =
+      UserGroupInformation.createUserForTesting("ambari", new String[]{"ambari"});
+    mockStatic(UserGroupInformation.class);
+    expect(UserGroupInformation.getCurrentUser()).andReturn(ugi).anyTimes();
+    expect(UserGroupInformation.isSecurityEnabled()).andReturn(false).anyTimes();
+
+    ExitUtil.disableSystemExit();
+    try {
+      historyServer = ApplicationHistoryServer.launchAppHistoryServer(new String[0]);
+    } catch (ExitUtil.ExitException e) {
+      assertEquals(0, e.status);
+      ExitUtil.resetFirstExitException();
+      fail();
+    }
+  }
+
+  @After
+  public void stop() {
+    if (historyServer != null) {
+      historyServer.stop();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
new file mode 100644
index 0000000..bc16d36
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+import java.net.URI;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFileSystemApplicationHistoryStore extends
+    ApplicationHistoryStoreTestUtils {
+
+  private FileSystem fs;
+  private Path fsWorkingPath;
+
+  @Before
+  public void setup() throws Exception {
+    fs = new RawLocalFileSystem();
+    Configuration conf = new Configuration();
+    fs.initialize(new URI("/"), conf);
+    fsWorkingPath = new Path("Test");
+    fs.delete(fsWorkingPath, true);
+    conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
+    store = new FileSystemApplicationHistoryStore();
+    store.init(conf);
+    store.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    store.stop();
+    fs.delete(fsWorkingPath, true);
+    fs.close();
+  }
+
+  @Test
+  public void testReadWriteHistoryData() throws IOException {
+    testWriteHistoryData(5);
+    testReadHistoryData(5);
+  }
+
+  private void testWriteHistoryData(int num) throws IOException {
+    testWriteHistoryData(num, false, false);
+  }
+  
+  private void testWriteHistoryData(
+      int num, boolean missingContainer, boolean missingApplicationAttempt)
+          throws IOException {
+    // write application history data
+    for (int i = 1; i <= num; ++i) {
+      ApplicationId appId = ApplicationId.newInstance(0, i);
+      writeApplicationStartData(appId);
+
+      // write application attempt history data
+      for (int j = 1; j <= num; ++j) {
+        ApplicationAttemptId appAttemptId =
+            ApplicationAttemptId.newInstance(appId, j);
+        writeApplicationAttemptStartData(appAttemptId);
+
+        if (missingApplicationAttempt && j == num) {
+          continue;
+        }
+        // write container history data
+        for (int k = 1; k <= num; ++k) {
+          ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
+          writeContainerStartData(containerId);
+          if (missingContainer && k == num) {
+            continue;
+          }
+          writeContainerFinishData(containerId);
+        }
+        writeApplicationAttemptFinishData(appAttemptId);
+      }
+      writeApplicationFinishData(appId);
+    }
+  }
+
+  private void testReadHistoryData(int num) throws IOException {
+    testReadHistoryData(num, false, false);
+  }
+  
+  private void testReadHistoryData(
+      int num, boolean missingContainer, boolean missingApplicationAttempt)
+          throws IOException {
+    // read application history data
+    Assert.assertEquals(num, store.getAllApplications().size());
+    for (int i = 1; i <= num; ++i) {
+      ApplicationId appId = ApplicationId.newInstance(0, i);
+      ApplicationHistoryData appData = store.getApplication(appId);
+      Assert.assertNotNull(appData);
+      Assert.assertEquals(appId.toString(), appData.getApplicationName());
+      Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo());
+
+      // read application attempt history data
+      Assert.assertEquals(num, store.getApplicationAttempts(appId).size());
+      for (int j = 1; j <= num; ++j) {
+        ApplicationAttemptId appAttemptId =
+            ApplicationAttemptId.newInstance(appId, j);
+        ApplicationAttemptHistoryData attemptData =
+            store.getApplicationAttempt(appAttemptId);
+        Assert.assertNotNull(attemptData);
+        Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
+        
+        if (missingApplicationAttempt && j == num) {
+          Assert.assertNull(attemptData.getDiagnosticsInfo());
+          continue;
+        } else {
+          Assert.assertEquals(appAttemptId.toString(),
+              attemptData.getDiagnosticsInfo());
+        }
+
+        // read container history data
+        Assert.assertEquals(num, store.getContainers(appAttemptId).size());
+        for (int k = 1; k <= num; ++k) {
+          ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
+          ContainerHistoryData containerData = store.getContainer(containerId);
+          Assert.assertNotNull(containerData);
+          Assert.assertEquals(Priority.newInstance(containerId.getId()),
+            containerData.getPriority());
+          if (missingContainer && k == num) {
+            Assert.assertNull(containerData.getDiagnosticsInfo());
+          } else {
+            Assert.assertEquals(containerId.toString(),
+                containerData.getDiagnosticsInfo());
+          }
+        }
+        ContainerHistoryData masterContainer =
+            store.getAMContainer(appAttemptId);
+        Assert.assertNotNull(masterContainer);
+        Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
+          masterContainer.getContainerId());
+      }
+    }
+  }
+
+  @Test
+  public void testWriteAfterApplicationFinish() throws IOException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    writeApplicationFinishData(appId);
+    // write application attempt history data
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    try {
+      writeApplicationAttemptStartData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is not opened"));
+    }
+    try {
+      writeApplicationAttemptFinishData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is not opened"));
+    }
+    // write container history data
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    try {
+      writeContainerStartData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is not opened"));
+    }
+    try {
+      writeContainerFinishData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is not opened"));
+    }
+  }
+
+  @Test
+  public void testMassiveWriteContainerHistoryData() throws IOException {
+    long mb = 1024 * 1024;
+    long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    writeApplicationStartData(appId);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    for (int i = 1; i <= 100000; ++i) {
+      ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
+      writeContainerStartData(containerId);
+      writeContainerFinishData(containerId);
+    }
+    writeApplicationFinishData(appId);
+    long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
+    Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20);
+  }
+
+  @Test
+  public void testMissingContainerHistoryData() throws IOException {
+    testWriteHistoryData(3, true, false);
+    testReadHistoryData(3, true, false);
+  }
+  
+  @Test
+  public void testMissingApplicationAttemptHistoryData() throws IOException {
+    testWriteHistoryData(3, false, true);
+    testReadHistoryData(3, false, true);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
new file mode 100644
index 0000000..7a45405
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMemoryApplicationHistoryStore extends
+    ApplicationHistoryStoreTestUtils {
+
+  @Before
+  public void setup() {
+    store = new MemoryApplicationHistoryStore();
+  }
+
+  @Test
+  public void testReadWriteApplicationHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    try {
+      writeApplicationFinishData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+        "is stored before the start information"));
+    }
+    // Normal
+    int numApps = 5;
+    for (int i = 1; i <= numApps; ++i) {
+      appId = ApplicationId.newInstance(0, i);
+      writeApplicationStartData(appId);
+      writeApplicationFinishData(appId);
+    }
+    Assert.assertEquals(numApps, store.getAllApplications().size());
+    for (int i = 1; i <= numApps; ++i) {
+      appId = ApplicationId.newInstance(0, i);
+      ApplicationHistoryData data = store.getApplication(appId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(appId.toString(), data.getApplicationName());
+      Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo());
+    }
+    // Write again
+    appId = ApplicationId.newInstance(0, 1);
+    try {
+      writeApplicationStartData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeApplicationFinishData(appId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+  }
+
+  @Test
+  public void testReadWriteApplicationAttemptHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    try {
+      writeApplicationAttemptFinishData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+        "is stored before the start information"));
+    }
+    // Normal
+    int numAppAttempts = 5;
+    writeApplicationStartData(appId);
+    for (int i = 1; i <= numAppAttempts; ++i) {
+      appAttemptId = ApplicationAttemptId.newInstance(appId, i);
+      writeApplicationAttemptStartData(appAttemptId);
+      writeApplicationAttemptFinishData(appAttemptId);
+    }
+    Assert.assertEquals(numAppAttempts, store.getApplicationAttempts(appId)
+      .size());
+    for (int i = 1; i <= numAppAttempts; ++i) {
+      appAttemptId = ApplicationAttemptId.newInstance(appId, i);
+      ApplicationAttemptHistoryData data =
+          store.getApplicationAttempt(appAttemptId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(appAttemptId.toString(), data.getHost());
+      Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
+    }
+    writeApplicationFinishData(appId);
+    // Write again
+    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    try {
+      writeApplicationAttemptStartData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeApplicationAttemptFinishData(appAttemptId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+  }
+
+  @Test
+  public void testReadWriteContainerHistory() throws Exception {
+    // Out of order
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    try {
+      writeContainerFinishData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains(
+        "is stored before the start information"));
+    }
+    // Normal
+    writeApplicationAttemptStartData(appAttemptId);
+    int numContainers = 5;
+    for (int i = 1; i <= numContainers; ++i) {
+      containerId = ContainerId.newInstance(appAttemptId, i);
+      writeContainerStartData(containerId);
+      writeContainerFinishData(containerId);
+    }
+    Assert
+      .assertEquals(numContainers, store.getContainers(appAttemptId).size());
+    for (int i = 1; i <= numContainers; ++i) {
+      containerId = ContainerId.newInstance(appAttemptId, i);
+      ContainerHistoryData data = store.getContainer(containerId);
+      Assert.assertNotNull(data);
+      Assert.assertEquals(Priority.newInstance(containerId.getId()),
+        data.getPriority());
+      Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo());
+    }
+    ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
+    Assert.assertNotNull(masterContainer);
+    Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
+      masterContainer.getContainerId());
+    writeApplicationAttemptFinishData(appAttemptId);
+    // Write again
+    containerId = ContainerId.newInstance(appAttemptId, 1);
+    try {
+      writeContainerStartData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+    try {
+      writeContainerFinishData(containerId);
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e.getMessage().contains("is already stored"));
+    }
+  }
+
+  @Test
+  public void testMassiveWriteContainerHistory() throws IOException {
+    long mb = 1024 * 1024;
+    Runtime runtime = Runtime.getRuntime();
+    long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+    int numContainers = 100000;
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    for (int i = 1; i <= numContainers; ++i) {
+      ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
+      writeContainerStartData(containerId);
+      writeContainerFinishData(containerId);
+    }
+    long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+    Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
new file mode 100644
index 0000000..499dab6
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestAppMetrics.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
+
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAppMetrics {
+  private static final String SAMPLE_SINGLE_METRIC_HOST_JSON = "{\n" +
+    "  \"metrics\" : [ {\n" +
+    "    \"instanceid\" : \"\",\n" +
+    "    \"hostname\" : \"localhost\",\n" +
+    "    \"metrics\" : {\n" +
+    "      \"0\" : \"5.35\",\n" +
+    "      \"5000\" : \"5.35\",\n" +
+    "      \"10000\" : \"5.35\",\n" +
+    "      \"15000\" : \"5.35\"\n" +
+    "    },\n" +
+    "    \"starttime\" : \"1411663170112\",\n" +
+    "    \"appid\" : \"HOST\",\n" +
+    "    \"metricname\" : \"disk_free\"\n" +
+    "  } ]\n" +
+    "}";
+
+  private static final String SAMPLE_TWO_METRIC_HOST_JSON = "{\n" +
+    "  \"metrics\" : [ {\n" +
+    "    \"instanceid\" : \"\",\n" +
+    "    \"hostname\" : \"localhost\",\n" +
+    "    \"metrics\" : {\n" +
+    "      \"0\" : \"5.35\",\n" +
+    "      \"5000\" : \"5.35\",\n" +
+    "      \"10000\" : \"5.35\",\n" +
+    "      \"15000\" : \"5.35\"\n" +
+    "    },\n" +
+    "    \"starttime\" : \"0\",\n" +
+    "    \"appid\" : \"HOST\",\n" +
+    "    \"metricname\" : \"disk_free\"\n" +
+    "  }, {\n" +
+    "    \"instanceid\" : \"\",\n" +
+    "    \"hostname\" : \"localhost\",\n" +
+    "    \"metrics\" : {\n" +
+    "      \"0\" : \"94.0\",\n" +
+    "      \"5000\" : \"94.0\",\n" +
+    "      \"10000\" : \"94.0\",\n" +
+    "      \"15000\" : \"94.0\"\n" +
+    "    },\n" +
+    "    \"starttime\" : \"0\",\n" +
+    "    \"appid\" : \"HOST\",\n" +
+    "    \"metricname\" : \"mem_cached\"\n" +
+    "  } ]\n" +
+    "}";
+
+  private long[] timestamps;
+
+  @Before
+  public void setUp() throws Exception {
+    timestamps = new long[4];
+    timestamps[0] = 0;
+    timestamps[1] = timestamps[0] + 5000;
+    timestamps[2] = timestamps[1] + 5000;
+    timestamps[3] = timestamps[2] + 5000;
+
+  }
+
+  @Test
+  public void testHostDiskMetricsSerialization() throws IOException {
+    long timestamp = 1411663170112L;
+    AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamp);
+
+    Metric diskFree = appMetrics.createMetric("disk_free");
+    double value = 5.35;
+
+    diskFree.putMetric(timestamps[0], Double.toString(value));
+    diskFree.putMetric(timestamps[1], Double.toString(value));
+    diskFree.putMetric(timestamps[2], Double.toString(value));
+    diskFree.putMetric(timestamps[3], Double.toString(value));
+
+    appMetrics.addMetric(diskFree);
+
+    String expected = SAMPLE_SINGLE_METRIC_HOST_JSON;
+    String s = new Json(true).serialize(appMetrics);
+
+    assertEquals("Serialized Host Metrics", expected, s);
+  }
+
+
+  @Test
+  public void testSingleHostManyMetricsSerialization() throws IOException {
+    AppMetrics appMetrics = new AppMetrics(new ApplicationInstance("localhost", AppID.HOST, ""), timestamps[0]);
+
+    Metric diskFree = appMetrics.createMetric("disk_free");
+    double value = 5.35;
+    diskFree.putMetric(timestamps[0], Double.toString(value));
+    diskFree.putMetric(timestamps[1], Double.toString(value));
+    diskFree.putMetric(timestamps[2], Double.toString(value));
+    diskFree.putMetric(timestamps[3], Double.toString(value));
+
+    appMetrics.addMetric(diskFree);
+
+    Metric memCache = appMetrics.createMetric("mem_cached");
+    double memVal = 94;
+    memCache.putMetric(timestamps[0], Double.toString(memVal));
+    memCache.putMetric(timestamps[1], Double.toString(memVal));
+    memCache.putMetric(timestamps[2], Double.toString(memVal));
+    memCache.putMetric(timestamps[3], Double.toString(memVal));
+
+    appMetrics.addMetric(memCache);
+
+    String expected = SAMPLE_TWO_METRIC_HOST_JSON;
+    String s = new Json(true).serialize(appMetrics);
+
+    assertEquals("Serialized Host Metrics", expected, s);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/a52f8a55/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
new file mode 100644
index 0000000..a0572a2
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/TestMetric.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data;
+
+import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.util.Json;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.entry;
+import static org.junit.Assert.assertEquals;
+
+public class TestMetric {
+  private static final String SAMPLE_METRIC_IN_JSON = "{\n" +
+    "  \"instanceid\" : \"\",\n" +
+    "  \"hostname\" : \"localhost\",\n" +
+    "  \"metrics\" : {\n" +
+    "    \"0\" : \"5.35\",\n" +
+    "    \"5000\" : \"5.35\",\n" +
+    "    \"10000\" : \"5.35\",\n" +
+    "    \"15000\" : \"5.35\"\n" +
+    "  },\n" +
+    "  \"starttime\" : \"0\",\n" +
+    "  \"appid\" : \"HOST\",\n" +
+    "  \"metricname\" : \"disk_free\"\n" +
+    "}";
+
+  @Test
+  public void testSerializeToJson() throws IOException {
+    Metric diskOnHostMetric = new Metric(new ApplicationInstance("localhost", AppID.HOST, ""), "disk_free", 0);
+
+    long timestamp = 0;
+    double value = 5.35;
+
+    diskOnHostMetric.putMetric(timestamp, Double.toString(value));
+    diskOnHostMetric.putMetric(timestamp + 5000, Double.toString(value));
+    diskOnHostMetric.putMetric(timestamp + 10000, Double.toString(value));
+    diskOnHostMetric.putMetric(timestamp + 15000, Double.toString(value));
+
+    String expected = SAMPLE_METRIC_IN_JSON;
+    String s = new Json(true).serialize(diskOnHostMetric);
+
+    assertEquals("Json should match", expected, s);
+  }
+
+  @Test
+  public void testDeserializeObjectFromString() throws IOException {
+    String source = SAMPLE_METRIC_IN_JSON;
+
+    Metric m = new Json().deserialize(source, Metric.class);
+
+    assertEquals("localhost", m.getHostname());
+    assertEquals("HOST", m.getAppid());
+    assertEquals("", m.getInstanceid());
+    assertEquals("disk_free", m.getMetricname());
+    assertEquals("0", m.getStarttime());
+
+    assertThat(m.getMetrics()).isNotEmpty().hasSize(4).contains(
+      entry("0", "5.35"),
+      entry("5000", "5.35"),
+      entry("10000", "5.35"),
+      entry("15000", "5.35"));
+  }
+}
\ No newline at end of file


Mime
View raw message