ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-9954. Spark on tez apps fails needs tez.tar.gz copied to HDFS (alejandro)
Date Mon, 09 Mar 2015 21:35:57 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.0.0 f933450b9 -> 29e8780e3


AMBARI-9954. Spark on tez apps fails needs tez.tar.gz copied to HDFS (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/29e8780e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/29e8780e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/29e8780e

Branch: refs/heads/branch-2.0.0
Commit: 29e8780e360f2d44a3dac54e3919de8d70fba291
Parents: f933450
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Mon Mar 9 14:35:19 2015 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Mon Mar 9 14:35:19 2015 -0700

----------------------------------------------------------------------
 .../SPARK/1.2.0.2.2/metainfo.xml                |   7 +
 .../package/scripts/job_history_server.py       |  10 +-
 .../stacks/2.2/SPARK/test_job_history_server.py | 106 ++++++++++++
 .../2.2/configs/spark-job-history-server.json   | 165 +++++++++++++++++++
 4 files changed, 286 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/29e8780e/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
index ce8ad7a..5cd6038 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
@@ -33,6 +33,13 @@
           <versionAdvertised>true</versionAdvertised>
           <dependencies>
             <dependency>
+              <name>TEZ/TEZ_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
               <name>HDFS/HDFS_CLIENT</name>
               <scope>host</scope>
               <auto-deploy>

http://git-wip-us.apache.org/repos/asf/ambari/blob/29e8780e/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
index 518352f..ff04b58 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
@@ -20,8 +20,11 @@ limitations under the License.
 
 import sys
 import os
-from resource_management import *
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.core.resources import Execute
 from resource_management.core.exceptions import ComponentIsNotRunning
 from resource_management.core.logger import Logger
 from resource_management.core import shell
@@ -39,6 +42,7 @@ class JobHistoryServer(Script):
     env.set_params(params)
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
       Execute(format("hdp-select set spark-historyserver {version}"))
+      copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user,
params.user_group)
 
   def install(self, env):
     self.install_packages(env)
@@ -62,12 +66,14 @@ class JobHistoryServer(Script):
     import params
 
     env.set_params(params)
-    setup_spark(env, 'server', action = 'start')
+    setup_spark(env, 'server', action='start')
 
     if params.security_enabled:
       spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal};
")
       Execute(spark_kinit_cmd, user=params.spark_user)
 
+    copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user,
params.user_group)
+
     daemon_cmd = format('{spark_history_server_start}')
     no_op_test = format(
       'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat
{spark_history_server_pid_file}` >/dev/null 2>&1')

http://git-wip-us.apache.org/repos/asf/ambari/blob/29e8780e/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
new file mode 100644
index 0000000..469f68e
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import sys
+import os
+from mock.mock import MagicMock, patch
+
+from stacks.utils.RMFTestCase import *
+from resource_management.core import shell
+from resource_management.libraries.functions import dynamic_variable_interpretation
+
+class TestJobHistoryServer(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
+  STACK_VERSION = "2.2"
+
+  def setUp(self):
+    sys.path.insert(0, os.path.join(os.getcwd(),
+      "../../main/resources/common-services", self.COMMON_SERVICES_PACKAGE_DIR,
+      "scripts"))
+
+  @patch.object(shell, "call")
+  @patch("setup_spark.create_file")
+  @patch("setup_spark.write_properties_to_file")
+  @patch.object(dynamic_variable_interpretation, "copy_tarballs_to_hdfs")
+  def test_start(self, copy_tarball_mock, write_properties_to_file_mock, create_file_mock,
call_mock):
+    hdp_version = "2.2.2.0-2538"
+    call_mock.return_value = (0, hdp_version)
+    copy_tarball_mock.return_value = 0
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+                         classname="JobHistoryServer",
+                         command="start",
+                         config_file="spark-job-history-server.json",
+                         hdp_stack_version=self.STACK_VERSION,
+                         target=RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    self.assertTrue(create_file_mock.called)
+    self.assertTrue(write_properties_to_file_mock.called)
+
+
+    self.assertResourceCalled("Directory", "/var/run/spark",
+                              owner="spark",
+                              group="hadoop",
+                              recursive=True
+    )
+    self.assertResourceCalled("Directory", "/var/log/spark",
+                              owner="spark",
+                              group="hadoop",
+                              recursive=True
+    )
+    self.assertResourceCalled("HdfsDirectory", "/user/spark",
+                              security_enabled=False,
+                              keytab=UnknownConfigurationMock(),
+                              conf_dir="/etc/hadoop/conf",
+                              hdfs_user="hdfs",
+                              kinit_path_local="/usr/bin/kinit",
+                              mode=509,
+                              owner="spark",
+                              bin_dir="/usr/hdp/current/hadoop-client/bin",
+                              action=["create"]
+    )
+    self.assertResourceCalled("File", "/etc/spark/conf/spark-env.sh",
+                              owner="spark",
+                              group="spark",
+                              content=InlineTemplate(self.getConfig()['configurations']['spark-env']['content'])
+    )
+    self.assertResourceCalled("File", "/etc/spark/conf/log4j.properties",
+                              owner="spark",
+                              group="spark",
+                              content=self.getConfig()['configurations']['spark-log4j-properties']['content']
+    )
+    self.assertResourceCalled("File", "/etc/spark/conf/metrics.properties",
+                              owner="spark",
+                              group="spark",
+                              content=InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content'])
+    )
+    self.assertResourceCalled("File", "/etc/spark/conf/java-opts",
+                              owner="spark",
+                              group="spark",
+                              content="  -Dhdp.version=" + hdp_version
+    )
+
+    copy_tarball_mock.assert_called_with("tez", "spark-historyserver", "spark", "hdfs", "hadoop")
+    
+    self.assertResourceCalled("Execute", "/usr/hdp/current/spark-historyserver/sbin/start-history-server.sh",
+                              not_if="ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid
>/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid`
>/dev/null 2>&1",
+                              environment={'JAVA_HOME': '/usr/jdk64/jdk1.7.0_67'},
+                              user="spark"
+    )

http://git-wip-us.apache.org/repos/asf/ambari/blob/29e8780e/ambari-server/src/test/python/stacks/2.2/configs/spark-job-history-server.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/spark-job-history-server.json
b/ambari-server/src/test/python/stacks/2.2/configs/spark-job-history-server.json
new file mode 100644
index 0000000..e9f6e9a
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.2/configs/spark-job-history-server.json
@@ -0,0 +1,165 @@
+{
+    "configuration_attributes": {
+        "spark-defaults": {},
+        "spark-javaopts-properties": {}, 
+        "spark-log4j-properties": {},
+        "spark-env": {},
+        "core-site": {
+            "final": {
+                "fs.defaultFS": "true"
+            }
+        },
+        "hadoop-env": {},
+        "spark-metrics-properties": {},
+        "cluster-env": {}
+    }, 
+    "commandParams": {
+        "service_package_folder": "common-services/SPARK/1.2.0.2.2/package", 
+        "script": "scripts/job_history_server.py", 
+        "hooks_folder": "HDP/2.0.6/hooks", 
+        "version": "2.2.2.0-2538", 
+        "command_timeout": "600", 
+        "script_type": "PYTHON"
+    }, 
+    "roleCommand": "START", 
+    "kerberosCommandParams": [], 
+    "clusterName": "c1", 
+    "hostname": "c6408.ambari.apache.org", 
+    "hostLevelParams": {
+        "jdk_location": "http://c6408.ambari.apache.org:8080/resources/", 
+        "ambari_db_rca_password": "mapred", 
+        "java_home": "/usr/jdk64/jdk1.7.0_67", 
+        "ambari_db_rca_url": "jdbc:postgresql://c6408.ambari.apache.org/ambarirca",
+        "repo_info": "[{\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.2.0-2538\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.2\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0\",\"latestBaseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.2.2.0-2538\",\"baseSaved\":true},{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.20\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\",\"baseSaved\":true}]",

+        "group_list": "[\"hadoop\",\"users\",\"spark\"]", 
+        "package_list": "[{\"name\":\"spark_2_2_*\"},{\"name\":\"spark_2_2_*-python\"}]",

+        "stack_version": "2.2", 
+        "stack_name": "HDP", 
+        "db_name": "ambari",
+        "user_list": "[\"mapred\",\"ambari-qa\",\"zookeeper\",\"tez\",\"false\",\"hdfs\",\"yarn\",\"spark\"]",
+        "clientsToUpdateConfigs": "[\"*\"]"
+    }, 
+    "commandType": "EXECUTION_COMMAND", 
+    "roleParams": {}, 
+    "serviceName": "SPARK", 
+    "role": "SPARK_JOBHISTORYSERVER", 
+    "forceRefreshConfigTags": [], 
+    "taskId": 70, 
+    "public_hostname": "c6408.ambari.apache.org", 
+    "configurations": {
+        "spark-defaults": {
+            "spark.yarn.applicationMaster.waitTries": "10", 
+            "spark.history.kerberos.keytab": "none", 
+            "spark.yarn.preserve.staging.files": "false", 
+            "spark.yarn.submit.file.replication": "3", 
+            "spark.history.kerberos.principal": "none", 
+            "spark.yarn.driver.memoryOverhead": "384", 
+            "spark.yarn.queue": "default", 
+            "spark.yarn.containerLauncherMaxThreads": "25", 
+            "spark.yarn.scheduler.heartbeat.interval-ms": "5000", 
+            "spark.history.ui.port": "18080", 
+            "spark.yarn.max.executor.failures": "3", 
+            "spark.driver.extraJavaOptions": "", 
+            "spark.history.provider": "org.apache.spark.deploy.yarn.history.YarnHistoryProvider",

+            "spark.yarn.am.extraJavaOptions": "", 
+            "spark.yarn.executor.memoryOverhead": "384"
+        },
+        "spark-javaopts-properties": {
+            "content": " "
+        }, 
+        "spark-log4j-properties": {
+            "content": "\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO,
console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd
HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO"
+        },
+        "spark-env": {
+            "content": "\n#!/usr/bin/env bash\n\n# This file is sourced when running various
Spark programs.\n# Copy it as spark-env.sh and edit that to configure Spark for your site.\n\n#
Options read in YARN client mode\n#SPARK_EXECUTOR_INSTANCES=\"2\" #Number of workers to start
(Default: 2)\n#SPARK_EXECUTOR_CORES=\"1\" #Number of cores for the workers (Default: 1).\n#SPARK_EXECUTOR_MEMORY=\"1G\"
#Memory per Worker (e.g. 1000M, 2G) (Default: 1G)\n#SPARK_DRIVER_MEMORY=\"512 Mb\" #Memory
for Master (e.g. 1000M, 2G) (Default: 512 Mb)\n#SPARK_YARN_APP_NAME=\"spark\" #The name of
your application (Default: Spark)\n#SPARK_YARN_QUEUE=\"~@~Xdefault~@~Y\" #The hadoop queue
to use for allocation requests (Default: @~Xdefault~@~Y)\n#SPARK_YARN_DIST_FILES=\"\" #Comma
separated list of files to be distributed with the job.\n#SPARK_YARN_DIST_ARCHIVES=\"\" #Comma
separated list of archives to be distributed with the job.\n\n# Generic options for the daemons
used in the standalone deploy mode\n\
 n# Alternate conf dir. (Default: ${SPARK_HOME}/conf)\nexport SPARK_CONF_DIR=${SPARK_HOME:-{{spark_home}}}/conf\n\n#
Where log files are stored.(Default:${SPARK_HOME}/logs)\n#export SPARK_LOG_DIR=${SPARK_HOME:-{{spark_home}}}/logs\nexport
SPARK_LOG_DIR={{spark_log_dir}}\n\n# Where the pid file is stored. (Default: /tmp)\nexport
SPARK_PID_DIR={{spark_pid_dir}}\n\n# A string representing this instance of spark.(Default:
$USER)\nSPARK_IDENT_STRING=$USER\n\n# The scheduling priority for daemons. (Default: 0)\nSPARK_NICENESS=0\n\nexport
HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\nexport HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}\n\n#
The java implementation to use.\nexport JAVA_HOME={{java_home}}\n\nif [ -d \"/etc/tez/conf/\"
]; then\n  export TEZ_CONF_DIR=/etc/tez/conf\nelse\n  export TEZ_CONF_DIR=\nfi", 
+            "spark_pid_dir": "/var/run/spark", 
+            "spark_log_dir": "/var/log/spark", 
+            "spark_group": "spark", 
+            "spark_user": "spark"
+        },
+        "hadoop-env": {
+            "dtnode_heapsize": "1024m", 
+            "namenode_opt_maxnewsize": "256m", 
+            "hdfs_log_dir_prefix": "/var/log/hadoop", 
+            "namenode_heapsize": "1024m", 
+            "proxyuser_group": "users", 
+            "hadoop_pid_dir_prefix": "/var/run/hadoop", 
+            "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only
required environment variable is JAVA_HOME.  All others are\n# optional.  When running a distributed
configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined
on\n# remote nodes.\n\n# The java implementation to use.  Required.\nexport JAVA_HOME={{java_home}}\nexport
HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n#
Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required
by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap
to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport
HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options.
 Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n#
Command specific options appende
 d to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8
-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}}
-XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}}
-Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}}
-Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server
-XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log
-XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date
+'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStam
 ps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT
-Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server
-Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console
${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4
-XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m
-XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date
+'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps
-Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT
${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport
HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The following
  applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m
-XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode
as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n#
Extra ssh options.  Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n#
Where log files are stored.  $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n#
History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where
log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n#
File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n#
host:path where hadoop code should be rsync'd from.  Unset by default.\n# export HAD
 OOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands.  Unset
by default.  This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise
arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The
directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport
HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server
pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n#
A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n#
The scheduling priority for daemon processes.  See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n#
Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql
connector\nfor jarFile in `ls /usr/share/
 java/*mysql* 2>/dev/null`\ndo\n  JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add
libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n
 JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the
HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n  if [ -d \"/etc/tez/conf/\"
]; then\n    # When using versioned RPMs, the tez-client will be a symlink to the current
folder of tez in HDP.\n    export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n
 fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n#
Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION
$HADOOP_
 OPTS\"", 
+            "hdfs_user": "hdfs", 
+            "namenode_opt_newsize": "256m", 
+            "dfs.datanode.data.dir.mount.file": "/etc/hadoop/conf/dfs_data_dir_mount.hist",

+            "hadoop_root_logger": "INFO,RFA", 
+            "hadoop_heapsize": "1024", 
+            "namenode_opt_maxpermsize": "256m", 
+            "namenode_opt_permsize": "128m"
+        },
+        "spark-metrics-properties": {
+            "content": "\n# syntax: [instance].sink|source.[name].[options]=[value]\n\n#
This file configures Spark's internal metrics system. The metrics system is\n# divided into
instances which correspond to internal components.\n# Each instance can be configured to report
its metrics to one or more sinks.\n# Accepted values for [instance] are \"master\", \"worker\",
\"executor\", \"driver\",\n# and \"applications\". A wild card \"*\" can be used as an instance
name, in\n# which case all instances will inherit the supplied property.\n#\n# Within an instance,
a \"source\" specifies a particular set of grouped metrics.\n# there are two kinds of sources:\n#
1. Spark internal sources, like MasterSource, WorkerSource, etc, which will\n# collect a Spark
component's internal state. Each instance is paired with a\n# Spark source that is added automatically.\n#
2. Common sources, like JvmSource, which will collect low level state.\n# These can be added
through configuration options and ar
 e then loaded\n# using reflection.\n#\n# A \"sink\" specifies where metrics are delivered
to. Each instance can be\n# assigned one or more sinks.\n#\n# The sink|source field specifies
whether the property relates to a sink or\n# source.\n#\n# The [name] field specifies the
name of source or sink.\n#\n# The [options] field is the specific property of this source
or sink. The\n# source or sink is responsible for parsing this property.\n#\n# Notes:\n# 1.
To add a new sink, set the \"class\" option to a fully qualified class\n# name (see examples
below).\n# 2. Some sinks involve a polling period. The minimum allowed polling period\n# is
1 second.\n# 3. Wild card properties can be overridden by more specific properties.\n# For
example, master.sink.console.period takes precedence over\n# *.sink.console.period.\n# 4.
A metrics specific configuration\n# \"spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties\"
should be\n# added to Java properties using -Dspark.metrics.conf=xxx if you wa
 nt to\n# customize metrics system. You can also put the file in ${SPARK_HOME}/conf\n# and
it will be loaded automatically.\n# 5. MetricsServlet is added by default as a sink in master,
worker and client\n# driver, you can send http request \"/metrics/json\" to get a snapshot
of all the\n# registered metrics in json format. For master, requests \"/metrics/master/json\"
and\n# \"/metrics/applications/json\" can be sent seperately to get metrics snapshot of\n#
instance master and applications. MetricsServlet may not be configured by self.\n#\n\n## List
of available sinks and their properties.\n\n# org.apache.spark.metrics.sink.ConsoleSink\n#
Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n\n#
org.apache.spark.metrics.sink.CSVSink\n# Name: Default: Description:\n# period 10 Poll period\n#
unit seconds Units of poll period\n# directory /tmp Where to store CSV files\n\n# org.apache.spark.metrics.sink.GangliaSink\n#
Name: Default: Description:\n# 
 host NONE Hostname or multicast group of Ganglia server\n# port NONE Port of Ganglia server(s)\n#
period 10 Poll period\n# unit seconds Units of poll period\n# ttl 1 TTL of messages sent by
Ganglia\n# mode multicast Ganglia network mode ('unicast' or 'multicast')\n\n# org.apache.spark.metrics.sink.JmxSink\n\n#
org.apache.spark.metrics.sink.MetricsServlet\n# Name: Default: Description:\n# path VARIES*
Path prefix from the web server root\n# sample false Whether to show entire set of samples
for histograms ('false' or 'true')\n#\n# * Default path is /metrics/json for all instances
except the master. The master has two paths:\n# /metrics/aplications/json # App information\n#
/metrics/master/json # Master information\n\n# org.apache.spark.metrics.sink.GraphiteSink\n#
Name: Default: Description:\n# host NONE Hostname of Graphite server\n# port NONE Port of
Graphite server\n# period 10 Poll period\n# unit seconds Units of poll period\n# prefix EMPTY
STRING Prefix to prepend to metric name
 \n\n## Examples\n# Enable JmxSink for all instances by class name\n#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n\n#
Enable ConsoleSink for all instances by class name\n#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink\n\n#
Polling period for ConsoleSink\n#*.sink.console.period=10\n\n#*.sink.console.unit=seconds\n\n#
Master instance overlap polling period\n#master.sink.console.period=15\n\n#master.sink.console.unit=seconds\n\n#
Enable CsvSink for all instances\n#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink\n\n#
Polling period for CsvSink\n#*.sink.csv.period=1\n\n#*.sink.csv.unit=minutes\n\n# Polling
directory for CsvSink\n#*.sink.csv.directory=/tmp/\n\n# Worker instance overlap polling period\n#worker.sink.csv.period=10\n\n#worker.sink.csv.unit=minutes\n\n#
Enable jvm source for instance master, worker, driver and executor\n#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#worker.source.jvm.class=org.apache.spark.metrics.source
 .JvmSource\n\n#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource"
+        },
+        "cluster-env": {
+            "security_enabled": "false", 
+            "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", 
+            "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version
}}/mapreduce/", 
+            "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", 
+            "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/",

+            "ignore_groupsusers_create": "false", 
+            "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar",

+            "kerberos_domain": "EXAMPLE.COM", 
+            "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",

+            "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",

+            "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab", 
+            "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/",

+            "user_group": "hadoop", 
+            "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", 
+            "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz", 
+            "smokeuser": "ambari-qa", 
+            "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",

+            "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz"
+        }
+    }, 
+    "configurationTags": {
+        "spark-defaults": {
+            "tag": "version1425610281212"
+        },
+        "spark-javaopts-properties": {
+            "tag": "version1425610281212"
+        }, 
+        "spark-log4j-properties": {
+            "tag": "version1425610281212"
+        },
+        "spark-env": {
+            "tag": "version1425610281212"
+        },
+        "core-site": {
+            "tag": "version1"
+        },
+        "hadoop-env": {
+            "tag": "version1"
+        },
+        "spark-metrics-properties": {
+            "tag": "version1425610281212"
+        },
+        "cluster-env": {
+            "tag": "version1"
+        }
+    }, 
+    "commandId": "12-0", 
+    "clusterHostInfo": {
+        "all_hosts": [
+            "c6408.ambari.apache.org", 
+            "c6410.ambari.apache.org", 
+            "c6409.ambari.apache.org"
+        ], 
+        "slave_hosts": [
+            "c6408.ambari.apache.org", 
+            "c6410.ambari.apache.org", 
+            "c6409.ambari.apache.org"
+        ],
+        "spark_jobhistoryserver_hosts": [
+            "c6408.ambari.apache.org"
+        ], 
+        "ambari_server_host": [
+            "c6408.ambari.apache.org"
+        ]
+    }
+}
\ No newline at end of file


Mime
View raw message