ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-16209. Write out new log property files under /etc/hive2/conf (smohanty)
Date Tue, 03 May 2016 14:31:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 51c0cb4ff -> 46d1d7b18


AMBARI-16209. Write out new log property files under /etc/hive2/conf (smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46d1d7b1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46d1d7b1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46d1d7b1

Branch: refs/heads/trunk
Commit: 46d1d7b188d3f8c020f992c91406d994bac5eebc
Parents: 51c0cb4
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Tue May 3 07:30:40 2016 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Tue May 3 07:30:40 2016 -0700

----------------------------------------------------------------------
 .../package/scripts/hive_interactive.py         |  29 +++--
 .../0.12.0.2.0/package/scripts/params_linux.py  |   5 +
 .../HIVE/configuration/llap-cli-log4j2.xml      | 108 +++++++++++++++++++
 .../HIVE/configuration/llap-daemon-log4j.xml    |  21 +++-
 .../stacks/2.5/HIVE/test_hive_server_int.py     |  12 +++
 .../python/stacks/2.5/configs/hsi_default.json  |   6 ++
 6 files changed, 167 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index 24047ff..e8532da 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -126,34 +126,43 @@ def hive_interactive(name=None):
   File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
        owner=params.hive_user,
        group=params.user_group,
-       content=InlineTemplate(params.hive_interactive_env_sh_template)
-       )
+       content=InlineTemplate(params.hive_interactive_env_sh_template))
+
+  llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
+  File(format("{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
+       mode=0644,
+       group=params.user_group,
+       owner=params.hive_user,
+       content=params.llap_daemon_log4j)
+
+  llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
+  File(format("{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
+       mode=0644,
+       group=params.user_group,
+       owner=params.hive_user,
+       content=params.llap_cli_log4j2)
 
   # On some OS this folder could be not exists, so we will create it before pushing there
files
   Directory(params.limits_conf_dir,
             create_parents = True,
             owner='root',
-            group='root'
-  )
+            group='root')
 
   File(os.path.join(params.limits_conf_dir, 'hive.conf'),
        owner='root',
        group='root',
        mode=0644,
-       content=Template("hive.conf.j2")
-  )
+       content=Template("hive.conf.j2"))
 
   if not os.path.exists(params.target_hive_interactive):
     jdbc_connector(params.target_hive_interactive)
 
   File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
        content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
-       mode = 0644,
-       )
+       mode = 0644)
   File(params.start_hiveserver2_interactive_path,
        mode=0755,
-       content=Template(format('{start_hiveserver2_interactive_script}'))
-       )
+       content=Template(format('{start_hiveserver2_interactive_script}')))
 
   Directory(params.hive_pid_dir,
             create_parents=True,

http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 2e1a2af..cef2cbc 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -538,12 +538,16 @@ HdfsResource = functools.partial(
 hive_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
 has_hive_interactive = len(hive_interactive_hosts) > 0
 if has_hive_interactive:
+  llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content']
+  llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content']
+
   hive_server_interactive_conf_dir = status_params.hive_server_interactive_conf_dir
   execute_path_hive_interactive = os.path.join(os.environ['PATH'], hive_interactive_bin,
hadoop_bin_dir)
   start_hiveserver2_interactive_script = 'startHiveserver2Interactive.sh.j2'
   start_hiveserver2_interactive_path = format("{tmp_dir}/start_hiveserver2_interactive_script")
   hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
   hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive',
False)
+
   # Service check related
   if hive_transport_mode.lower() == "http":
     hive_server_interactive_port = config['configurations']['hive-interactive-site']['hive.server2.thrift.http.port']
@@ -563,6 +567,7 @@ if has_hive_interactive:
   if security_enabled:
     hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file']
     hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal']
+  pass
 
 # ranger host
 stack_supports_ranger_audit_db = stack_version_formatted_major and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT,
stack_version_formatted_major)

http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml
new file mode 100644
index 0000000..5c8f9a8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+  <name>content</name>
+  <description>Custom log4j.properties</description>
+  <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = WARN
+name = LlapCliLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = console
+property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
+property.hive.log.file = llap-cli.log
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+# appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
+appender.console.layout.pattern = %p %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename
if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = ZooKeeper, DataNucleus, Datastore, JPOX, HadoopConf
+
+logger.ZooKeeper.name = org.apache.zookeeper
+logger.ZooKeeper.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.HadoopConf.name = org.apache.hadoop.conf.Configuration
+logger.HadoopConf.level = ERROR
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
+  </value>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
index 1c60285..793ea7a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
@@ -40,12 +40,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-status = WARN
+
+status = INFO
 name = LlapDaemonLog4j2
 packages = org.apache.hadoop.hive.ql.log
 
 # list of properties
-property.llap.daemon.log.level = WARN
+property.llap.daemon.log.level = INFO
 property.llap.daemon.root.logger = console
 property.llap.daemon.log.dir = .
 property.llap.daemon.log.file = llapdaemon.log
@@ -90,7 +91,19 @@ appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy
 appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex}
 
 # list of all loggers
-loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger,
LlapIoImpl, LlapIoOrc, LlapIoCache, LlapIoLocking
+
+logger.LlapIoImpl.name = LlapIoImpl
+logger.LlapIoImpl.level = INFO
+
+logger.LlapIoOrc.name = LlapIoOrc
+logger.LlapIoOrc.level = WARN
+
+logger.LlapIoCache.name = LlapIoCache
+logger.LlapIOCache.level = WARN
+
+logger.LlapIoLocking.name = LlapIoLocking
+logger.LlapIoLocking.level = WARN
 
 logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
 logger.NIOServerCnxn.level = WARN
@@ -108,7 +121,7 @@ logger.JPOX.name = JPOX
 logger.JPOX.level = ERROR
 
 logger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger
-logger.HistoryLogger.level = WARN
+logger.HistoryLogger.level = INFO
 logger.HistoryLogger.additivity = false
 logger.HistoryLogger.appenderRefs = HistoryAppender
 logger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER

http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index 5712995..6a0d777 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -260,6 +260,18 @@ class TestHiveServerInteractive(RMFTestCase):
                               owner='hive',
                               group='hadoop',
     )
+    self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/conf.server/llap-daemon-log4j2.properties',
+                              content='con\ntent',
+                              owner='hive',
+                              group='hadoop',
+                              mode=0644,
+    )
+    self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/conf.server/llap-cli-log4j2.properties',
+                              content='con\ntent',
+                              owner='hive',
+                              group='hadoop',
+                              mode=0644,
+    )
     self.assertResourceCalled('Directory', '/etc/security/limits.d',
                               owner='root',
                               group='root',

http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
index e7bb056..9bff890 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
@@ -48,6 +48,12 @@
     "taskId": 152,
     "public_hostname": "c6401.ambari.apache.org",
     "configurations": {
+        "llap-daemon-log4j": {
+          "content" : "con\ntent"
+        },
+        "llap-cli-log4j2": {
+          "content" : "con\ntent"
+        },
         "sqoop-site": {
             "atlas.cluster.name": "c1",
             "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"


Mime
View raw message