ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-18307. Add sqlline capability to AMS collector to help debugging. (swagle)
Date Tue, 18 Oct 2016 22:57:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 ac32c2885 -> db4aef04e


AMBARI-18307. Add sqlline capability to AMS collector to help debugging. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db4aef04
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db4aef04
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db4aef04

Branch: refs/heads/branch-2.5
Commit: db4aef04e99697d4681e592294889c925bbd64ce
Parents: ac32c28
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Tue Oct 18 15:56:11 2016 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Tue Oct 18 15:56:30 2016 -0700

----------------------------------------------------------------------
 ambari-metrics/ambari-metrics-assembly/pom.xml  |  13 ++
 .../conf/unix/sqlline/log4j.properties          |  76 ++++++++
 .../conf/unix/sqlline/phoenix_utils.py          | 193 +++++++++++++++++++
 .../conf/unix/sqlline/sqlline.py                | 105 ++++++++++
 .../ambari-metrics-timelineservice/pom.xml      |  13 ++
 5 files changed, 400 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index f6cb361..a55ae57 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -309,6 +309,19 @@
                       </sources>
                     </mapping>
                     <mapping>
+                      <directory>/usr/lib/ambari-metrics-collector/bin</directory>
+                      <filemode>755</filemode>
+                      <username>root</username>
+                      <groupname>root</groupname>
+                      <directoryIncluded>false</directoryIncluded>
+                      <sources>
+                        <source>
+                          <location>${collector.dir}/conf/unix/sqlline</location>
+                          <filter>false</filter>
+                        </source>
+                      </sources>
+                    </mapping>
+                    <mapping>
                       <directory>/etc/ambari-metrics-collector/conf</directory>
                       <configuration>true</configuration>
                       <sources>

http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
new file mode 100644
index 0000000..3f7760a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
@@ -0,0 +1,76 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+# Define some default values that can be overridden by system properties
+psql.root.logger=WARN,console
+psql.log.dir=.
+psql.log.file=psql.log
+hadoop.log.dir=.
+
+# Define the root logger to the system property "plsql.root.logger".
+log4j.rootLogger=${psql.root.logger}
+
+# Logging Threshold to INFO for query-server. root logger still at WARN for sqlline clients.
+log4j.threshold=INFO
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${psql.log.dir}/${psql.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=ERROR
+log4j.logger.org.apache.hadoop.hbase.HBaseConfiguration=ERROR
+
+# query server packages
+log4j.logger.org.apache.calcite.avatica=INFO
+log4j.logger.org.apache.phoenix.queryserver.server=INFO
+log4j.logger.org.eclipse.jetty.server=INFO

http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
new file mode 100644
index 0000000..6d9b791
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+import os
+import fnmatch
+import subprocess
+
+def find(pattern, classPaths):
+    paths = classPaths.split(os.pathsep)
+
+    # for each class path
+    for path in paths:
+        # remove * if it's at the end of path
+        if ((path is not None) and (len(path) > 0) and (path[-1] == '*')) :
+            path = path[:-1]
+    
+        for root, dirs, files in os.walk(path):
+            # sort the file names so *-client always precedes *-thin-client
+            files.sort()
+            for name in files:
+                if fnmatch.fnmatch(name, pattern):
+                    return os.path.join(root, name)
+                
+    return ""
+
+def findFileInPathWithoutRecursion(pattern, path):
+    if not os.path.exists(path):
+        return ""
+    files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
+    # sort the file names so *-client always precedes *-thin-client
+    files.sort()
+    for name in files:
+        if fnmatch.fnmatch(name, pattern):
+            return os.path.join(path, name)
+
+    return ""
+
+def which(file):
+    for path in os.environ["PATH"].split(os.pathsep):
+        if os.path.exists(os.path.join(path, file)):
+            return os.path.join(path, file)
+    return None
+
+def findClasspath(file):
+    aPath = which(file)
+    command = "%s%s" %(aPath, ' classpath')
+    return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read()
+
+def setPath():
+    PHOENIX_CLIENT_JAR_PATTERN = "phoenix-*-client.jar"
+    PHOENIX_THIN_CLIENT_JAR_PATTERN = "phoenix-*-thin-client.jar"
+    PHOENIX_QUERYSERVER_JAR_PATTERN = "phoenix-server-*-runnable.jar"
+    PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar"
+
+    # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH
+    global phoenix_class_path
+    phoenix_class_path = os.getenv('PHOENIX_LIB_DIR','')
+    if phoenix_class_path == "":
+        phoenix_class_path = os.getenv('PHOENIX_CLASS_PATH','')
+
+    global hbase_conf_dir
+    # if HBASE_CONF_DIR set explicitly, use that
+    hbase_conf_dir = os.getenv('HBASE_CONF_DIR', os.getenv('HBASE_CONF_PATH'))
+    if not hbase_conf_dir:
+        # else fall back to HBASE_HOME
+        if os.getenv('HBASE_HOME'):
+            hbase_conf_dir = os.path.join(os.getenv('HBASE_HOME'), "conf")
+        elif os.name == 'posix':
+            # default to the bigtop configuration dir
+            hbase_conf_dir = '/etc/hbase/conf'
+        else:
+            # Try to provide something valid
+            hbase_conf_dir = '.'
+    global hbase_conf_path # keep conf_path around for backward compatibility
+    hbase_conf_path = hbase_conf_dir
+
+    global current_dir
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+
+    global phoenix_jar_path
+    phoenix_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+    global phoenix_client_jar
+    phoenix_client_jar = find("phoenix-*-client.jar", phoenix_jar_path)
+    if phoenix_client_jar == "":
+        phoenix_client_jar = findFileInPathWithoutRecursion(PHOENIX_CLIENT_JAR_PATTERN, os.path.join(current_dir,
".."))
+    if phoenix_client_jar == "":
+        phoenix_client_jar = find(PHOENIX_CLIENT_JAR_PATTERN, phoenix_class_path)
+
+    global phoenix_test_jar_path
+    phoenix_test_jar_path = os.path.join(current_dir, "..", "phoenix-core", "target","*")
+
+    global hadoop_conf
+    hadoop_conf = os.getenv('HADOOP_CONF_DIR', None)
+    if not hadoop_conf:
+        if os.name == 'posix':
+            # Try to provide a sane configuration directory for Hadoop if not otherwise provided.
+            # If there's no jaas file specified by the caller, this is necessary when Kerberos
is enabled.
+            hadoop_conf = '/etc/hadoop/conf'
+        else:
+            # Try to provide something valid..
+            hadoop_conf = '.'
+
+    global hadoop_classpath
+    if (os.name != 'nt'):
+        hadoop_classpath = findClasspath('hadoop')
+    else:
+        hadoop_classpath = os.getenv('HADOOP_CLASSPATH', '')
+
+    global hadoop_common_jar_path
+    hadoop_common_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+    global hadoop_common_jar
+    hadoop_common_jar = find("hadoop-common*.jar", hadoop_common_jar_path)
+
+    global hadoop_hdfs_jar_path
+    hadoop_hdfs_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+    global hadoop_hdfs_jar
+    hadoop_hdfs_jar = find("hadoop-hdfs*.jar", hadoop_hdfs_jar_path)
+
+    global testjar
+    testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_test_jar_path)
+    if testjar == "":
+        testjar = findFileInPathWithoutRecursion(PHOENIX_TESTS_JAR_PATTERN, os.path.join(current_dir,
".."))
+    if testjar == "":
+        testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_class_path)
+
+    global phoenix_queryserver_jar
+    phoenix_queryserver_jar = find(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir,
"..", "phoenix-server", "target", "*"))
+    if phoenix_queryserver_jar == "":
+        phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN,
os.path.join(current_dir, "..", "lib"))
+    if phoenix_queryserver_jar == "":
+        phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN,
os.path.join(current_dir, ".."))
+
+
+    global phoenix_thin_client_jar
+    phoenix_thin_client_jar = find(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir,
"..", "phoenix-server-client", "target", "*"))
+    if phoenix_thin_client_jar == "":
+        phoenix_thin_client_jar = findFileInPathWithoutRecursion(PHOENIX_THIN_CLIENT_JAR_PATTERN,
os.path.join(current_dir, ".."))
+
+    return ""
+
+def shell_quote(args):
+    """
+    Return the platform specific shell quoted string. Handles Windows and *nix platforms.
+
+    :param args: array of shell arguments
+    :return: shell quoted string
+    """
+    if os.name == 'nt':
+        import subprocess
+        return subprocess.list2cmdline(args)
+    else:
+        # pipes module isn't available on Windows
+        import pipes
+        return " ".join([pipes.quote(v) for v in args])
+
+if __name__ == "__main__":
+    setPath()
+    print "phoenix_class_path:", phoenix_class_path
+    print "hbase_conf_dir:", hbase_conf_dir
+    print "hbase_conf_path:", hbase_conf_path
+    print "current_dir:", current_dir
+    print "phoenix_jar_path:", phoenix_jar_path
+    print "phoenix_client_jar:", phoenix_client_jar
+    print "phoenix_test_jar_path:", phoenix_test_jar_path
+    print "hadoop_common_jar_path:", hadoop_common_jar_path
+    print "hadoop_common_jar:", hadoop_common_jar
+    print "hadoop_hdfs_jar_path:", hadoop_hdfs_jar_path
+    print "hadoop_hdfs_jar:", hadoop_hdfs_jar
+    print "testjar:", testjar
+    print "phoenix_queryserver_jar:", phoenix_queryserver_jar
+    print "phoenix_thin_client_jar:", phoenix_thin_client_jar
+    print "hadoop_classpath:", hadoop_classpath

http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
new file mode 100644
index 0000000..852f264
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+import os
+import subprocess
+import sys
+import phoenix_utils
+import atexit
+import fnmatch
+
+# Set JAVA_HOME or JDK_HOME before running sqlline.py
+# Example: ./sqlline.py localhost:61181:/ams-hbase-unsecure
+
+global childProc
+childProc = None
+def kill_child():
+    if childProc is not None:
+        childProc.terminate()
+        childProc.kill()
+        if os.name != 'nt':
+            os.system("reset")
+atexit.register(kill_child)
+
+def find_java():
+  try:
+    java = os.environ.get('JAVA_HOME')
+    if not java:
+      java = os.environ.get('JDK_HOME')
+  except:
+    java = None
+  if java and os.path.exists(java):
+    if java.endswith('bin') or java.endswith('bin/'):
+      return os.path.join(java, 'java')
+    if java.find('/bin/java') == -1:
+      return os.path.join(java, 'bin', 'java')
+  else:
+    return 'java'
+
+#phoenix_utils.setPath()
+
+if len(sys.argv) < 2:
+    print "Zookeeper not specified. \nUsage: sqlline.py <zookeeper> \
+<optional_sql_file> \nExample: \n 1. sqlline.py localhost:2181:/hbase \n 2. sqlline.py
\
+localhost:2181:/hbase ../examples/stock_symbol.sql"
+    sys.exit()
+
+sqlfile = ""
+
+if len(sys.argv) > 2:
+    sqlfile = "--run=" + phoenix_utils.shell_quote([sys.argv[2]])
+
+colorSetting = "true"
+# disable color setting for windows OS
+if os.name == 'nt':
+    colorSetting = "false"
+
+# Set custom path vars
+hbase_conf_dir = os.environ.get('HBASE_CONF_DIR', '/etc/ams-hbase/conf')
+hadoop_conf = os.environ.get('HADOOP_CONF_DIR', '/etc/ams-hbase/conf')
+hadoop_classpath = os.environ.get('HADOOP_CLASSPATH', '/usr/lib/ams-hbase/lib')
+
+PATH_PREFIX = '/usr/lib/ambari-metrics-collector/'
+if not os.path.exists(PATH_PREFIX):
+  print 'Fatal Error: ' + PATH_PREFIX + ' not a valid path.'
+  sys.exit(1)
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+phoenix_client_path = PATH_PREFIX + '*'
+
+java_cmd = find_java() + ' -cp "' + phoenix_client_path + \
+    os.pathsep + hadoop_conf + '" -Dlog4j.configuration=file:' + \
+    os.path.join(current_dir, "log4j.properties") + \
+    " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \
+-u jdbc:phoenix:" + phoenix_utils.shell_quote([sys.argv[1]]) + \
+    " -n none -p none --color=" + colorSetting + " --fastConnect=false --verbose=true \
+--isolation=TRANSACTION_READ_COMMITTED " + sqlfile
+
+print 'java command: %s' % str(java_cmd)
+
+childProc = subprocess.Popen(java_cmd, shell=True)
+#Wait for child process exit
+(output, error) = childProc.communicate()
+returncode = childProc.returncode
+childProc = None
+# Propagate Java return code to this script
+if returncode is not None:
+    sys.exit(returncode)

http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 1144573..fdbfdcb 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -182,6 +182,19 @@
               </sources>
             </mapping>
             <mapping>
+              <directory>/usr/lib/ambari-metrics-collector/bin</directory>
+              <filemode>755</filemode>
+              <username>root</username>
+              <groupname>root</groupname>
+              <directoryIncluded>false</directoryIncluded>
+              <sources>
+                <source>
+                  <location>conf/unix/sqlline</location>
+                  <filter>false</filter>
+                </source>
+              </sources>
+            </mapping>
+            <mapping>
               <directory>/etc/ambari-metrics-collector/conf</directory>
               <configuration>true</configuration>
               <sources>


Mime
View raw message