kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [kylin] 05/10: KYLIN-3277 Kylin should override hiveconf settings when connecting to hive using jdbc
Date Wed, 04 Apr 2018 04:12:16 GMT
This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit cfe84a18f3f54d405ceb394ed3b40bd4b09a692b
Author: ycq <ycq@meitu.com>
AuthorDate: Wed Mar 7 17:10:38 2018 +0800

    KYLIN-3277 Kylin should override hiveconf settings when connecting to hive using jdbc
    
    Signed-off-by: nichunen <chunen.ni@kyligence.io>
---
 .../apache/kylin/common/util/HiveCmdBuilder.java   |  47 +---------
 .../kylin/common/util/HiveConfigurationUtil.java   | 101 +++++++++++++++++++++
 .../kylin/source/hive/BeelineHiveClient.java       |  16 +++-
 3 files changed, 115 insertions(+), 49 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index fcfc598..9a9c65b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -46,12 +46,12 @@ public class HiveCmdBuilder {
     }
 
     private KylinConfig kylinConfig;
-    final private Map<String, String> hiveConfProps = new HashMap<>();
+    final private Map<String, String> hiveConfProps;
     final private ArrayList<String> statements = Lists.newArrayList();
 
     public HiveCmdBuilder() {
         kylinConfig = KylinConfig.getInstanceFromEnv();
-        loadHiveConfiguration();
+        hiveConfProps = HiveConfigurationUtil.loadHiveConfiguration();
     }
 
     public String build() {
@@ -153,47 +153,4 @@ public class HiveCmdBuilder {
         return build();
     }
 
-    private void loadHiveConfiguration() {
-
-        File hiveConfFile;
-        String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
-        String path = System.getProperty(KylinConfig.KYLIN_CONF);
-
-        if (StringUtils.isNotEmpty(path)) {
-            hiveConfFile = new File(path, hiveConfFileName);
-        } else {
-            path = KylinConfig.getKylinHome();
-            if (StringUtils.isEmpty(path)) {
-                logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME);
-                return;
-            }
-            hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
-        }
-
-        if (!hiveConfFile.exists()) {
-            throw new RuntimeException("Missing config file: " + hiveConfFile.getAbsolutePath());
-        }
-
-        String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
-
-        try {
-            File file = new File(fileUrl);
-            if (file.exists()) {
-                DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-                DocumentBuilder builder = factory.newDocumentBuilder();
-                Document doc = builder.parse(file);
-                NodeList nl = doc.getElementsByTagName("property");
-                hiveConfProps.clear();
-                for (int i = 0; i < nl.getLength(); i++) {
-                    String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
-                    String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
-                    if (!key.equals("tmpjars")) {
-                        hiveConfProps.put(key, value);
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new RuntimeException("Failed to parse hive conf file ", e);
-        }
-    }
 }
\ No newline at end of file
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java
b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java
new file mode 100644
index 0000000..a8cba61
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.kylin.common.util.HiveCmdBuilder.HIVE_CONF_FILENAME;
+
+/**
+ * @author ycq
+ * @since 2018-03-05
+ */
+public class HiveConfigurationUtil {
+
+    private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HiveConfigurationUtil.class);
+    private static final String HIVE_CONF_PREFIX = "hiveconf:";
+
+    public static Properties loadHiveJDBCProperties() {
+        Map<String, String> hiveConfiguration = loadHiveConfiguration();
+        Properties ret = new Properties();
+        for (Map.Entry<String, String> entry : hiveConfiguration.entrySet()) {
+            ret.put(HIVE_CONF_PREFIX + entry.getKey(), entry.getValue());
+        }
+        return ret;
+    }
+
+    public static Map<String, String> loadHiveConfiguration() {
+        Map<String, String> hiveConfProps = new HashMap<>();
+        File hiveConfFile;
+        String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+        String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+        if (StringUtils.isNotEmpty(path)) {
+            hiveConfFile = new File(path, hiveConfFileName);
+        } else {
+            path = KylinConfig.getKylinHome();
+            if (StringUtils.isEmpty(path)) {
+                logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME);
+                return hiveConfProps;
+            }
+            hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
+        }
+
+        if (!hiveConfFile.exists()) {
+            throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml");
+        }
+
+        String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
+
+        try {
+            File file = new File(fileUrl);
+            if (file.exists()) {
+                DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+                DocumentBuilder builder = factory.newDocumentBuilder();
+                Document doc = builder.parse(file);
+                NodeList nl = doc.getElementsByTagName("property");
+                hiveConfProps.clear();
+                for (int i = 0; i < nl.getLength(); i++) {
+                    String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+                    String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+                    if (!key.equals("tmpjars")) {
+                        hiveConfProps.put(key, value);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to parse hive conf file ", e);
+        }
+        return hiveConfProps;
+    }
+
+}
+
+
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 747b1bb..593ad96 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -26,25 +26,30 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.List;
+import java.util.Properties;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.util.DBUtils;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.HiveConfigurationUtil;
 
 public class BeelineHiveClient implements IHiveClient {
 
+    private static final String HIVE_AUTH_USER = "user";
+    private static final String HIVE_AUTH_PASSWD = "password";
     private Connection cnct;
     private Statement stmt;
     private DatabaseMetaData metaData;
 
+
     public BeelineHiveClient(String beelineParams) {
         if (StringUtils.isEmpty(beelineParams)) {
             throw new IllegalArgumentException("BeelineParames cannot be empty");
         }
         String[] splits = StringUtils.split(beelineParams);
-        String url = null, username = null, password = null;
+        String url = "", username = "", password = "";
         for (int i = 0; i < splits.length; i++) {
             if ("-u".equals(splits[i])) {
                 url = stripQuotes(splits[i + 1]);
@@ -56,13 +61,16 @@ public class BeelineHiveClient implements IHiveClient {
                 password = stripQuotes(splits[i + 1]);
             }
         }
-        this.init(url, username, password);
+        Properties jdbcProperties = HiveConfigurationUtil.loadHiveJDBCProperties();
+        jdbcProperties.put(HIVE_AUTH_PASSWD, password);
+        jdbcProperties.put(HIVE_AUTH_USER, username);
+        this.init(url, jdbcProperties);
     }
 
-    private void init(String url, String username, String password) {
+    private void init(String url, Properties hiveProperties) {
         try {
             Class.forName("org.apache.hive.jdbc.HiveDriver");
-            cnct = DriverManager.getConnection(url, username, password);
+            cnct = DriverManager.getConnection(url, hiveProperties);
             stmt = cnct.createStatement();
             metaData = cnct.getMetaData();
         } catch (SQLException | ClassNotFoundException e) {

-- 
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.

Mime
View raw message