hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1611190 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ data/conf/ metastore/src/java/org/apache/hadoop/hive/metastore/ service/src/java/org/apache/hive/service/cli/thrift/ service/src/java/org/apache/hive/service/server/
Date Wed, 16 Jul 2014 21:05:44 GMT
Author: thejas
Date: Wed Jul 16 21:05:43 2014
New Revision: 1611190

URL: http://svn.apache.org/r1611190
Log:
HIVE-7342 : support hiveserver2,metastore specific config files (Thejas Nair, reviewed by
Jason, Sushanth, Prasad)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/data/conf/hive-site.xml
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
    hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Jul 16 21:05:43
2014
@@ -57,16 +57,23 @@ public class HiveConf extends Configurat
   protected Properties origProp;
   protected String auxJars;
   private static final Log l4j = LogFactory.getLog(HiveConf.class);
+  private static boolean loadMetastoreConfig = false;
+  private static boolean loadHiveServer2Config = false;
   private static URL hiveDefaultURL = null;
   private static URL hiveSiteURL = null;
+  private static URL hivemetastoreSiteUrl = null;
+  private static URL hiveServer2SiteUrl = null;
+
   private static byte[] confVarByteArray = null;
 
+
   private static final Map<String, ConfVars> vars = new HashMap<String, ConfVars>();
   private final List<String> restrictList = new ArrayList<String>();
 
   private boolean isWhiteListRestrictionEnabled = false;
   private final List<String> modWhiteList = new ArrayList<String>();
 
+
   static {
     ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
     if (classLoader == null) {
@@ -77,6 +84,9 @@ public class HiveConf extends Configurat
 
     // Look for hive-site.xml on the CLASSPATH and log its location if found.
     hiveSiteURL = classLoader.getResource("hive-site.xml");
+    hivemetastoreSiteUrl = classLoader.getResource("hivemetastore-site.xml");
+    hiveServer2SiteUrl = classLoader.getResource("hiveserver2-site.xml");
+
     for (ConfVars confVar : ConfVars.values()) {
       vars.put(confVar.varname, confVar);
     }
@@ -2052,6 +2062,26 @@ public class HiveConf extends Configurat
       addResource(hiveSiteURL);
     }
 
+    // if embedded metastore is to be used as per config so far
+    // then this is considered like the metastore server case
+    String msUri = this.getVar(HiveConf.ConfVars.METASTOREURIS);
+    if(HiveConfUtil.isEmbeddedMetaStore(msUri)){
+      setLoadMetastoreConfig(true);
+    }
+
+    // load hivemetastore-site.xml if this is metastore and file exists
+    if (isLoadMetastoreConfig() && hivemetastoreSiteUrl != null) {
+      addResource(hivemetastoreSiteUrl);
+    }
+
+    // load hiveserver2-site.xml if this is hiveserver2 and file exists
+    // metastore can be embedded within hiveserver2, in such cases
+    // the conf params in hiveserver2-site.xml will override whats defined
+    // in hivemetastore-site.xml
+    if (isLoadHiveServer2Config() && hiveServer2SiteUrl != null) {
+      addResource(hiveServer2SiteUrl);
+    }
+
     // Overlay the values of any system properties whose names appear in the list of ConfVars
     applySystemProperties();
 
@@ -2111,7 +2141,6 @@ public class HiveConf extends Configurat
     setupRestrictList();
   }
 
-
   /**
    * Apply system properties to this object if the property name is defined in ConfVars
    * and the value is non-null and not an empty string.
@@ -2200,6 +2229,15 @@ public class HiveConf extends Configurat
     return hiveSiteURL;
   }
 
+  public static URL getMetastoreSiteLocation() {
+    return hivemetastoreSiteUrl;
+  }
+
+  public static URL getHiveServer2SiteLocation() {
+    return hiveServer2SiteUrl;
+  }
+
+
   /**
    * @return the user name set in hadoop.job.ugi param or the current user from System
    * @throws IOException
@@ -2283,4 +2321,20 @@ public class HiveConf extends Configurat
     restrictList.add(ConfVars.HIVE_IN_TEST.varname);
     restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
   }
+
+  public static boolean isLoadMetastoreConfig() {
+    return loadMetastoreConfig;
+  }
+
+  public static void setLoadMetastoreConfig(boolean loadMetastoreConfig) {
+    HiveConf.loadMetastoreConfig = loadMetastoreConfig;
+  }
+
+  public static boolean isLoadHiveServer2Config() {
+    return loadHiveServer2Config;
+  }
+
+  public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) {
+    HiveConf.loadHiveServer2Config = loadHiveServer2Config;
+  }
 }

Modified: hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hive-site.xml?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/data/conf/hive-site.xml (original)
+++ hive/trunk/data/conf/hive-site.xml Wed Jul 16 21:05:43 2014
@@ -221,4 +221,19 @@
   <value>false</value>
 </property>
 
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.override</name>
+  <value>from.hive-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.hivesite</name>
+  <value>from.hive-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+
+
 </configuration>

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Wed
Jul 16 21:05:43 2014
@@ -5066,6 +5066,7 @@ public class HiveMetaStore extends Thrif
    * @param args
    */
   public static void main(String[] args) throws Throwable {
+    HiveConf.setLoadMetastoreConfig(true);
     HiveMetastoreCli cli = new HiveMetastoreCli();
     cli.parse(args);
     final boolean isCliVerbose = cli.isVerbose();

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Wed Jul 16 21:05:43 2014
@@ -34,12 +34,10 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.Set;
 
 import javax.security.auth.login.LoginException;
 
@@ -47,9 +45,9 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnListImpl;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
 import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
@@ -68,7 +66,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Function;
 import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
 import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest;
 import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse;
 import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest;
@@ -112,7 +109,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.TableStatsRequest;
 import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
 import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
-import org.apache.hadoop.hive.metastore.api.TxnInfo;
 import org.apache.hadoop.hive.metastore.api.TxnOpenException;
 import org.apache.hadoop.hive.metastore.api.Type;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
@@ -167,7 +163,7 @@ public class HiveMetaStoreClient impleme
     this.conf = conf;
 
     String msUri = conf.getVar(HiveConf.ConfVars.METASTOREURIS);
-    localMetaStore = (msUri == null) ? true : msUri.trim().isEmpty();
+    localMetaStore = HiveConfUtil.isEmbeddedMetaStore(msUri);
     if (localMetaStore) {
       // instantiate the metastore server handler directly instead of connecting
       // through the network

Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
(original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java
Wed Jul 16 21:05:43 2014
@@ -32,6 +32,7 @@ public class EmbeddedThriftBinaryCLIServ
   public EmbeddedThriftBinaryCLIService() {
     super(new CLIService());
     isEmbedded = true;
+    HiveConf.setLoadHiveServer2Config(true);
     cliService.init(new HiveConf());
     cliService.start();
   }

Modified: hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java?rev=1611190&r1=1611189&r2=1611190&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/server/HiveServer2.java Wed Jul 16
21:05:43 2014
@@ -25,7 +25,6 @@ import org.apache.hadoop.hive.common.Log
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
 import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.hive.service.CompositeService;
 import org.apache.hive.service.cli.CLIService;
@@ -45,6 +44,7 @@ public class HiveServer2 extends Composi
 
   public HiveServer2() {
     super("HiveServer2");
+    HiveConf.setLoadHiveServer2Config(true);
   }
 
 
@@ -142,7 +142,7 @@ public class HiveServer2 extends Composi
       // before any of the other core hive classes are loaded
       String initLog4jMessage = LogUtils.initHiveLog4j();
       LOG.debug(initLog4jMessage);
-      
+
       HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG);
       //log debug message from "oproc" after log4j initialize properly
       LOG.debug(oproc.getDebugMessage().toString());



Mime
View raw message