hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1611198 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ data/conf/ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/
Date Wed, 16 Jul 2014 21:30:41 GMT
Author: thejas
Date: Wed Jul 16 21:30:40 2014
New Revision: 1611198

URL: http://svn.apache.org/r1611198
Log:
HIVE-7342 (add missing files) : support hiveserver2,metastore specific config files (Thejas
Nair, reviewed by Jason, Sushanth, Prasad)

Added:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
    hive/trunk/data/conf/hivemetastore-site.xml
    hive/trunk/data/conf/hiveserver2-site.xml
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java

Added: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java?rev=1611198&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java Wed Jul 16 21:30:40
2014
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.conf;
+
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
+
+/**
+ * Hive Configuration utils
+ */
+@Private
+public class HiveConfUtil {
+  /**
+   * Check if metastore is being used in embedded mode.
+   * This utility function exists so that the logic for determining the mode is same
+   * in HiveConf and HiveMetaStoreClient
+   * @param msUri - metastore server uri
+   * @return
+   */
+  public static boolean isEmbeddedMetaStore(String msUri) {
+    return (msUri == null) ? true : msUri.trim().isEmpty();
+  }
+}

Added: hive/trunk/data/conf/hivemetastore-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hivemetastore-site.xml?rev=1611198&view=auto
==============================================================================
--- hive/trunk/data/conf/hivemetastore-site.xml (added)
+++ hive/trunk/data/conf/hivemetastore-site.xml Wed Jul 16 21:30:40 2014
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.override</name>
+  <value>from.hivemetastore-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.metastoresite</name>
+  <value>from.hivemetastore-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+  <name>hive.conf.restricted.list</name>
+  <value>from.hivemetastore-site.xml</value>
+  <description>Using property defined in HiveConf.ConfVars to test System property
overriding</description>
+</property>
+
+
+</configuration>

Added: hive/trunk/data/conf/hiveserver2-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/data/conf/hiveserver2-site.xml?rev=1611198&view=auto
==============================================================================
--- hive/trunk/data/conf/hiveserver2-site.xml (added)
+++ hive/trunk/data/conf/hiveserver2-site.xml Wed Jul 16 21:30:40 2014
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.override</name>
+  <value>from.hiveserver2-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+  <name>hive.dummyparam.test.server.specific.config.hiveserver2site</name>
+  <value>from.hiveserver2-site.xml</value>
+  <description>Using dummy param to test server specific configuration</description>
+</property>
+
+<property>
+  <name>hive.conf.restricted.list</name>
+  <value>from.hiveserver2-site.xml</value>
+  <description>Using property defined in HiveConf.ConfVars to test System property
overriding</description>
+</property>
+
+
+</configuration>

Added: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java?rev=1611198&view=auto
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java
(added)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java
Wed Jul 16 21:30:40 2014
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.service.server.HiveServer2;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestServerSpecificConfig {
+
+  private static URL oldDefaultHiveSite = HiveConf.getHiveSiteLocation();
+
+  /**
+   * Verify if appropriate server configuration (metastore, hiveserver2) get
+   * loaded when the embedded clients are loaded
+   *
+   * Checks values used in the configs used for testing.
+   *
+   * @throws IOException
+   * @throws Throwable
+   */
+  @Test
+  public void testServerConfigsEmbeddedMetastore() throws IOException, Throwable {
+
+    // set hive-site.xml to default hive-site.xml that has embedded metastore
+    HiveConf.setHiveSiteLocation(oldDefaultHiveSite);
+
+    HiveConf conf = new HiveConf();
+
+    // check config properties expected with embedded metastore client
+    assertTrue(HiveConf.isLoadMetastoreConfig());
+    assertEquals("from.hivemetastore-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.override"));
+
+    assertEquals("from.hivemetastore-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.metastoresite"));
+
+    assertEquals("from.hive-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.hivesite"));
+
+    // verify that hiveserver2 config is not loaded
+    assertFalse(HiveConf.isLoadHiveServer2Config());
+    assertNull(conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site"));
+
+    // check if hiveserver2 config gets loaded when HS2 is started
+    new HiveServer2();
+    conf = new HiveConf();
+    verifyHS2ConfParams(conf);
+
+    assertEquals("from.hivemetastore-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.metastoresite"));
+  }
+
+  private void verifyHS2ConfParams(HiveConf conf) {
+    assertTrue(HiveConf.isLoadHiveServer2Config());
+    assertEquals("from.hiveserver2-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.override"));
+
+    assertEquals("from.hiveserver2-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site"));
+
+    assertEquals("from.hive-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.hivesite"));
+  }
+
+  /**
+   * Ensure that system properties still get precedence. Config params set as
+   * -hiveconf on commandline get set as system properties They should have the
+   * final say
+   */
+  @Test
+  public void testSystemPropertyPrecedence() {
+    // Using property defined in HiveConf.ConfVars to test System property
+    // overriding
+    final String OVERRIDE_KEY = "hive.conf.restricted.list";
+    try {
+      HiveConf.setHiveSiteLocation(oldDefaultHiveSite);
+      System.setProperty(OVERRIDE_KEY, "from.sysprop");
+      HiveConf conf = new HiveConf();
+      // ensure metatore site.xml does not get to override this
+      assertEquals("from.sysprop", conf.get(OVERRIDE_KEY));
+
+      // get HS2 site.xml loaded
+      new HiveServer2();
+      conf = new HiveConf();
+      assertTrue(HiveConf.isLoadHiveServer2Config());
+      // ensure hiveserver2 site.xml does not get to override this
+      assertEquals("from.sysprop", conf.get(OVERRIDE_KEY));
+
+    } finally {
+      System.getProperties().remove(OVERRIDE_KEY);
+    }
+  }
+
+  @Before
+  public void resetDefaults() throws SecurityException, IllegalArgumentException,
+      NoSuchFieldException, IllegalAccessException {
+    // re-set the static variables in HiveConf to default values
+
+    // set load server conf booleans to false
+    HiveConf.setLoadMetastoreConfig(false);
+    HiveConf.setLoadHiveServer2Config(false);
+
+  }
+
+  /**
+   * Test to ensure that HiveConf does not try to load hivemetastore-site.xml,
+   * when remote metastore is used.
+   *
+   * @throws IOException
+   * @throws Throwable
+   */
+  @Test
+  public void testHiveMetastoreRemoteConfig() throws IOException, Throwable {
+    // switch to hive-site.xml with remote metastore
+    setHiveSiteWithRemoteMetastore();
+
+    // Set HiveConf statics to default values
+    resetDefaults();
+
+    // create hiveconf again to run initialization code, to see if value changes
+    HiveConf conf = new HiveConf();
+
+    // check the properties expected in hive client without metastore
+    verifyMetastoreConfNotLoaded(conf);
+    assertEquals("from.hive-site.xml",
+        conf.get("hive.dummyparam.test.server.specific.config.override"));
+
+    // get HS2 site.xml loaded
+    new HiveServer2();
+    conf = new HiveConf();
+    verifyHS2ConfParams(conf);
+    verifyMetastoreConfNotLoaded(conf);
+  }
+
+  private void verifyMetastoreConfNotLoaded(HiveConf conf) {
+    assertFalse(HiveConf.isLoadMetastoreConfig());
+    assertNull(conf.get("hive.dummyparam.test.server.specific.config.metastoresite"));
+  }
+
+  /**
+   * Set new hive-site.xml file location that has remote metastore config
+   *
+   * @throws IOException
+   */
+  private void setHiveSiteWithRemoteMetastore() throws IOException {
+    // new *hive-site.xml file
+    String newConfFile = System.getProperty("test.tmp.dir") + File.separator
+        + this.getClass().getSimpleName() + "hive-site.xml";
+
+    // create a new conf file, using contents from current one
+    // modifying the meastore.uri property
+    File hiveSite = new File(newConfFile);
+    FileOutputStream out = new FileOutputStream(hiveSite);
+    HiveConf.setHiveSiteLocation(oldDefaultHiveSite);
+    HiveConf defaultHiveConf = new HiveConf();
+    defaultHiveConf.setVar(ConfVars.METASTOREURIS, "dummyvalue");
+    // reset to the hive-site.xml values for following param
+    defaultHiveConf.set("hive.dummyparam.test.server.specific.config.override",
+        "from.hive-site.xml");
+    defaultHiveConf.unset("hive.dummyparam.test.server.specific.config.metastoresite");
+    defaultHiveConf.writeXml(out);
+
+    // set the new hive-site.xml
+    HiveConf.setHiveSiteLocation(hiveSite.toURI().toURL());
+  }
+
+}
\ No newline at end of file



Mime
View raw message