hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1593453 - in /hive/branches/branch-0.13/hcatalog/webhcat/svr/src: main/config/ main/java/org/apache/hive/hcatalog/templeton/ main/java/org/apache/hive/hcatalog/templeton/tool/ test/java/org/apache/hive/hcatalog/templeton/tool/
Date Thu, 08 May 2014 23:24:17 GMT
Author: khorgath
Date: Thu May  8 23:24:17 2014
New Revision: 1593453

URL: http://svn.apache.org/r1593453
Log:
HIVE-4576 : templeton.hive.properties does not allow values with commas (Eugene Koifman via
Ashutosh Chauhan)
((from https://issues.apache.org/jira/secure/attachment/12643464/HIVE-4576.0.13.patch)

Modified:
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
    hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Thu
May  8 23:24:17 2014
@@ -136,8 +136,9 @@
 
   <property>
     <name>templeton.hive.properties</name>
-    <value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
-    <description>Properties to set when running hive.</description>
+    <value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
+    <description>Properties to set when running hive.  This is expected to be a comma-separated
prop=value list.
+        If some value is itself a comma-separated list the escape character is '\'</description>
   </property>
 
   <property>

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Thu May  8 23:24:17 2014
@@ -21,6 +21,8 @@ package org.apache.hive.hcatalog.templet
 import java.io.File;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -31,8 +33,10 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hive.hcatalog.templeton.tool.JobState;
+import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
 import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup;
 import org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage;
 
@@ -104,6 +108,11 @@ public class AppConfig extends Configura
    * see webhcat-default.xml
    */
   public static final String HCAT_HOME_PATH      = "templeton.hcat.home";
+  /**
+   * is a comma separated list of name=value pairs;
+   * In case some value is itself a comma-separated list, the comma needs to
+   * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}
+   */
   public static final String HIVE_PROPS_NAME     = "templeton.hive.properties";
   public static final String LIB_JARS_NAME       = "templeton.libjars";
   public static final String PIG_ARCHIVE_NAME    = "templeton.pig.archive";
@@ -247,8 +256,18 @@ public class AppConfig extends Configura
   public String controllerMRChildOpts() { 
     return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS); 
   }
-
-
+  /**
+   * @see  #HIVE_PROPS_NAME
+   */
+  public Collection<String> hiveProps() {
+    String[] props= StringUtils.split(get(HIVE_PROPS_NAME));
+    //since raw data was (possibly) escaped to make split work,
+    //now need to remove escape chars so they don't interfere with downstream processing
+    for(int i = 0; i < props.length; i++) {
+      props[i] = TempletonUtils.unEscape(props[i]);
+    }
+    return Arrays.asList(props);
+  }
 
   public String[] overrideJars() {
     if (getBoolean(OVERRIDE_JARS_ENABLED, true))

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
Thu May  8 23:24:17 2014
@@ -82,7 +82,7 @@ public class HiveDelegator extends Launc
       args.add("--hiveconf");
       args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);
 
-      for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
+      for (String prop : appConf.hiveProps()) {
         args.add("--hiveconf");
         args.add(TempletonUtils.quoteForWindows(prop));
       }

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
Thu May  8 23:24:17 2014
@@ -45,6 +45,7 @@ public interface JobSubmissionConstants 
     "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__";
   /**
    * constants needed for Pig job submission
+   * The string values here are what Pig expects to see in it's environment
    */
   public static interface PigConstants {
     public static final String HIVE_HOME = "HIVE_HOME";

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Thu May  8 23:24:17 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hive.hcatalog.templeton.BadParam;
 import org.apache.hive.hcatalog.templeton.LauncherDelegator;
 
@@ -79,8 +80,8 @@ public class LaunchMapper extends Mapper
     }
     if(conf.get(PigConstants.PIG_OPTS) != null) {
       StringBuilder pigOpts = new StringBuilder();
-      for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) {
-        pigOpts.append("-D").append(prop).append(" ");
+      for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) {
+        pigOpts.append("-D").append(TempletonUtils.unEscape(prop)).append(" ");
       }
       env.put(PigConstants.PIG_OPTS, pigOpts.toString());
     }

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
Thu May  8 23:24:17 2014
@@ -36,6 +36,7 @@ import java.util.regex.Pattern;
 
 import javax.ws.rs.core.UriBuilder;
 
+import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -354,4 +355,8 @@ public class TempletonUtils {
       args.add("call");
     }
   }
+
+  public static String unEscape(String input) {
+    return StringEscapeUtils.unescapeJava(input);
+  }
 }

Modified: hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1593453&r1=1593452&r2=1593453&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
(original)
+++ hive/branches/branch-0.13/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
Thu May  8 23:24:17 2014
@@ -290,5 +290,23 @@ public class TestTempletonUtils {
     } catch (URISyntaxException ex) {
     }
   }
-
+  
+  @Test
+  public void testPropertiesParsing() throws Exception {
+    String[] props = {"hive.metastore.uris=thrift://localhost:9933\\,thrift://127.0.0.1:9933",
+      "hive.metastore.sasl.enabled=false",
+    "hive.some.fake.path=C:\\foo\\bar.txt\\"};
+    StringBuilder input = new StringBuilder();
+    for(String prop : props) {
+      if(input.length() > 0) {
+        input.append(',');
+      }
+      input.append(prop);
+    }
+    String[] newProps = StringUtils.split(input.toString());
+    for(int i = 0; i < newProps.length; i++) {
+      Assert.assertEquals("Pre/post split values don't match",
+        TempletonUtils.unEscape(props[i]), TempletonUtils.unEscape(newProps[i]));
+    }
+  }
 }



Mime
View raw message