hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1526035 - in /hive/branches/vectorization: ./ hcatalog/src/docs/src/documentation/content/xdocs/ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/t...
Date Tue, 24 Sep 2013 22:38:35 GMT
Author: hashutosh
Date: Tue Sep 24 22:38:33 2013
New Revision: 1526035

URL: http://svn.apache.org/r1526035
Log:
Merged in with latest trunk

Added:
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HiveJobIDParser.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/HiveJobIDParser.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JarJobIDParser.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JarJobIDParser.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobIDParser.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobIDParser.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/PigJobIDParser.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/PigJobIDParser.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/hive/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/hive/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/hive/stderr
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/hive/stderr
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/jar/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/jar/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/jar/stderr
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/jar/stderr
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/pig/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/pig/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/pig/stderr
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/pig/stderr
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/streaming/
      - copied from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/streaming/
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/data/status/streaming/stderr
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/data/status/streaming/stderr
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java
      - copied unchanged from r1526034, hive/trunk/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java
      - copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe4.java
      - copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe4.java
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe5.java
      - copied unchanged from r1526034, hive/trunk/ql/src/test/org/apache/hadoop/hive/serde2/CustomSerDe5.java
    hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_wise_fileformat18.q
      - copied unchanged from r1526034, hive/trunk/ql/src/test/queries/clientpositive/partition_wise_fileformat18.q
    hive/branches/vectorization/ql/src/test/results/clientpositive/partition_wise_fileformat18.q.out
      - copied unchanged from r1526034, hive/trunk/ql/src/test/results/clientpositive/partition_wise_fileformat18.q.out
    hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
      - copied unchanged from r1526034, hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
Modified:
    hive/branches/vectorization/   (props changed)
    hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml
    hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml
    hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml
    hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
    hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
    hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
    hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java

Propchange: hive/branches/vectorization/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1525675-1526034

Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/hive.xml Tue Sep 24 22:38:33 2013
@@ -66,6 +66,13 @@
         <td>None</td>
     </tr>
 
+    <tr><td><strong>enablelog</strong></td>
+        <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+         statusdir must be set as well to use this feature.</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
     <tr><td><strong>callback</strong></td>
         <td>Define a URL to be called upon job completion. You may embed a specific
          job ID into this URL using <code>$jobId</code>.  This tag

Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducejar.xml Tue Sep 24 22:38:33 2013
@@ -86,6 +86,13 @@
         <td>None</td>
     </tr>
 
+    <tr><td><strong>enablelog</strong></td>
+        <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+         statusdir must be set as well to use this feature.</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
     <tr><td><strong>callback</strong></td>
         <td>Define a URL to be called upon job completion. You may embed a specific
          job ID into this URL using <code>$jobId</code>.  This tag

Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/mapreducestreaming.xml Tue Sep 24 22:38:33 2013
@@ -101,6 +101,13 @@
         <td>None</td>
     </tr>
 
+    <tr><td><strong>enablelog</strong></td>
+        <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+         statusdir must be set as well to use this feature.</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
     <tr><td><strong>callback</strong></td>
         <td>Define a URL to be called upon job completion. You may embed a specific
          job ID into this URL using <code>$jobId</code>.  This tag

Modified: hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml (original)
+++ hive/branches/vectorization/hcatalog/src/docs/src/documentation/content/xdocs/pig.xml Tue Sep 24 22:38:33 2013
@@ -70,6 +70,13 @@
         <td>None</td>
     </tr>
 
+    <tr><td><strong>enablelog</strong></td>
+        <td>Collecting hadoop job config and logs into $statusdir/logs folder.
+         statusdir must be set as well to use this feature.</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
     <tr><td><strong>callback</strong></td>
         <td>Define a URL to be called upon job completion. You may embed a specific
          job ID into this URL using <code>$jobId</code>.  This tag

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,25 +43,25 @@ public class HiveDelegator extends Launc
   public EnqueueBean run(String user,
                String execute, String srcFile, List<String> defines,
                List<String> hiveArgs, String otherFiles,
-               String statusdir, String callback, String completedUrl)
+               String statusdir, String callback, String completedUrl, boolean enablelog)
     throws NotAuthorizedException, BadParam, BusyException, QueueException,
     ExecuteException, IOException, InterruptedException
   {
     runAs = user;
     List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir,
-                   completedUrl);
+                   completedUrl, enablelog);
 
     return enqueueController(user, callback, args);
   }
 
   private List<String> makeArgs(String execute, String srcFile,
              List<String> defines, List<String> hiveArgs, String otherFiles,
-             String statusdir, String completedUrl)
+             String statusdir, String completedUrl, boolean enablelog)
     throws BadParam, IOException, InterruptedException
   {
     ArrayList<String> args = new ArrayList<String>();
     try {
-      args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl));
+      args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
       args.add("--");
       args.add(appConf.hivePath());
 
@@ -99,7 +99,8 @@ public class HiveDelegator extends Launc
   }
 
   private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles,
-                                         String statusdir, String completedUrl)
+                                         String statusdir, String completedUrl,
+                                         boolean enablelog)
     throws URISyntaxException, FileNotFoundException, IOException,
     InterruptedException
   {
@@ -115,7 +116,8 @@ public class HiveDelegator extends Launc
       allFiles.addAll(Arrays.asList(ofs));
     }
 
-    args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
+    args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
+                enablelog, JobType.HIVE));
 
     args.add("-archives");
     args.add(appConf.hiveArchive());

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Sep 24 22:38:33 2013
@@ -41,13 +41,14 @@ public class JarDelegator extends Launch
   public EnqueueBean run(String user, String jar, String mainClass,
                String libjars, String files,
                List<String> jarArgs, List<String> defines,
-               String statusdir, String callback, String completedUrl)
+               String statusdir, String callback, String completedUrl,
+               boolean enablelog, JobType jobType)
     throws NotAuthorizedException, BadParam, BusyException, QueueException,
     ExecuteException, IOException, InterruptedException {
     runAs = user;
     List<String> args = makeArgs(jar, mainClass,
       libjars, files, jarArgs, defines,
-      statusdir, completedUrl);
+      statusdir, completedUrl, enablelog, jobType);
 
     return enqueueController(user, callback, args);
   }
@@ -55,7 +56,8 @@ public class JarDelegator extends Launch
   private List<String> makeArgs(String jar, String mainClass,
                   String libjars, String files,
                   List<String> jarArgs, List<String> defines,
-                  String statusdir, String completedUrl)
+                  String statusdir, String completedUrl,
+                  boolean enablelog, JobType jobType)
     throws BadParam, IOException, InterruptedException {
     ArrayList<String> args = new ArrayList<String>();
     try {
@@ -63,7 +65,7 @@ public class JarDelegator extends Launch
       allFiles.add(TempletonUtils.hadoopFsFilename(jar, appConf, runAs));
 
       args.addAll(makeLauncherArgs(appConf, statusdir,
-        completedUrl, allFiles));
+        completedUrl, allFiles, enablelog, jobType));
       args.add("--");
       args.add(appConf.clusterHadoop());
       args.add("jar");

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,6 +43,7 @@ import org.apache.hive.hcatalog.templeto
 public class LauncherDelegator extends TempletonDelegator {
   private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
   protected String runAs = null;
+  static public enum JobType {JAR, STREAMING, PIG, HIVE};
 
   public LauncherDelegator(AppConfig appConf) {
     super(appConf);
@@ -105,7 +106,9 @@ public class LauncherDelegator extends T
 
   public List<String> makeLauncherArgs(AppConfig appConf, String statusdir,
                      String completedUrl,
-                     List<String> copyFiles) {
+                     List<String> copyFiles,
+                     boolean enablelog,
+                     JobType jobType) {
     ArrayList<String> args = new ArrayList<String>();
 
     args.add("-libjars");
@@ -123,6 +126,10 @@ public class LauncherDelegator extends T
       TempletonUtils.encodeArray(copyFiles));
     addDef(args, TempletonControllerJob.OVERRIDE_CLASSPATH,
       makeOverrideClasspath(appConf));
+    addDef(args, TempletonControllerJob.ENABLE_LOG,
+      Boolean.toString(enablelog));
+    addDef(args, TempletonControllerJob.JOB_TYPE,
+      jobType.toString());
 
     // Hadoop queue information
     addDef(args, "mapred.job.queue.name", appConf.hadoopQueueName());

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Sep 24 22:38:33 2013
@@ -42,20 +42,20 @@ public class PigDelegator extends Launch
   public EnqueueBean run(String user,
                String execute, String srcFile,
                List<String> pigArgs, String otherFiles,
-               String statusdir, String callback, String completedUrl)
+               String statusdir, String callback, String completedUrl, boolean enablelog)
     throws NotAuthorizedException, BadParam, BusyException, QueueException,
     ExecuteException, IOException, InterruptedException {
     runAs = user;
     List<String> args = makeArgs(execute,
       srcFile, pigArgs,
-      otherFiles, statusdir, completedUrl);
+      otherFiles, statusdir, completedUrl, enablelog);
 
     return enqueueController(user, callback, args);
   }
 
   private List<String> makeArgs(String execute, String srcFile,
                   List<String> pigArgs, String otherFiles,
-                  String statusdir, String completedUrl)
+                  String statusdir, String completedUrl, boolean enablelog)
     throws BadParam, IOException, InterruptedException {
     ArrayList<String> args = new ArrayList<String>();
     try {
@@ -68,7 +68,7 @@ public class PigDelegator extends Launch
         allFiles.addAll(Arrays.asList(ofs));
       }
 
-      args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
+      args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
       args.add("-archives");
       args.add(appConf.pigArchive());
 

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Sep 24 22:38:33 2013
@@ -50,6 +50,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.hive.hcatalog.templeton.LauncherDelegator.JobType;
 import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
 
 /**
@@ -591,18 +592,21 @@ public class Server {
                       @FormParam("cmdenv") List<String> cmdenvs,
                       @FormParam("arg") List<String> args,
                       @FormParam("statusdir") String statusdir,
-                      @FormParam("callback") String callback)
+                      @FormParam("callback") String callback,
+                      @FormParam("enablelog") boolean enablelog)
     throws NotAuthorizedException, BusyException, BadParam, QueueException,
     ExecuteException, IOException, InterruptedException {
     verifyUser();
     verifyParam(inputs, "input");
     verifyParam(mapper, "mapper");
     verifyParam(reducer, "reducer");
+    
+    checkEnableLogPrerequisite(enablelog, statusdir);
 
     StreamingDelegator d = new StreamingDelegator(appConf);
     return d.run(getDoAsUser(), inputs, output, mapper, reducer,
       files, defines, cmdenvs, args,
-      statusdir, callback, getCompletedUrl());
+      statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING);
   }
 
   /**
@@ -618,18 +622,21 @@ public class Server {
                   @FormParam("arg") List<String> args,
                   @FormParam("define") List<String> defines,
                   @FormParam("statusdir") String statusdir,
-                  @FormParam("callback") String callback)
+                  @FormParam("callback") String callback,
+                  @FormParam("enablelog") boolean enablelog)
     throws NotAuthorizedException, BusyException, BadParam, QueueException,
     ExecuteException, IOException, InterruptedException {
     verifyUser();
     verifyParam(jar, "jar");
     verifyParam(mainClass, "class");
+    
+    checkEnableLogPrerequisite(enablelog, statusdir);
 
     JarDelegator d = new JarDelegator(appConf);
     return d.run(getDoAsUser(),
       jar, mainClass,
       libjars, files, args, defines,
-      statusdir, callback, getCompletedUrl());
+      statusdir, callback, getCompletedUrl(), enablelog, JobType.JAR);
   }
 
   /**
@@ -643,18 +650,21 @@ public class Server {
                @FormParam("arg") List<String> pigArgs,
                @FormParam("files") String otherFiles,
                @FormParam("statusdir") String statusdir,
-               @FormParam("callback") String callback)
+               @FormParam("callback") String callback,
+               @FormParam("enablelog") boolean enablelog)
     throws NotAuthorizedException, BusyException, BadParam, QueueException,
     ExecuteException, IOException, InterruptedException {
     verifyUser();
     if (execute == null && srcFile == null)
       throw new BadParam("Either execute or file parameter required");
+    
+    checkEnableLogPrerequisite(enablelog, statusdir);
 
     PigDelegator d = new PigDelegator(appConf);
     return d.run(getDoAsUser(),
       execute, srcFile,
       pigArgs, otherFiles,
-      statusdir, callback, getCompletedUrl());
+      statusdir, callback, getCompletedUrl(), enablelog);
   }
 
   /**
@@ -670,6 +680,7 @@ public class Server {
    * @param defines    shortcut for command line arguments "--define"
    * @param statusdir  where the stderr/stdout of templeton controller job goes
    * @param callback   callback url when the hive job finishes
+   * @param enablelog  whether to collect mapreduce log into statusdir/logs
    */
   @POST
   @Path("hive")
@@ -680,16 +691,19 @@ public class Server {
               @FormParam("files") String otherFiles,
               @FormParam("define") List<String> defines,
               @FormParam("statusdir") String statusdir,
-              @FormParam("callback") String callback)
+              @FormParam("callback") String callback,
+              @FormParam("enablelog") boolean enablelog)
     throws NotAuthorizedException, BusyException, BadParam, QueueException,
     ExecuteException, IOException, InterruptedException {
     verifyUser();
     if (execute == null && srcFile == null)
       throw new BadParam("Either execute or file parameter required");
+    
+    checkEnableLogPrerequisite(enablelog, statusdir);
 
     HiveDelegator d = new HiveDelegator(appConf);
     return d.run(getDoAsUser(), execute, srcFile, defines, hiveArgs, otherFiles,
-      statusdir, callback, getCompletedUrl());
+      statusdir, callback, getCompletedUrl(), enablelog);
   }
 
   /**
@@ -902,6 +916,7 @@ public class Server {
     return theUriInfo.getBaseUri() + VERSION
       + "/internal/complete/$jobId";
   }
+
   /**
    * Returns canonical host name from which the request is made; used for doAs validation  
    */
@@ -930,4 +945,9 @@ public class Server {
       return unkHost;
     }
   }
+  
+  private void checkEnableLogPrerequisite(boolean enablelog, String statusdir) throws BadParam {
+    if (enablelog == true && !TempletonUtils.isset(statusdir))
+      throw new BadParam("enablelog is only applicable when statusdir is set");
+  }
 }

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java Tue Sep 24 22:38:33 2013
@@ -43,7 +43,9 @@ public class StreamingDelegator extends 
                List<String> jarArgs,
                String statusdir,
                String callback,
-               String completedUrl)
+               String completedUrl,
+               boolean enableLog,
+               JobType jobType)
     throws NotAuthorizedException, BadParam, BusyException, QueueException,
     ExecuteException, IOException, InterruptedException {
     List<String> args = makeArgs(inputs, output, mapper, reducer,
@@ -53,7 +55,7 @@ public class StreamingDelegator extends 
     return d.run(user,
       appConf.streamingJar(), null,
       null, null, args, defines,
-      statusdir, callback, completedUrl);
+      statusdir, callback, completedUrl, enableLog, jobType);
   }
 
   private List<String> makeArgs(List<String> inputs,

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Sep 24 22:38:33 2013
@@ -24,6 +24,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -53,6 +54,7 @@ import org.apache.hadoop.security.UserGr
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.templeton.LauncherDelegator;
 
 /**
  * A Map Reduce job that will start another job.
@@ -69,6 +71,8 @@ import org.apache.hadoop.util.ToolRunner
 public class TempletonControllerJob extends Configured implements Tool {
   public static final String COPY_NAME = "templeton.copy";
   public static final String STATUSDIR_NAME = "templeton.statusdir";
+  public static final String ENABLE_LOG = "templeton.enablelog";
+  public static final String JOB_TYPE = "templeton.jobtype";
   public static final String JAR_ARGS_NAME = "templeton.args";
   public static final String OVERRIDE_CLASSPATH = "templeton.override-classpath";
 
@@ -155,9 +159,17 @@ public class TempletonControllerJob exte
       String statusdir = conf.get(STATUSDIR_NAME);
 
       if (statusdir != null) {
-        statusdir = TempletonUtils.addUserHomeDirectoryIfApplicable(statusdir, conf.get("user.name"), conf);
+        try {
+          statusdir = TempletonUtils.addUserHomeDirectoryIfApplicable(statusdir,
+            conf.get("user.name"));
+        } catch (URISyntaxException e) {
+          throw new IOException("Invalid status dir URI", e);
+        }
       }
 
+      Boolean enablelog = Boolean.parseBoolean(conf.get(ENABLE_LOG));
+      LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
+
       ExecutorService pool = Executors.newCachedThreadPool();
       executeWatcher(pool, conf, context.getJobID(),
         proc.getInputStream(), statusdir, STDOUT_FNAME);
@@ -177,6 +189,13 @@ public class TempletonControllerJob exte
       state.setCompleteStatus("done");
       state.close();
 
+      if (enablelog && TempletonUtils.isset(statusdir)) {
+        System.err.println("templeton: collecting logs for " + context.getJobID().toString()
+          + " to " + statusdir + "/logs");
+        LogRetriever logRetriever = new LogRetriever(statusdir, jobType, conf);
+        logRetriever.run();
+      }
+
       if (proc.exitValue() != 0)
         System.err.println("templeton: job failed with exit code "
           + proc.exitValue());

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java Tue Sep 24 22:38:33 2013
@@ -33,11 +33,11 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.ws.rs.core.UriBuilder;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hive.hcatalog.templeton.UgiFactory;
@@ -213,22 +213,19 @@ public class TempletonUtils {
     }
   }
 
-  public static String addUserHomeDirectoryIfApplicable(String origPathStr, String user, Configuration conf) throws IOException {
-    Path path = new Path(origPathStr);
-    String result = origPathStr;
-
-    // shortcut for s3/asv
-    // If path contains scheme, user should mean an absolute path,
-    // However, path.isAbsolute tell us otherwise.
-    // So we skip conversion for non-hdfs.
-    if (!(path.getFileSystem(conf) instanceof DistributedFileSystem)&&
-        !(path.getFileSystem(conf) instanceof LocalFileSystem)) {
-      return result;
-    }
-    if (!path.isAbsolute()) {
-      result = "/user/" + user + "/" + origPathStr;
-    }
-    return result;
+  public static String addUserHomeDirectoryIfApplicable(String origPathStr, String user)
+    throws IOException, URISyntaxException {
+    URI uri = new URI(origPathStr);
+
+    if (uri.getPath().isEmpty()) {
+      String newPath = "/user/" + user;
+      uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
+    } else if (!new Path(uri.getPath()).isAbsolute()) {
+      String newPath = "/user/" + user + "/" + uri.getPath();
+      uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
+    } // no work needed for absolute paths
+
+    return uri.toString();
   }
 
   public static Path hadoopFsPath(String fname, final Configuration conf, String user)
@@ -254,7 +251,7 @@ public class TempletonUtils {
           }
         });
 
-    fname = addUserHomeDirectoryIfApplicable(fname, user, conf);
+    fname = addUserHomeDirectoryIfApplicable(fname, user);
     URI u = new URI(fname);
     Path p = new Path(u).makeQualified(defaultFs);
 

Modified: hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/branches/vectorization/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Tue Sep 24 22:38:33 2013
@@ -20,6 +20,7 @@ package org.apache.hive.hcatalog.templet
 
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.net.URISyntaxException;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
@@ -245,4 +246,32 @@ public class TestTempletonUtils {
     }
   }
 
+  @Test
+  public void testConstructingUserHomeDirectory() throws Exception {
+    String[] sources = new String[] { "output+", "/user/hadoop/output",
+      "hdfs://container", "hdfs://container/", "hdfs://container/path",
+      "output#link", "hdfs://cointaner/output#link",
+      "hdfs://container@acc/test" };
+    String[] expectedResults = new String[] { "/user/webhcat/output+",
+      "/user/hadoop/output", "hdfs://container/user/webhcat",
+      "hdfs://container/", "hdfs://container/path",
+      "/user/webhcat/output#link", "hdfs://cointaner/output#link",
+      "hdfs://container@acc/test" };
+    for (int i = 0; i < sources.length; i++) {
+      String source = sources[i];
+      String expectedResult = expectedResults[i];
+      String result = TempletonUtils.addUserHomeDirectoryIfApplicable(source,
+          "webhcat");
+      Assert.assertEquals(result, expectedResult);
+    }
+
+    String badUri = "c:\\some\\path";
+    try {
+      TempletonUtils.addUserHomeDirectoryIfApplicable(badUri, "webhcat");
+      Assert.fail("addUserHomeDirectoryIfApplicable should fail for bad URI: "
+          + badUri);
+    } catch (URISyntaxException ex) {
+    }
+  }
+
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Tue Sep 24 22:38:33 2013
@@ -353,6 +353,11 @@ public class FetchOperator implements Se
     }
   }
 
+  /**
+   * A cache of Object Inspector Settable Properties.
+   */
+  private static Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
+
   private RecordReader<WritableComparable, Writable> getRecordReader() throws Exception {
     if (currPath == null) {
       getNextPath();
@@ -408,7 +413,8 @@ public class FetchOperator implements Se
 
       ObjectInspector outputOI = ObjectInspectorConverters.getConvertedOI(
           serde.getObjectInspector(),
-          partitionedTableOI == null ? tblSerde.getObjectInspector() : partitionedTableOI, true);
+          partitionedTableOI == null ? tblSerde.getObjectInspector() : partitionedTableOI,
+          oiSettableProperties);
 
       partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
           serde.getObjectInspector(), outputOI);
@@ -634,7 +640,7 @@ public class FetchOperator implements Se
         partSerde.initialize(job, listPart.getOverlayedProperties());
 
         partitionedTableOI = ObjectInspectorConverters.getConvertedOI(
-            partSerde.getObjectInspector(), tableOI, true);
+            partSerde.getObjectInspector(), tableOI, oiSettableProperties);
         if (!partitionedTableOI.equals(tableOI)) {
           break;
         }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Sep 24 22:38:33 2013
@@ -277,6 +277,8 @@ public class MapOperator extends Operato
         new HashMap<TableDesc, StructObjectInspector>();
     Set<TableDesc> identityConverterTableDesc = new HashSet<TableDesc>();
     try {
+      Map<ObjectInspector, Boolean> oiSettableProperties = new HashMap<ObjectInspector, Boolean>();
+
       for (String onefile : conf.getPathToAliases().keySet()) {
         PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
         TableDesc tableDesc = pd.getTableDesc();
@@ -310,7 +312,7 @@ public class MapOperator extends Operato
           tblRawRowObjectInspector =
               (StructObjectInspector) ObjectInspectorConverters.getConvertedOI(
                   partRawRowObjectInspector,
-                  tblDeserializer.getObjectInspector(), true);
+                  tblDeserializer.getObjectInspector(), oiSettableProperties);
 
           if (identityConverterTableDesc.contains(tableDesc)) {
             if (!partRawRowObjectInspector.equals(tblRawRowObjectInspector)) {

Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java Tue Sep 24 22:38:33 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -141,7 +142,6 @@ public final class ObjectInspectorConver
     if (inputOI.equals(outputOI)) {
       return new IdentityConverter();
     }
-    // TODO: Add support for UNION once SettableUnionObjectInspector is implemented.
     switch (outputOI.getCategory()) {
     case PRIMITIVE:
       return getConverter((PrimitiveObjectInspector) inputOI, (PrimitiveObjectInspector) outputOI);
@@ -154,6 +154,9 @@ public final class ObjectInspectorConver
     case MAP:
       return new MapConverter(inputOI,
           (SettableMapObjectInspector) outputOI);
+    case UNION:
+      return new UnionConverter(inputOI,
+          (SettableUnionObjectInspector) outputOI);
     default:
       throw new RuntimeException("Hive internal error: conversion of "
           + inputOI.getTypeName() + " to " + outputOI.getTypeName()
@@ -161,19 +164,49 @@ public final class ObjectInspectorConver
     }
   }
 
+  /*
+   * getConvertedOI with caching to store settable properties of the object
+   * inspector. Caching might help when the object inspector
+   * contains complex nested data types. Caching is not explicitly required for
+   * the returned object inspector across multiple invocations since the
+   * ObjectInspectorFactory already takes care of it.
+   */
   public static ObjectInspector getConvertedOI(
+      ObjectInspector inputOI, ObjectInspector outputOI,
+      Map<ObjectInspector, Boolean> oiSettableProperties
+      ) {
+    return getConvertedOI(inputOI, outputOI, oiSettableProperties, true);
+  }
+
+  /*
+   * getConvertedOI without any caching.
+   */
+  public static ObjectInspector getConvertedOI(
+      ObjectInspector inputOI,
+      ObjectInspector outputOI
+      ) {
+    return getConvertedOI(inputOI, outputOI, null, true);
+  }
+
+  /*
+   * Utility function to convert from one object inspector type to another.
+   */
+  private static ObjectInspector getConvertedOI(
       ObjectInspector inputOI,
       ObjectInspector outputOI,
+      Map<ObjectInspector, Boolean> oiSettableProperties,
       boolean equalsCheck) {
+    ObjectInspector retOI = outputOI.getCategory() == Category.PRIMITIVE ? inputOI : outputOI;
     // If the inputOI is the same as the outputOI, just return it
-    if (equalsCheck && inputOI.equals(outputOI)) {
-      return outputOI;
+    // If the retOI has all fields settable, return it
+    if ((equalsCheck && inputOI.equals(outputOI)) ||
+        ObjectInspectorUtils.hasAllFieldsSettable(retOI, oiSettableProperties) == true) {
+      return retOI;
     }
     // Return the settable equivalent object inspector for primitive categories
     // For eg: for table T containing partitions p1 and p2 (possibly different
     // from the table T), return the settable inspector for T. The inspector for
     // T is settable recursively i.e all the nested fields are also settable.
-    // TODO: Add support for UNION once SettableUnionObjectInspector is implemented.
     switch (outputOI.getCategory()) {
     case PRIMITIVE:
       PrimitiveObjectInspector primInputOI = (PrimitiveObjectInspector) inputOI;
@@ -189,7 +222,7 @@ public final class ObjectInspectorConver
       for (StructField listField : listFields) {
         structFieldNames.add(listField.getFieldName());
         structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(),
-            listField.getFieldObjectInspector(), false));
+            listField.getFieldObjectInspector(), oiSettableProperties, false));
       }
       return ObjectInspectorFactory.getStandardStructObjectInspector(
           structFieldNames,
@@ -198,14 +231,25 @@ public final class ObjectInspectorConver
       ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
       return ObjectInspectorFactory.getStandardListObjectInspector(
           getConvertedOI(listOutputOI.getListElementObjectInspector(),
-              listOutputOI.getListElementObjectInspector(), false));
+              listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
     case MAP:
       MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
       return ObjectInspectorFactory.getStandardMapObjectInspector(
           getConvertedOI(mapOutputOI.getMapKeyObjectInspector(),
-              mapOutputOI.getMapKeyObjectInspector(), false),
+              mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false),
           getConvertedOI(mapOutputOI.getMapValueObjectInspector(),
-              mapOutputOI.getMapValueObjectInspector(), false));
+              mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
+    case UNION:
+      UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
+      // create a standard settable union object inspector
+      List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
+      List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(
+          unionListFields.size());
+      for (ObjectInspector listField : unionListFields) {
+        unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties,
+            false));
+      }
+      return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
     default:
       throw new RuntimeException("Hive internal error: conversion of "
           + inputOI.getTypeName() + " to " + outputOI.getTypeName()
@@ -335,6 +379,67 @@ public final class ObjectInspectorConver
   }
 
   /**
+   * A converter class for Union.
+   */
+  public static class UnionConverter implements Converter {
+
+    UnionObjectInspector inputOI;
+    SettableUnionObjectInspector outputOI;
+
+    List<? extends ObjectInspector> inputFields;
+    List<? extends ObjectInspector> outputFields;
+
+    ArrayList<Converter> fieldConverters;
+
+    Object output;
+
+    public UnionConverter(ObjectInspector inputOI,
+        SettableUnionObjectInspector outputOI) {
+      if (inputOI instanceof UnionObjectInspector) {
+        this.inputOI = (UnionObjectInspector)inputOI;
+        this.outputOI = outputOI;
+        inputFields = this.inputOI.getObjectInspectors();
+        outputFields = outputOI.getObjectInspectors();
+
+        // If the output has some extra fields, set them to NULL in convert().
+        int minFields = Math.min(inputFields.size(), outputFields.size());
+        fieldConverters = new ArrayList<Converter>(minFields);
+        for (int f = 0; f < minFields; f++) {
+          fieldConverters.add(getConverter(inputFields.get(f), outputFields.get(f)));
+        }
+
+        // Create an empty output object which will be populated when convert() is invoked.
+        output = outputOI.create();
+      } else if (!(inputOI instanceof VoidObjectInspector)) {
+        throw new RuntimeException("Hive internal error: conversion of " +
+            inputOI.getTypeName() + " to " + outputOI.getTypeName() +
+            "not supported yet.");
+      }
+    }
+
+    @Override
+    public Object convert(Object input) {
+      if (input == null) {
+        return null;
+      }
+
+      int minFields = Math.min(inputFields.size(), outputFields.size());
+      // Convert the fields
+      for (int f = 0; f < minFields; f++) {
+        Object outputFieldValue = fieldConverters.get(f).convert(inputOI);
+        outputOI.addField(output, (ObjectInspector)outputFieldValue);
+      }
+
+      // set the extra fields to null
+      for (int f = minFields; f < outputFields.size(); f++) {
+        outputOI.addField(output, null);
+      }
+
+      return output;
+    }
+  }
+
+  /**
    * A converter class for Map.
    */
   public static class MapConverter implements Converter {

Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Sep 24 22:38:33 2013
@@ -46,11 +46,25 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.BytesWritable;
@@ -1003,6 +1017,138 @@ public final class ObjectInspectorUtils 
     return (oi instanceof ConstantObjectInspector);
   }
 
+  private static boolean setOISettablePropertiesMap(ObjectInspector oi,
+      Map<ObjectInspector, Boolean> oiSettableProperties, boolean value) {
+    // Cache if the client asks for it, else just return the value
+    if (!(oiSettableProperties == null)) {
+      oiSettableProperties.put(oi, value);
+    }
+    return value;
+  }
+
+  private static boolean isInstanceOfSettablePrimitiveOI(PrimitiveObjectInspector oi) {
+    switch (oi.getPrimitiveCategory()) {
+    case BOOLEAN:
+      return oi instanceof SettableBooleanObjectInspector;
+    case BYTE:
+      return oi instanceof SettableByteObjectInspector;
+    case SHORT:
+      return oi instanceof SettableShortObjectInspector;
+    case INT:
+      return oi instanceof SettableIntObjectInspector;
+    case LONG:
+      return oi instanceof SettableLongObjectInspector ;
+    case FLOAT:
+      return oi instanceof SettableFloatObjectInspector;
+    case DOUBLE:
+      return oi instanceof SettableDoubleObjectInspector;
+    case STRING:
+      return oi instanceof WritableStringObjectInspector ||
+          oi instanceof JavaStringObjectInspector;
+    case VARCHAR:
+      return oi instanceof SettableHiveVarcharObjectInspector;
+    case DATE:
+      return oi instanceof SettableDateObjectInspector;
+    case TIMESTAMP:
+      return oi instanceof SettableTimestampObjectInspector;
+    case BINARY:
+      return oi instanceof SettableBinaryObjectInspector;
+    case DECIMAL:
+      return oi instanceof SettableHiveDecimalObjectInspector;
+    default:
+      throw new RuntimeException("Hive internal error inside isAssignableFromSettablePrimitiveOI "
+                          + oi.getTypeName() + " not supported yet.");
+    }
+  }
+
+  private static boolean isInstanceOfSettableOI(ObjectInspector oi)
+  {
+    switch (oi.getCategory()) {
+      case PRIMITIVE:
+        return isInstanceOfSettablePrimitiveOI((PrimitiveObjectInspector)oi);
+      case STRUCT:
+        return oi instanceof SettableStructObjectInspector;
+      case LIST:
+        return oi instanceof SettableListObjectInspector;
+      case MAP:
+        return oi instanceof SettableMapObjectInspector;
+      case UNION:
+        return oi instanceof SettableUnionObjectInspector;
+      default:
+        throw new RuntimeException("Hive internal error inside isAssignableFromSettableOI : "
+            + oi.getTypeName() + " not supported yet.");
+    }
+  }
+
+  /*
+   * hasAllFieldsSettable without any caching.
+   */
+  public static Boolean hasAllFieldsSettable(ObjectInspector oi) {
+    return hasAllFieldsSettable(oi, null);
+  }
+
+  /**
+   *
+   * @param oi - Input object inspector
+   * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
+   * @return - true if : (1) oi is an instance of settable<DataType>OI.
+   *                     (2) All the embedded object inspectors are instances of settable<DataType>OI.
+   *           If (1) or (2) is false, return false.
+   */
+  public static boolean hasAllFieldsSettable(ObjectInspector oi,
+      Map<ObjectInspector, Boolean> oiSettableProperties) {
+    // If the result is already present in the cache, return it.
+    if (!(oiSettableProperties == null) &&
+        oiSettableProperties.containsKey(oi)) {
+      return oiSettableProperties.get(oi).booleanValue();
+    }
+    // If the top-level object inspector is non-settable return false
+    if (!(isInstanceOfSettableOI(oi))) {
+      return setOISettablePropertiesMap(oi, oiSettableProperties, false);
+    }
+
+    Boolean returnValue = true;
+
+    switch (oi.getCategory()) {
+    case PRIMITIVE:
+      break;
+    case STRUCT:
+      StructObjectInspector structOutputOI = (StructObjectInspector) oi;
+      List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
+      for (StructField listField : listFields) {
+        if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
+          returnValue = false;
+          break;
+        }
+      }
+      break;
+    case LIST:
+      ListObjectInspector listOutputOI = (ListObjectInspector) oi;
+      returnValue = hasAllFieldsSettable(listOutputOI.getListElementObjectInspector(),
+          oiSettableProperties);
+      break;
+    case MAP:
+      MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
+      returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) &&
+          hasAllFieldsSettable(mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
+      break;
+    case UNION:
+      UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
+      List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
+      for (ObjectInspector listField : unionListFields) {
+        if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
+          returnValue = false;
+          break;
+        }
+      }
+      break;
+    default:
+      throw new RuntimeException("Hive internal error inside hasAllFieldsSettable : "
+          + oi.getTypeName() + " not supported yet.");
+    }
+    return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
+  }
+
   private ObjectInspectorUtils() {
     // prevent instantiation
   }

Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java?rev=1526035&r1=1526034&r2=1526035&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java Tue Sep 24 22:38:33 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.serde2.objectinspector;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -30,7 +31,7 @@ import java.util.List;
  * Always use the {@link ObjectInspectorFactory} to create new ObjectInspector
  * objects, instead of directly creating an instance of this class.
  */
-public class StandardUnionObjectInspector implements UnionObjectInspector {
+public class StandardUnionObjectInspector extends SettableUnionObjectInspector {
   private List<ObjectInspector> ois;
 
   protected StandardUnionObjectInspector() {
@@ -116,4 +117,17 @@ public class StandardUnionObjectInspecto
     return sb.toString();
   }
 
+  @Override
+  public Object create() {
+    ArrayList<Object> a = new ArrayList<Object>();
+    return a;
+  }
+
+  @Override
+  public Object addField(Object union, ObjectInspector oi) {
+    ArrayList<Object> a = (ArrayList<Object>) union;
+    a.add(oi);
+    return a;
+  }
+
 }



Mime
View raw message