incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1298039 - in /incubator/hcatalog/trunk: ./ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapreduce/
Date Wed, 07 Mar 2012 18:57:33 GMT
Author: gates
Date: Wed Mar  7 18:57:33 2012
New Revision: 1298039

URL: http://svn.apache.org/viewvc?rev=1298039&view=rev
Log:
HCATALOG-282 HCatInputFormat shouldn't expect storageHandler to be serializable

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/PartInfo.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1298039&r1=1298038&r2=1298039&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Wed Mar  7 18:57:33 2012
@@ -77,6 +77,8 @@ Release 0.4.0 - Unreleased
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-282 HCatInputFormat shouldn't expect storageHandler to be serializable (khorgath via
gates)
+
   HCAT-281 HCat use table schema instead of partition schema to read partition data (daijy
and khorgath via gates)
 
   HCAT-286 NPE in HdfsAuthorizationProvider (enis via gates)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1298039&r1=1298038&r2=1298039&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Wed Mar  7
18:57:33 2012
@@ -74,6 +74,7 @@ import org.apache.hcatalog.mapreduce.HCa
 import org.apache.hcatalog.mapreduce.HCatStorageHandler;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
 import org.apache.hcatalog.mapreduce.OutputJobInfo;
+import org.apache.hcatalog.mapreduce.PartInfo;
 import org.apache.hcatalog.mapreduce.StorerInfo;
 import org.apache.thrift.TException;
 
@@ -477,6 +478,15 @@ public class HCatUtil {
                                  storerInfo.getIfClass(),
                                  storerInfo.getOfClass());
     }
+    
+    public static HCatStorageHandler getStorageHandler(Configuration conf, PartInfo partitionInfo)
throws IOException {
+      return HCatUtil.getStorageHandler(
+          conf, 
+          partitionInfo.getStorageHandlerClassName(), 
+          partitionInfo.getSerdeClassName(), 
+          partitionInfo.getInputFormatClassName(),
+          partitionInfo.getOutputFormatClassName());
+    }
 
     /**
      * Create an instance of a storage handler. If storageHandler == null,
@@ -497,7 +507,7 @@ public class HCatUtil {
                                                        String outputFormat) 
     throws IOException {
 
-        if (storageHandler == null) {
+        if ((storageHandler == null) || (storageHandler.equals(FosterStorageHandler.class.getName()))){
             try {
                 return new FosterStorageHandler(inputFormat,
                                                 outputFormat,

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java?rev=1298039&r1=1298038&r2=1298039&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
Wed Mar  7 18:57:33 2012
@@ -30,11 +30,9 @@ import java.util.Map;
  *  artifacts of tables which don't define a SerDe. This StorageHandler assumes
  *  the supplied storage artifacts are for a file-based storage system.
  */
-public class FosterStorageHandler extends HCatStorageHandler 
-  implements Serializable {
+public class FosterStorageHandler extends HCatStorageHandler {
 
-    public Configuration conf
-            ;
+    public Configuration conf;
     /** The directory under which data is initially written for a partitioned table */
     protected static final String DYNTEMP_DIR_NAME = "_DYN";
 

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java?rev=1298039&r1=1298038&r2=1298039&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
Wed Mar  7 18:57:33 2012
@@ -151,7 +151,8 @@ public abstract class HCatBaseInputForma
 
       HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
 
-      storageHandler = partitionInfo.getStorageHandler();
+      storageHandler = HCatUtil.getStorageHandler(
+          jobConf, partitionInfo);
 
       //Get the input format
       Class inputFormatClass = storageHandler.getInputFormatClass();
@@ -194,7 +195,9 @@ public abstract class HCatBaseInputForma
     PartInfo partitionInfo = hcatSplit.getPartitionInfo();
     JobContext jobContext = taskContext;
 
-    HCatStorageHandler storageHandler = partitionInfo.getStorageHandler();
+    HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(
+        jobContext.getConfiguration(), partitionInfo);
+    
     JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext);
 
     Class inputFormatClass = storageHandler.getInputFormatClass();

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/PartInfo.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/PartInfo.java?rev=1298039&r1=1298038&r2=1298039&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/PartInfo.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/PartInfo.java Wed Mar
 7 18:57:33 2012
@@ -37,7 +37,10 @@ public class PartInfo implements Seriali
   private final HCatSchema partitionSchema;
 
   /** The information about which input storage handler to use */
-  private final HCatStorageHandler storageHandler;
+  private final String storageHandlerClassName;
+  private final String inputFormatClassName;
+  private final String outputFormatClassName;
+  private final String serdeClassName;
 
   /** HCat-specific properties set at the partition */
   private final Properties hcatProperties;
@@ -65,12 +68,16 @@ public class PartInfo implements Seriali
                   String location, Properties hcatProperties, 
                   Map<String,String> jobProperties, HCatTableInfo tableInfo){
     this.partitionSchema = partitionSchema;
-    this.storageHandler = storageHandler;
     this.location = location;
     this.hcatProperties = hcatProperties;
     this.jobProperties = jobProperties;
     this.tableInfo = tableInfo;
-  }
+
+    this.storageHandlerClassName = storageHandler.getClass().getName();
+    this.inputFormatClassName = storageHandler.getInputFormatClass().getName();
+    this.serdeClassName = storageHandler.getSerDeClass().getName();
+    this.outputFormatClassName = storageHandler.getOutputFormatClass().getName();
+}
 
   /**
    * Gets the value of partitionSchema.
@@ -80,15 +87,33 @@ public class PartInfo implements Seriali
     return partitionSchema;
   }
 
+  /**
+   * @return the storage handler class name
+   */
+  public String getStorageHandlerClassName() {
+    return storageHandlerClassName;
+  }
 
   /**
-   * Gets the value of input storage driver class name.
-   * @return the input storage driver class name
+   * @return the inputFormatClassName
    */
-  public HCatStorageHandler getStorageHandler() {
-    return storageHandler;
+  public String getInputFormatClassName() {
+    return inputFormatClassName;
   }
 
+  /**
+   * @return the outputFormatClassName
+   */
+  public String getOutputFormatClassName() {
+    return outputFormatClassName;
+  }
+
+  /**
+   * @return the serdeClassName
+   */
+  public String getSerdeClassName() {
+    return serdeClassName;
+  }
 
   /**
    * Gets the value of hcatProperties.



Mime
View raw message