hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1550461 - in /hive/trunk: metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/java/org/apache/had...
Date Thu, 12 Dec 2013 17:59:31 GMT
Author: hashutosh
Date: Thu Dec 12 17:59:31 2013
New Revision: 1550461

URL: http://svn.apache.org/r1550461
Log:
HIVE-6018 : FetchTask should not reference metastore classes (Navis via Prasad Mujumdar)

Modified:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Thu
Dec 12 17:59:31 2013
@@ -310,42 +310,6 @@ public class MetaStoreUtils {
   /**
    * getDeserializer
    *
-   * Get the Deserializer for a table given its name and properties.
-   *
-   * @param conf
-   *          hadoop config
-   * @param schema
-   *          the properties to use to instantiate the deserializer
-   * @return
-   *   Returns instantiated deserializer by looking up class name of deserializer stored
in passed
-   *   in properties. Also, initializes the deserializer with schema stored in passed in
properties.
-   * @exception MetaException
-   *              if any problems instantiating the Deserializer
-   *
-   *              todo - this should move somewhere into serde.jar
-   *
-   */
-  static public Deserializer getDeserializer(Configuration conf,
-      Properties schema) throws MetaException {
-    try {
-      String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB);
-      if(clazzName == null) {
-        throw new IllegalStateException("Property " + serdeConstants.SERIALIZATION_LIB +
" cannot be null");
-      }
-      Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(clazzName)
-          .asSubclass(Deserializer.class), conf);
-      deserializer.initialize(conf, schema);
-      return deserializer;
-    } catch (Exception e) {
-      LOG.error("error in initSerDe: " + e.getClass().getName() + " "
-          + e.getMessage(), e);
-      throw new MetaException(e.getClass().getName() + " " + e.getMessage());
-    }
-  }
-
-  /**
-   * getDeserializer
-   *
    * Get the Deserializer for a table.
    *
    * @param conf

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Thu Dec 12 17:59:31
2013
@@ -400,7 +400,7 @@ public class FetchOperator implements Se
       this.inputSplits = inputSplits;
 
       splitNum = 0;
-      serde = partDesc.getDeserializer();
+      serde = partDesc.getDeserializer(job);
       serde.initialize(job, partDesc.getOverlayedProperties());
 
       if (currTbl != null) {
@@ -646,7 +646,7 @@ public class FetchOperator implements Se
       // Get the OI corresponding to all the partitions
       for (PartitionDesc listPart : listParts) {
         partition = listPart;
-        Deserializer partSerde = listPart.getDeserializer();
+        Deserializer partSerde = listPart.getDeserializer(job);
         partSerde.initialize(job, listPart.getOverlayedProperties());
 
         partitionedTableOI = ObjectInspectorConverters.getConvertedOI(

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java Thu Dec
12 17:59:31 2013
@@ -307,8 +307,12 @@ public class CombineHiveInputFormat<K ex
       Class inputFormatClass = part.getInputFileFormatClass();
       String inputFormatClassName = inputFormatClass.getName();
       InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
-      String deserializerClassName = part.getDeserializer() == null ? null
-          : part.getDeserializer().getClass().getName();
+      String deserializerClassName = null;
+      try {
+        deserializerClassName = part.getDeserializer(job).getClass().getName();
+      } catch (Exception e) {
+        // ignore
+      }
 
       // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree
or not,
       // we use a configuration variable for the same

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Thu Dec 12 17:59:31
2013
@@ -260,19 +260,6 @@ public class Partition implements Serial
     return deserializer;
   }
 
-  final public Deserializer getDeserializer(Properties props) {
-    if (deserializer == null) {
-      try {
-        deserializer = MetaStoreUtils.getDeserializer(Hive.get().getConf(), props);
-      } catch (HiveException e) {
-        throw new RuntimeException(e);
-      } catch (MetaException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    return deserializer;
-  }
-
   public Properties getSchema() {
     return MetaStoreUtils.getSchema(tPartition, table.getTTable());
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Thu Dec 12 17:59:31
2013
@@ -23,18 +23,18 @@ import java.util.Enumeration;
 import java.util.LinkedHashMap;
 import java.util.Properties;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
 
 /**
  * PartitionDesc.
@@ -105,14 +105,19 @@ public class PartitionDesc implements Se
   }
 
   /**
-   * Return a deserializer object corresponding to the tableDesc.
+   * Return a deserializer object corresponding to the partitionDesc.
    */
-  public Deserializer getDeserializer() {
-    try {
-      return MetaStoreUtils.getDeserializer(Hive.get().getConf(), getProperties());
-    } catch (Exception e) {
-      return null;
-    }
+  public Deserializer getDeserializer(Configuration conf) throws Exception {
+    Properties schema = getProperties();
+    String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB);
+    if (clazzName == null) {
+      throw new IllegalStateException("Property " + serdeConstants.SERIALIZATION_LIB +
+          " cannot be null");
+    }
+    Deserializer deserializer = ReflectionUtils.newInstance(conf.getClassByName(clazzName)
+        .asSubclass(Deserializer.class), conf);
+    deserializer.initialize(conf, schema);
+    return deserializer;
   }
 
   public void setInputFileFormatClass(



Mime
View raw message