hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1594315 [2/3] - in /hive/trunk: hbase-handler/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/ hbase-handler/src/test/org/apache/hadoop/hive/hbase/ hbase-handler/src/test/queries/positive/ hbase-handler/src/test/results/positive/ ites...
Date Tue, 13 May 2014 18:23:15 GMT
Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java Tue May 13 18:23:14 2014
@@ -18,20 +18,17 @@
 
 package org.apache.hadoop.hive.hbase;
 
-import java.util.List;
-import java.util.Properties;
-
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.util.List;
+import java.util.Properties;
 
 /**
  * HBaseSerDeParameters encapsulates SerDeParameters and additional configurations that are specific for
@@ -39,318 +36,72 @@ import org.apache.hadoop.io.Text;
  *
  */
 public class HBaseSerDeParameters {
-  private SerDeParameters serdeParams;
 
-  private String columnMappingString;
-  private List<ColumnMapping> columnMapping;
-  private boolean doColumnRegexMatching;
-
-  private long putTimestamp;
-
-  private Class<?> compositeKeyClass;
-  private int keyIndex;
-
-  void init(Configuration job, Properties tbl, String serdeName) throws SerDeException {
-    serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, serdeName);
-    putTimestamp = Long.valueOf(tbl.getProperty(HBaseSerDe.HBASE_PUT_TIMESTAMP, "-1"));
-
-    String compKeyClass = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS);
-    if (compKeyClass != null) {
-      try {
-        compositeKeyClass = job.getClassByName(compKeyClass);
-      } catch (ClassNotFoundException e) {
-        throw new SerDeException(e);
-      }
-    }
+  private final String serdeName;
+  private final SerDeParameters serdeParams;
+
+  private final String columnMappingString;
+  private final ColumnMappings columnMappings;
+  private final boolean doColumnRegexMatching;
+
+  private final long putTimestamp;
+  private final HBaseKeyFactory keyFactory;
+
+  HBaseSerDeParameters(Configuration job, Properties tbl, String serdeName) throws SerDeException {
+    this.serdeName = serdeName;
+    this.serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, serdeName);
+    this.putTimestamp = Long.valueOf(tbl.getProperty(HBaseSerDe.HBASE_PUT_TIMESTAMP, "-1"));
 
     // Read configuration parameters
     columnMappingString = tbl.getProperty(HBaseSerDe.HBASE_COLUMNS_MAPPING);
     doColumnRegexMatching = Boolean.valueOf(tbl.getProperty(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, "true"));
     // Parse and initialize the HBase columns mapping
-    columnMapping = HBaseSerDe.parseColumnsMapping(columnMappingString, doColumnRegexMatching);
+    columnMappings = HBaseSerDe.parseColumnsMapping(columnMappingString, doColumnRegexMatching);
+    columnMappings.setHiveColumnDescription(serdeName, serdeParams.getColumnNames(), serdeParams.getColumnTypes());
+
+    // Precondition: make sure this is done after the rest of the SerDe initialization is done.
+    String hbaseTableStorageType = tbl.getProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE);
+    columnMappings.parseColumnStorageTypes(hbaseTableStorageType);
 
     // Build the type property string if not supplied
     String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
     if (columnTypeProperty == null) {
-      StringBuilder sb = new StringBuilder();
-
-      for (int i = 0; i < columnMapping.size(); i++) {
-        if (sb.length() > 0) {
-          sb.append(":");
-        }
-
-        ColumnMapping colMap = columnMapping.get(i);
-
-        if (colMap.hbaseRowKey) {
-          // the row key column becomes a STRING
-          sb.append(serdeConstants.STRING_TYPE_NAME);
-        } else if (colMap.qualifierName == null)  {
-          // a column family become a MAP
-          sb.append(serdeConstants.MAP_TYPE_NAME + "<" + serdeConstants.STRING_TYPE_NAME + ","
-              + serdeConstants.STRING_TYPE_NAME + ">");
-        } else {
-          // an individual column becomes a STRING
-          sb.append(serdeConstants.STRING_TYPE_NAME);
-        }
-      }
-      tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, sb.toString());
-    }
-
-    if (columnMapping.size() != serdeParams.getColumnNames().size()) {
-      throw new SerDeException(serdeName + ": columns has " +
-        serdeParams.getColumnNames().size() +
-        " elements while hbase.columns.mapping has " +
-        columnMapping.size() + " elements" +
-        " (counting the key if implicit)");
-    }
-
-    // check that the mapping schema is right;
-    // check that the "column-family:" is mapped to  Map<key,?>
-    // where key extends LazyPrimitive<?, ?> and thus has type Category.PRIMITIVE
-    for (int i = 0; i < columnMapping.size(); i++) {
-      ColumnMapping colMap = columnMapping.get(i);
-      if (colMap.qualifierName == null && !colMap.hbaseRowKey) {
-        TypeInfo typeInfo = serdeParams.getColumnTypes().get(i);
-        if ((typeInfo.getCategory() != Category.MAP) ||
-          (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getCategory()
-            !=  Category.PRIMITIVE)) {
-
-          throw new SerDeException(
-            serdeName + ": hbase column family '" + colMap.familyName
-            + "' should be mapped to Map<? extends LazyPrimitive<?, ?>,?>, that is "
-            + "the Key for the map should be of primitive type, but is mapped to "
-            + typeInfo.getTypeName());
-        }
-      }
+      tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, columnMappings.toTypesString());
     }
 
-    // Precondition: make sure this is done after the rest of the SerDe initialization is done.
-    String hbaseTableStorageType = tbl.getProperty(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE);
-    parseColumnStorageTypes(hbaseTableStorageType);
-    setKeyColumnOffset();
+    this.keyFactory = initKeyFactory(job, tbl);
   }
 
-  /*
-   * Utility method for parsing a string of the form '-,b,s,-,s:b,...' as a means of specifying
-   * whether to use a binary or an UTF string format to serialize and de-serialize primitive
-   * data types like boolean, byte, short, int, long, float, and double. This applies to
-   * regular columns and also to map column types which are associated with an HBase column
-   * family. For the map types, we apply the specification to the key or the value provided it
-   * is one of the above primitive types. The specifier is a colon separated value of the form
-   * -:s, or b:b where we have 's', 'b', or '-' on either side of the colon. 's' is for string
-   * format storage, 'b' is for native fixed width byte oriented storage, and '-' uses the
-   * table level default.
-   *
-   * @param hbaseTableDefaultStorageType - the specification associated with the table property
-   *        hbase.table.default.storage.type
-   * @throws SerDeException on parse error.
-   */
-
-  public void parseColumnStorageTypes(String hbaseTableDefaultStorageType)
-      throws SerDeException {
-
-    boolean tableBinaryStorage = false;
-
-    if (hbaseTableDefaultStorageType != null && !"".equals(hbaseTableDefaultStorageType)) {
-      if (hbaseTableDefaultStorageType.equals("binary")) {
-        tableBinaryStorage = true;
-      } else if (!hbaseTableDefaultStorageType.equals("string")) {
-        throw new SerDeException("Error: " + HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE +
-            " parameter must be specified as" +
-            " 'string' or 'binary'; '" + hbaseTableDefaultStorageType +
-            "' is not a valid specification for this table/serde property.");
-      }
-    }
-
-    // parse the string to determine column level storage type for primitive types
-    // 's' is for variable length string format storage
-    // 'b' is for fixed width binary storage of bytes
-    // '-' is for table storage type, which defaults to UTF8 string
-    // string data is always stored in the default escaped storage format; the data types
-    // byte, short, int, long, float, and double have a binary byte oriented storage option
-    List<TypeInfo> columnTypes = serdeParams.getColumnTypes();
-
-    for (int i = 0; i < columnMapping.size(); i++) {
-
-      ColumnMapping colMap = columnMapping.get(i);
-      TypeInfo colType = columnTypes.get(i);
-      String mappingSpec = colMap.mappingSpec;
-      String [] mapInfo = mappingSpec.split("#");
-      String [] storageInfo = null;
-
-      if (mapInfo.length == 2) {
-        storageInfo = mapInfo[1].split(":");
-      }
-
-      if (storageInfo == null) {
-
-        // use the table default storage specification
-        if (colType.getCategory() == Category.PRIMITIVE) {
-          if (!colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else {
-            colMap.binaryStorage.add(false);
-          }
-        } else if (colType.getCategory() == Category.MAP) {
-          TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo();
-          TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo();
-
-          if (keyTypeInfo.getCategory() == Category.PRIMITIVE &&
-              !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else {
-            colMap.binaryStorage.add(false);
-          }
-
-          if (valueTypeInfo.getCategory() == Category.PRIMITIVE &&
-              !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else {
-            colMap.binaryStorage.add(false);
-          }
-        } else {
-          colMap.binaryStorage.add(false);
-        }
-
-      } else if (storageInfo.length == 1) {
-        // we have a storage specification for a primitive column type
-        String storageOption = storageInfo[0];
-
-        if ((colType.getCategory() == Category.MAP) ||
-            !(storageOption.equals("-") || "string".startsWith(storageOption) ||
-                "binary".startsWith(storageOption))) {
-          throw new SerDeException("Error: A column storage specification is one of the following:"
-              + " '-', a prefix of 'string', or a prefix of 'binary'. "
-              + storageOption + " is not a valid storage option specification for "
-              + serdeParams.getColumnNames().get(i));
-        }
-
-        if (colType.getCategory() == Category.PRIMITIVE &&
-            !colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-
-          if ("-".equals(storageOption)) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else if ("binary".startsWith(storageOption)) {
-            colMap.binaryStorage.add(true);
-          } else {
-              colMap.binaryStorage.add(false);
-          }
-        } else {
-          colMap.binaryStorage.add(false);
-        }
-
-      } else if (storageInfo.length == 2) {
-        // we have a storage specification for a map column type
-
-        String keyStorage = storageInfo[0];
-        String valStorage = storageInfo[1];
-
-        if ((colType.getCategory() != Category.MAP) ||
-            !(keyStorage.equals("-") || "string".startsWith(keyStorage) ||
-                "binary".startsWith(keyStorage)) ||
-            !(valStorage.equals("-") || "string".startsWith(valStorage) ||
-                "binary".startsWith(valStorage))) {
-          throw new SerDeException("Error: To specify a valid column storage type for a Map"
-              + " column, use any two specifiers from '-', a prefix of 'string', "
-              + " and a prefix of 'binary' separated by a ':'."
-              + " Valid examples are '-:-', 's:b', etc. They specify the storage type for the"
-              + " key and value parts of the Map<?,?> respectively."
-              + " Invalid storage specification for column "
-              + serdeParams.getColumnNames().get(i)
-              + "; " + storageInfo[0] + ":" + storageInfo[1]);
-        }
-
-        TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo();
-        TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo();
-
-        if (keyTypeInfo.getCategory() == Category.PRIMITIVE &&
-            !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-
-          if (keyStorage.equals("-")) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else if ("binary".startsWith(keyStorage)) {
-            colMap.binaryStorage.add(true);
-          } else {
-            colMap.binaryStorage.add(false);
-          }
-        } else {
-          colMap.binaryStorage.add(false);
-        }
-
-        if (valueTypeInfo.getCategory() == Category.PRIMITIVE &&
-            !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
-          if (valStorage.equals("-")) {
-            colMap.binaryStorage.add(tableBinaryStorage);
-          } else if ("binary".startsWith(valStorage)) {
-            colMap.binaryStorage.add(true);
-          } else {
-            colMap.binaryStorage.add(false);
-          }
-        } else {
-          colMap.binaryStorage.add(false);
-        }
-
-        if (colMap.binaryStorage.size() != 2) {
-          throw new SerDeException("Error: In parsing the storage specification for column "
-              + serdeParams.getColumnNames().get(i));
-        }
-
-      } else {
-        // error in storage specification
-        throw new SerDeException("Error: " + HBaseSerDe.HBASE_COLUMNS_MAPPING + " storage specification "
-            + mappingSpec + " is not valid for column: "
-            + serdeParams.getColumnNames().get(i));
+  private HBaseKeyFactory initKeyFactory(Configuration conf, Properties tbl) throws SerDeException {
+    try {
+      HBaseKeyFactory keyFactory = createKeyFactory(conf, tbl);
+      if (keyFactory != null) {
+        keyFactory.init(this, tbl);
       }
+      return keyFactory;
+    } catch (Exception e) {
+      throw new SerDeException(e);
     }
   }
 
-  void setKeyColumnOffset() throws SerDeException {
-    setKeyIndex(getRowKeyColumnOffset(columnMapping));
-  }
-
-  public static int getRowKeyColumnOffset(List<ColumnMapping> columnsMapping)
-      throws SerDeException {
-
-    for (int i = 0; i < columnsMapping.size(); i++) {
-      ColumnMapping colMap = columnsMapping.get(i);
-
-      if (colMap.hbaseRowKey && colMap.familyName.equals(HBaseSerDe.HBASE_KEY_COL)) {
-        return i;
-      }
+  private static HBaseKeyFactory createKeyFactory(Configuration job, Properties tbl) throws Exception {
+    String factoryClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_FACTORY);
+    if (factoryClassName != null) {
+      Class<?> factoryClazz = Class.forName(factoryClassName);
+      return (HBaseKeyFactory) ReflectionUtils.newInstance(factoryClazz, job);
     }
-
-    throw new SerDeException("HBaseSerDe Error: columns mapping list does not contain" +
-      " row key column.");
-  }
-
-  public StructTypeInfo getRowTypeInfo() {
-    return (StructTypeInfo) serdeParams.getRowTypeInfo();
+    String keyClassName = tbl.getProperty(HBaseSerDe.HBASE_COMPOSITE_KEY_CLASS);
+    if (keyClassName != null) {
+      Class<?> keyClass = Class.forName(keyClassName);
+      return new CompositeHBaseKeyFactory(keyClass);
+    }
+    return new DefaultHBaseKeyFactory();
   }
 
   public List<String> getColumnNames() {
     return serdeParams.getColumnNames();
   }
 
-  public byte[] getSeparators() {
-    return serdeParams.getSeparators();
-  }
-
-  public Text getNullSequence() {
-    return serdeParams.getNullSequence();
-  }
-
-  public boolean isLastColumnTakesRest() {
-    return serdeParams.isLastColumnTakesRest();
-  }
-
-  public boolean isEscaped() {
-    return serdeParams.isEscaped();
-  }
-
-  public byte getEscapeChar() {
-    return serdeParams.getEscapeChar();
-  }
-
   public List<TypeInfo> getColumnTypes() {
     return serdeParams.getColumnTypes();
   }
@@ -359,59 +110,38 @@ public class HBaseSerDeParameters {
     return serdeParams;
   }
 
-  public void setSerdeParams(SerDeParameters serdeParams) {
-    this.serdeParams = serdeParams;
-  }
-
-  public String getColumnMappingString() {
-    return columnMappingString;
-  }
-
-  public void setColumnMappingString(String columnMappingString) {
-    this.columnMappingString = columnMappingString;
-  }
-
   public long getPutTimestamp() {
     return putTimestamp;
   }
 
-  public void setPutTimestamp(long putTimestamp) {
-    this.putTimestamp = putTimestamp;
-  }
-
-  public boolean isDoColumnRegexMatching() {
-    return doColumnRegexMatching;
-  }
-
-  public void setDoColumnRegexMatching(boolean doColumnRegexMatching) {
-    this.doColumnRegexMatching = doColumnRegexMatching;
-  }
-
-  public Class<?> getCompositeKeyClass() {
-    return compositeKeyClass;
-  }
-
-  public void setCompositeKeyClass(Class<?> compositeKeyClass) {
-    this.compositeKeyClass = compositeKeyClass;
+  public int getKeyIndex() {
+    return columnMappings.getKeyIndex();
   }
 
-  public int getKeyIndex() {
-    return keyIndex;
+  public ColumnMapping getKeyColumnMapping() {
+    return columnMappings.getKeyMapping();
   }
 
-  public void setKeyIndex(int keyIndex) {
-    this.keyIndex = keyIndex;
+  public ColumnMappings getColumnMappings() {
+    return columnMappings;
   }
 
-  public List<ColumnMapping> getColumnMapping() {
-    return columnMapping;
+  public HBaseKeyFactory getKeyFactory() {
+    return keyFactory;
   }
 
-  public ColumnMapping getKeyColumnMapping() {
-      return columnMapping.get(keyIndex);
+  public TypeInfo getTypeForName(String columnName) {
+    List<String> columnNames = serdeParams.getColumnNames();
+    List<TypeInfo> columnTypes = serdeParams.getColumnTypes();
+    for (int i = 0; i < columnNames.size(); i++) {
+      if (columnName.equals(columnNames.get(i))) {
+        return columnTypes.get(i);
+      }
     }
+    throw new IllegalArgumentException("Invalid column name " + columnName);
+  }
 
-  public boolean[] getNeedsEscape() {
-    return serdeParams.getNeedsEscape();
+  public String toString() {
+    return "[" + columnMappingString + ":" + getColumnNames() + ":" + getColumnTypes() + "]";
   }
 }

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Tue May 13 18:23:14 2014
@@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.mapreduce
 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
 import org.apache.hadoop.hive.metastore.HiveMetaHook;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -155,7 +155,7 @@ public class HBaseStorageHandler extends
       Map<String, String> serdeParam = tbl.getSd().getSerdeInfo().getParameters();
       String hbaseColumnsMapping = serdeParam.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
 
-      List<ColumnMapping> columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, true);
+      ColumnMappings columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping);
 
       HTableDescriptor tableDesc;
 
@@ -166,7 +166,7 @@ public class HBaseStorageHandler extends
           tableDesc = new HTableDescriptor(tableName);
           Set<String> uniqueColumnFamilies = new HashSet<String>();
 
-          for (ColumnMapping colMap : columnsMapping) {
+          for (ColumnMapping colMap : columnMappings) {
             if (!colMap.hbaseRowKey) {
               uniqueColumnFamilies.add(colMap.familyName);
             }
@@ -192,8 +192,7 @@ public class HBaseStorageHandler extends
         // make sure the schema mapping is right
         tableDesc = getHBaseAdmin().getTableDescriptor(Bytes.toBytes(tableName));
 
-        for (int i = 0; i < columnsMapping.size(); i++) {
-          ColumnMapping colMap = columnsMapping.get(i);
+        for (ColumnMapping colMap : columnMappings) {
 
           if (colMap.hbaseRowKey) {
             continue;
@@ -378,6 +377,7 @@ public class HBaseStorageHandler extends
   @Override
   public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
     try {
+      HBaseSerDe.configureJobConf(tableDesc, jobConf);
       /*
        * HIVE-6356
        * The following code change is only needed for hbase-0.96.0 due to HBASE-9165, and
@@ -392,7 +392,7 @@ public class HBaseStorageHandler extends
       TableMapReduceUtil.addDependencyJars(copy);
       merged.addAll(copy.getConfiguration().getStringCollection("tmpjars"));
       jobConf.set("tmpjars", StringUtils.arrayToString(merged.toArray(new String[0])));
-    } catch (IOException e) {
+    } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }
@@ -403,22 +403,26 @@ public class HBaseStorageHandler extends
     Deserializer deserializer,
     ExprNodeDesc predicate)
   {
+    HBaseKeyFactory keyFactory = ((HBaseSerDe) deserializer).getKeyFactory();
+    return keyFactory.decomposePredicate(jobConf, deserializer, predicate);
+  }
+
+  public static DecomposedPredicate decomposePredicate(
+      JobConf jobConf,
+      HBaseSerDe hBaseSerDe,
+      ExprNodeDesc predicate) {
     String columnNameProperty = jobConf.get(
-      org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS);
+        org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS);
     List<String> columnNames =
-      Arrays.asList(columnNameProperty.split(","));
+        Arrays.asList(columnNameProperty.split(","));
 
-    HBaseSerDe hbaseSerde = (HBaseSerDe) deserializer;
-    int keyColPos = hbaseSerde.getKeyColumnOffset();
-    String keyColType = jobConf.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES).
-        split(",")[keyColPos];
-    IndexPredicateAnalyzer analyzer =
-      HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(columnNames.get(keyColPos), keyColType,
-        hbaseSerde.getStorageFormatOfCol(keyColPos).get(0));
+    ColumnMapping keyMapping = hBaseSerDe.getHBaseSerdeParam().getKeyColumnMapping();
+    IndexPredicateAnalyzer analyzer = HiveHBaseTableInputFormat.newIndexPredicateAnalyzer(
+        keyMapping.columnName, keyMapping.columnType, keyMapping.binaryStorage.get(0));
     List<IndexSearchCondition> searchConditions =
-      new ArrayList<IndexSearchCondition>();
+        new ArrayList<IndexSearchCondition>();
     ExprNodeGenericFuncDesc residualPredicate =
-      (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions);
+        (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions);
     int scSize = searchConditions.size();
     if (scSize < 1 || 2 < scSize) {
       // Either there was nothing which could be pushed down (size = 0),

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Tue May 13 18:23:14 2014
@@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.mapred.Ta
 import org.apache.hadoop.hbase.mapreduce.TableInputFormatBase;
 import org.apache.hadoop.hbase.mapreduce.TableSplit;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
 import org.apache.hadoop.hive.ql.exec.ExprNodeConstantEvaluator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer;
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.serde2.io.
 import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.FloatWritable;
@@ -93,15 +94,15 @@ public class HiveHBaseTableInputFormat e
     String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING);
     boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true);
     List<Integer> readColIDs = ColumnProjectionUtils.getReadColumnIDs(jobConf);
-    List<ColumnMapping> columnsMapping = null;
+    ColumnMappings columnMappings;
 
     try {
-      columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching);
+      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping, doColumnRegexMatching);
     } catch (SerDeException e) {
       throw new IOException(e);
     }
 
-    if (columnsMapping.size() < readColIDs.size()) {
+    if (columnMappings.size() < readColIDs.size()) {
       throw new IOException("Cannot read more columns than the given table contains.");
     }
 
@@ -113,8 +114,9 @@ public class HiveHBaseTableInputFormat e
     List<String> addedFamilies = new ArrayList<String>();
 
     if (!readAllColumns) {
+      ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping();
       for (int i : readColIDs) {
-        ColumnMapping colMap = columnsMapping.get(i);
+        ColumnMapping colMap = columnsMapping[i];
         if (colMap.hbaseRowKey) {
           continue;
         }
@@ -139,8 +141,7 @@ public class HiveHBaseTableInputFormat e
     // to the HBase scan so that we can retrieve all of the row keys and return them as the Hive
     // tables column projection.
     if (empty) {
-      for (int i = 0; i < columnsMapping.size(); i++) {
-        ColumnMapping colMap = columnsMapping.get(i);
+      for (ColumnMapping colMap: columnMappings) {
         if (colMap.hbaseRowKey) {
           continue;
         }
@@ -256,6 +257,18 @@ public class HiveHBaseTableInputFormat e
     // TODO: assert iKey is HBaseSerDe#HBASE_KEY_COL
 
     Scan scan = new Scan();
+    String filterObjectSerialized = jobConf.get(TableScanDesc.FILTER_OBJECT_CONF_STR);
+    if (filterObjectSerialized != null) {
+      HBaseScanRange range = Utilities.deserializeObject(filterObjectSerialized,
+          HBaseScanRange.class);
+      try {
+        range.setup(scan, jobConf);
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+      return scan;
+    }
+
     String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
     if (filterExprSerialized == null) {
       return scan;
@@ -324,6 +337,10 @@ public class HiveHBaseTableInputFormat e
     }
     scan.setStartRow(startRow);
     scan.setStopRow(stopRow);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(Bytes.toStringBinary(startRow) + " ~ " + Bytes.toStringBinary(stopRow));
+    }
     return scan;
   }
 
@@ -374,6 +391,12 @@ public class HiveHBaseTableInputFormat e
     System.arraycopy(current, 0, next, 0, current.length);
     return next;
   }
+
+  static IndexPredicateAnalyzer newIndexPredicateAnalyzer(
+      String keyColumnName, TypeInfo keyColType, boolean isKeyBinary) {
+    return newIndexPredicateAnalyzer(keyColumnName, keyColType.getTypeName(), isKeyBinary);
+  }
+
   /**
    * Instantiates a new predicate analyzer suitable for
    * determining how to push a filter down into the HBase scan,
@@ -423,20 +446,15 @@ public class HiveHBaseTableInputFormat e
       throw new IOException("hbase.columns.mapping required for HBase Table.");
     }
 
-    List<ColumnMapping> columnsMapping = null;
+    ColumnMappings columnMappings = null;
     try {
-      columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping,doColumnRegexMatching);
+      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping,doColumnRegexMatching);
     } catch (SerDeException e) {
       throw new IOException(e);
     }
 
-    int iKey;
-
-    try {
-      iKey = HBaseSerDe.getRowKeyColumnOffset(columnsMapping);
-    } catch (SerDeException e) {
-      throw new IOException(e);
-    }
+    int iKey = columnMappings.getKeyIndex();
+    ColumnMapping keyMapping = columnMappings.getKeyMapping();
 
     // Take filter pushdown into account while calculating splits; this
     // allows us to prune off regions immediately.  Note that although
@@ -445,7 +463,7 @@ public class HiveHBaseTableInputFormat e
     // definition into account and excludes regions which don't satisfy
     // the start/stop row conditions (HBASE-1829).
     Scan scan = createFilterScan(jobConf, iKey,
-        getStorageFormatOfKey(columnsMapping.get(iKey).mappingSpec,
+        getStorageFormatOfKey(keyMapping.mappingSpec,
             jobConf.get(HBaseSerDe.HBASE_TABLE_DEFAULT_STORAGE_TYPE, "string")));
 
 
@@ -454,8 +472,7 @@ public class HiveHBaseTableInputFormat e
 
     // REVIEW:  are we supposed to be applying the getReadColumnIDs
     // same as in getRecordReader?
-    for (int i = 0; i <columnsMapping.size(); i++) {
-      ColumnMapping colMap = columnsMapping.get(i);
+    for (ColumnMapping colMap : columnMappings) {
       if (colMap.hbaseRowKey) {
         continue;
       }

Modified: hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java (original)
+++ hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java Tue May 13 18:23:14 2014
@@ -20,19 +20,17 @@ package org.apache.hadoop.hive.hbase;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
+import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
-import org.apache.hadoop.hive.serde2.lazy.LazyObject;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
 import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
 /**
  * LazyObject for storing an HBase row.  The field of an HBase row can be
@@ -44,73 +42,47 @@ public class LazyHBaseRow extends LazySt
    * The HBase columns mapping of the row.
    */
   private Result result;
-  private List<ColumnMapping> columnsMapping;
-  private Object compositeKeyObj;
+  private ColumnMapping[] columnsMapping;
   private ArrayList<Object> cachedList;
 
-  /**
-   * Construct a LazyHBaseRow object with the ObjectInspector.
-   */
+  private final int iKey;
+  private final HBaseKeyFactory keyFactory;
+
   public LazyHBaseRow(LazySimpleStructObjectInspector oi) {
-    super(oi);
+    this(oi, -1, null);
   }
 
   /**
-   * Set the HBase row data(a Result writable) for this LazyStruct.
-   * @see LazyHBaseRow#init(Result)
+   * Construct a LazyHBaseRow object with the ObjectInspector.
    */
-  public void init(Result r, List<ColumnMapping> columnsMapping) {
-    init(r, columnsMapping, null);
+  public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory) {
+    super(oi);
+    this.iKey = iKey;
+    this.keyFactory = keyFactory;
   }
 
   /**
    * Set the HBase row data(a Result writable) for this LazyStruct.
-   *
-   * @see LazyHBaseRow#init(Result)
-   *
-   * @param compositeKeyClass
-   *          custom implementation to interpret the composite key
+   * @see LazyHBaseRow#init(org.apache.hadoop.hbase.client.Result)
    */
-  public void init(Result r, List<ColumnMapping> columnsMapping, Object compositeKeyObj) {
-
-    result = r;
-    this.columnsMapping = columnsMapping;
-    this.compositeKeyObj = compositeKeyObj;
+  public void init(Result r, ColumnMappings columnsMappings) {
+    this.result = r;
+    this.columnsMapping = columnsMappings.getColumnsMapping();
     setParsed(false);
   }
 
-  /**
-   * Parse the Result and fill each field.
-   * @see LazyStruct#parse()
-   */
-  private void parse() {
-
-    if (getFields() == null) {
-      List<? extends StructField> fieldRefs =
-        ((StructObjectInspector)getInspector()).getAllStructFieldRefs();
-      LazyObject<? extends ObjectInspector> [] fields = new LazyObject<?>[fieldRefs.size()];
-
-      for (int i = 0; i < fields.length; i++) {
-        ColumnMapping colMap = columnsMapping.get(i);
-
-        if (colMap.qualifierName == null && !colMap.hbaseRowKey) {
-          // a column family
-          fields[i] = new LazyHBaseCellMap(
-              (LazyMapObjectInspector) fieldRefs.get(i).getFieldObjectInspector());
-          continue;
-        }
-
-        fields[i] = LazyFactory.createLazyObject(
-            fieldRefs.get(i).getFieldObjectInspector(),
-            colMap.binaryStorage.get(0));
-      }
-
-      setFields(fields);
-      setFieldInited(new boolean[fields.length]);
+  @Override
+  protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException {
+    if (fieldID == iKey) {
+      return keyFactory.createKey(fieldRef.getFieldObjectInspector());
+    }
+    ColumnMapping colMap = columnsMapping[fieldID];
+    if (colMap.qualifierName == null && !colMap.hbaseRowKey) {
+      // a column family
+      return new LazyHBaseCellMap((LazyMapObjectInspector) fieldRef.getFieldObjectInspector());
     }
-
-    Arrays.fill(getFieldInited(), false);
-    setParsed(true);
+    return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector(),
+        colMap.binaryStorage.get(0));
   }
 
   /**
@@ -127,16 +99,17 @@ public class LazyHBaseRow extends LazySt
    */
   @Override
   public Object getField(int fieldID) {
-    if (!getParsed()) {
-      parse();
-    }
-
-    Object value = uncheckedGetField(fieldID);
+    initFields();
+    return uncheckedGetField(fieldID);
+  }
 
-    if (columnsMapping.get(fieldID).hbaseRowKey && compositeKeyObj != null) {
-      return compositeKeyObj;
-    } else {
-      return value;
+  private void initFields() {
+    if (getFields() == null) {
+      initLazyFields(oi.getAllStructFieldRefs());
+    }
+    if (!getParsed()) {
+      Arrays.fill(getFieldInited(), false);
+      setParsed(true);
     }
   }
 
@@ -149,12 +122,12 @@ public class LazyHBaseRow extends LazySt
    */
   private Object uncheckedGetField(int fieldID) {
 
-    LazyObject<?> [] fields = getFields();
+    LazyObjectBase[] fields = getFields();
     boolean [] fieldsInited = getFieldInited();
 
     if (!fieldsInited[fieldID]) {
       ByteArrayRef ref = null;
-      ColumnMapping colMap = columnsMapping.get(fieldID);
+      ColumnMapping colMap = columnsMapping[fieldID];
 
       if (colMap.hbaseRowKey) {
         ref = new ByteArrayRef();
@@ -182,12 +155,6 @@ public class LazyHBaseRow extends LazySt
 
       if (ref != null) {
         fields[fieldID].init(ref, 0, ref.getData().length);
-
-        // if it was a row key and we have been provided a custom composite key class, initialize it
-        // with the bytes for the row key
-        if (colMap.hbaseRowKey && compositeKeyObj != null) {
-          ((LazyStruct) compositeKeyObj).init(ref, 0, ref.getData().length);
-        }
       }
     }
 
@@ -203,9 +170,7 @@ public class LazyHBaseRow extends LazySt
    */
   @Override
   public ArrayList<Object> getFieldsAsList() {
-    if (!getParsed()) {
-      parse();
-    }
+    initFields();
     if (cachedList == null) {
       cachedList = new ArrayList<Object>();
     } else {

Modified: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestCompositeKey.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestCompositeKey.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestCompositeKey.java (original)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestCompositeKey.java Tue May 13 18:23:14 2014
@@ -18,13 +18,13 @@
 
 package org.apache.hadoop.hive.hbase;
 
-import java.util.Properties;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
 
+import java.util.Properties;
+
 public class HBaseTestCompositeKey extends HBaseCompositeKey {
 
   byte[] bytes;

Added: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java (added)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory.java Tue May 13 18:23:14 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class TestHBaseKeyFactory extends DefaultHBaseKeyFactory {
+
+  private static final String DELIMITER_PATTERN = "\\$\\$";
+  private static final byte[] DELIMITER_BINARY = "$$".getBytes();
+
+  @Override
+  public ObjectInspector createKeyObjectInspector(TypeInfo type) {
+    return new SlashSeparatedOI((StructTypeInfo)type);
+  }
+
+  @Override
+  public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException {
+    return new DoubleDollarSeparated();
+  }
+
+  private final ByteStream.Output output = new ByteStream.Output();
+
+  @Override
+  public byte[] serializeKey(Object object, StructField field) throws IOException {
+    ObjectInspector inspector = field.getFieldObjectInspector();
+    if (inspector.getCategory() != ObjectInspector.Category.STRUCT) {
+      throw new IllegalStateException("invalid type value " + inspector.getTypeName());
+    }
+    output.reset();
+    for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) {
+      if (output.getCount() > 0) {
+        output.write(DELIMITER_BINARY);
+      }
+      output.write(String.valueOf(element).getBytes());
+    }
+    return output.getCount() > 0 ? output.toByteArray() : null;
+  }
+
+  private static class DoubleDollarSeparated implements LazyObjectBase {
+
+    private Object[] fields;
+
+    @Override
+    public void init(ByteArrayRef bytes, int start, int length) {
+      fields = new String(bytes.getData(), start, length).split(DELIMITER_PATTERN);
+    }
+
+    @Override
+    public Object getObject() {
+      return this;
+    }
+  }
+
+  private static class SlashSeparatedOI extends BaseStructObjectInspector {
+
+    private int length;
+
+    private SlashSeparatedOI(StructTypeInfo type) {
+      List<String> names = type.getAllStructFieldNames();
+      List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+      for (int i = 0; i < names.size(); i++) {
+        ois.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+      }
+      init(names, ois, null);
+    }
+
+    @Override
+    public Object getStructFieldData(Object data, StructField fieldRef) {
+      return ((DoubleDollarSeparated)data).fields[((MyField)fieldRef).getFieldID()];
+    }
+
+    @Override
+    public List<Object> getStructFieldsDataAsList(Object data) {
+      return Arrays.asList(((DoubleDollarSeparated)data).fields);
+    }
+  }
+}

Added: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory2.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory2.java?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory2.java (added)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseKeyFactory2.java Tue May 13 18:23:14 2014
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.hbase;
+
+import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer;
+import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.serde2.BaseStructObjectInspector;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.mapred.JobConf;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TestHBaseKeyFactory2 extends AbstractHBaseKeyFactory {
+
+  private static final int FIXED_LENGTH = 10;
+
+  @Override
+  public ObjectInspector createKeyObjectInspector(TypeInfo type) {
+    return new StringArrayOI((StructTypeInfo)type);
+  }
+
+  @Override
+  public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException {
+    return new FixedLengthed(FIXED_LENGTH);
+  }
+
+  private final ByteStream.Output output = new ByteStream.Output();
+
+  @Override
+  public byte[] serializeKey(Object object, StructField field) throws IOException {
+    ObjectInspector inspector = field.getFieldObjectInspector();
+    if (inspector.getCategory() != ObjectInspector.Category.STRUCT) {
+      throw new IllegalStateException("invalid type value " + inspector.getTypeName());
+    }
+    output.reset();
+    for (Object element : ((StructObjectInspector)inspector).getStructFieldsDataAsList(object)) {
+      output.write(toBinary(String.valueOf(element).getBytes(), FIXED_LENGTH, false, false));
+    }
+    return output.getCount() > 0 ? output.toByteArray() : null;
+  }
+
+  private byte[] toBinary(String value, int max, boolean end, boolean nextBA) {
+    return toBinary(value.getBytes(), max, end, nextBA);
+  }
+
+  private byte[] toBinary(byte[] value, int max, boolean end, boolean nextBA) {
+    byte[] bytes = new byte[max + 1];
+    System.arraycopy(value, 0, bytes, 0, Math.min(value.length, max));
+    if (end) {
+      Arrays.fill(bytes, value.length, max, (byte)0xff);
+    }
+    if (nextBA) {
+      bytes[max] = 0x01;
+    }
+    return bytes;
+  }
+
+  @Override
+  public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
+      ExprNodeDesc predicate) {
+    String keyColName = keyMapping.columnName;
+
+    IndexPredicateAnalyzer analyzer = IndexPredicateAnalyzer.createAnalyzer(false);
+    analyzer.allowColumnName(keyColName);
+    analyzer.setAcceptsFields(true);
+
+    DecomposedPredicate decomposed = new DecomposedPredicate();
+
+    List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
+    decomposed.residualPredicate =
+        (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions);
+    if (!searchConditions.isEmpty()) {
+      decomposed.pushedPredicate = analyzer.translateSearchConditions(searchConditions);
+      try {
+        decomposed.pushedPredicateObject = setupFilter(keyColName, searchConditions);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return decomposed;
+  }
+
+  private HBaseScanRange setupFilter(String keyColName, List<IndexSearchCondition> conditions)
+      throws IOException {
+    Map<String, List<IndexSearchCondition>> fieldConds =
+        new HashMap<String, List<IndexSearchCondition>>();
+    for (IndexSearchCondition condition : conditions) {
+      assert keyColName.equals(condition.getColumnDesc().getColumn());
+      String fieldName = condition.getFields()[0];
+      List<IndexSearchCondition> fieldCond = fieldConds.get(fieldName);
+      if (fieldCond == null) {
+        fieldConds.put(fieldName, fieldCond = new ArrayList<IndexSearchCondition>());
+      }
+      fieldCond.add(condition);
+    }
+    HBaseScanRange range = new HBaseScanRange();
+
+    ByteArrayOutputStream startRow = new ByteArrayOutputStream();
+    ByteArrayOutputStream stopRow = new ByteArrayOutputStream();
+
+    StructTypeInfo type = (StructTypeInfo) keyMapping.columnType;
+    for (String name : type.getAllStructFieldNames()) {
+      List<IndexSearchCondition> fieldCond = fieldConds.get(name);
+      if (fieldCond == null || fieldCond.size() > 2) {
+        continue;
+      }
+      byte[] startElement = null;
+      byte[] stopElement = null;
+      for (IndexSearchCondition condition : fieldCond) {
+        if (condition.getConstantDesc().getValue() == null) {
+          continue;
+        }
+        String comparisonOp = condition.getComparisonOp();
+        String constantVal = String.valueOf(condition.getConstantDesc().getValue());
+
+        if (comparisonOp.endsWith("UDFOPEqual")) {
+          startElement = toBinary(constantVal, FIXED_LENGTH, false, false);
+          stopElement = toBinary(constantVal, FIXED_LENGTH, true, true);
+        } else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
+          startElement = toBinary(constantVal, FIXED_LENGTH, false, false);
+        } else if (comparisonOp.endsWith("UDFOPGreaterThan")) {
+          startElement = toBinary(constantVal, FIXED_LENGTH, false, true);
+        } else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) {
+          stopElement = toBinary(constantVal, FIXED_LENGTH, true, false);
+        } else if (comparisonOp.endsWith("UDFOPLessThan")) {
+          stopElement = toBinary(constantVal, FIXED_LENGTH, true, true);
+        } else {
+          throw new IOException(comparisonOp + " is not a supported comparison operator");
+        }
+      }
+      if (startRow != null) {
+        if (startElement != null) {
+          startRow.write(startElement);
+        } else {
+          if (startRow.size() > 0) {
+            range.setStartRow(startRow.toByteArray());
+          }
+          startRow = null;
+        }
+      }
+      if (stopRow != null) {
+        if (stopElement != null) {
+          stopRow.write(stopElement);
+        } else {
+          if (stopRow.size() > 0) {
+            range.setStopRow(stopRow.toByteArray());
+          }
+          stopRow = null;
+        }
+      }
+      if (startElement == null && stopElement == null) {
+        break;
+      }
+    }
+    if (startRow != null && startRow.size() > 0) {
+      range.setStartRow(startRow.toByteArray());
+    }
+    if (stopRow != null && stopRow.size() > 0) {
+      range.setStopRow(stopRow.toByteArray());
+    }
+    return range;
+  }
+
+  private static class FixedLengthed implements LazyObjectBase {
+
+    private final int fixedLength;
+    private final List<Object> fields = new ArrayList<Object>();
+
+    public FixedLengthed(int fixedLength) {
+      this.fixedLength = fixedLength;
+    }
+
+    @Override
+    public void init(ByteArrayRef bytes, int start, int length) {
+      fields.clear();
+      byte[] data = bytes.getData();
+      int rowStart = start;
+      int rowStop = rowStart + fixedLength;
+      for (; rowStart < length; rowStart = rowStop + 1, rowStop = rowStart + fixedLength) {
+        fields.add(new String(data, rowStart, rowStop - rowStart).trim());
+      }
+    }
+
+    @Override
+    public Object getObject() {
+      return this;
+    }
+  }
+
+  private static class StringArrayOI extends BaseStructObjectInspector {
+
+    private int length;
+
+    private StringArrayOI(StructTypeInfo type) {
+      List<String> names = type.getAllStructFieldNames();
+      List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+      for (int i = 0; i < names.size(); i++) {
+        ois.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+      }
+      init(names, ois, null);
+    }
+
+    @Override
+    public Object getStructFieldData(Object data, StructField fieldRef) {
+      return ((FixedLengthed)data).fields.get(((MyField)fieldRef).getFieldID());
+    }
+
+    @Override
+    public List<Object> getStructFieldsDataAsList(Object data) {
+      return ((FixedLengthed)data).fields;
+    }
+  }
+}

Modified: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java (original)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java Tue May 13 18:23:14 2014
@@ -28,7 +28,7 @@ import junit.framework.TestCase;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
+import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
@@ -463,17 +463,15 @@ public class TestLazyHBaseObject extends
     Text nullSequence = new Text("\\N");
 
     String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:c,cfb:d";
-    List<ColumnMapping> columnsMapping = null;
+    ColumnMappings columnMappings = null;
 
     try {
-      columnsMapping = parseColumnsMapping(hbaseColsMapping);
+      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
     } catch (SerDeException e) {
       fail(e.toString());
     }
 
-    for (int i = 0; i < columnsMapping.size(); i++) {
-      ColumnMapping colMap = columnsMapping.get(i);
-
+    for (ColumnMapping colMap : columnMappings) {
       if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
         colMap.binaryStorage.add(false);
         colMap.binaryStorage.add(false);
@@ -499,7 +497,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("hi")));
 
     Result r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':123,'b':['a','b','c'],"
@@ -513,7 +511,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=e:f=g")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
         ("{'key':'test-row','a':123,'b':null,"
@@ -529,7 +527,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
         ("{'key':'test-row','a':null,'b':['a'],"
@@ -543,7 +541,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':null,'b':['','a','',''],"
@@ -567,7 +565,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
@@ -587,18 +585,17 @@ public class TestLazyHBaseObject extends
     List<String> fieldNames = Arrays.asList(
       new String[]{"key", "a", "b", "c", "d"});
     Text nullSequence = new Text("\\N");
-    List<ColumnMapping> columnsMapping = null;
     String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:,cfc:d";
 
+    ColumnMappings columnMappings = null;
+
     try {
-      columnsMapping = parseColumnsMapping(hbaseColsMapping);
+      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
     } catch (SerDeException e) {
       fail(e.toString());
     }
 
-    for (int i = 0; i < columnsMapping.size(); i++) {
-      ColumnMapping colMap = columnsMapping.get(i);
-
+    for (ColumnMapping colMap : columnMappings) {
       if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
         colMap.binaryStorage.add(false);
         colMap.binaryStorage.add(false);
@@ -627,7 +624,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("hi")));
 
     Result r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':123,'b':['a','b','c'],"
@@ -643,7 +640,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':123,'b':null,"
@@ -659,7 +656,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':null,'b':['a'],"
@@ -673,7 +670,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       ("{'key':'test-row','a':null,'b':['','a','',''],"
@@ -689,7 +686,7 @@ public class TestLazyHBaseObject extends
         Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("")));
 
     r = new Result(kvs);
-    o.init(r, columnsMapping);
+    o.init(r, columnMappings);
 
     assertEquals(
       "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
@@ -713,16 +710,17 @@ public class TestLazyHBaseObject extends
     String hbaseColumnsMapping = ":key#str,cf-int:cq-int#bin,cf-byte:cq-byte#bin,"
       + "cf-short:cq-short#bin,cf-long:cq-long#bin,cf-float:cq-float#bin,cf-double:cq-double#bin,"
       + "cf-string:cq-string#str,cf-bool:cq-bool#bin";
-    List<ColumnMapping> columnsMapping = null;
+    ColumnMappings columnMappings = null;
 
     try {
-      columnsMapping = parseColumnsMapping(hbaseColumnsMapping);
-    } catch (SerDeException sde) {
-      fail(sde.toString());
+      columnMappings = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping);
+    } catch (SerDeException e) {
+      fail(e.toString());
     }
 
-    for (int i = 0; i < columnsMapping.size(); i++) {
-      ColumnMapping colMap = columnsMapping.get(i);
+    ColumnMapping[] columnsMapping = columnMappings.getColumnsMapping();
+    for (int i = 0; i < columnsMapping.length; i++) {
+      ColumnMapping colMap = columnsMapping[i];
 
       if (i == 0 || i == 7) {
         colMap.binaryStorage.add(false);
@@ -741,7 +739,7 @@ public class TestLazyHBaseObject extends
     List<KeyValue> kvs = new ArrayList<KeyValue>();
     byte [] value;
 
-    for (int i = 1; i < columnsMapping.size(); i++) {
+    for (int i = 1; i < columnsMapping.length; i++) {
 
       switch (i) {
 
@@ -781,13 +779,13 @@ public class TestLazyHBaseObject extends
         throw new RuntimeException("Not expected: " + i);
       }
 
-      ColumnMapping colMap = columnsMapping.get(i);
+      ColumnMapping colMap = columnsMapping[i];
       kvs.add(new KeyValue(rowKey, colMap.familyNameBytes, colMap.qualifierNameBytes, value));
     }
 
     Collections.sort(kvs, KeyValue.COMPARATOR);
     Result result = new Result(kvs);
-    o.init(result, columnsMapping);
+    o.init(result, columnMappings);
     List<? extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();
 
 
@@ -850,19 +848,4 @@ public class TestLazyHBaseObject extends
       }
     }
   }
-
-  /**
-   * Parses the HBase columns mapping specifier to identify the column families, qualifiers
-   * and also caches the byte arrays corresponding to them. One of the Hive table
-   * columns maps to the HBase row key, by default the first column.
-   *
-   * @param columnsMappingSpec string hbase.columns.mapping specified when creating table
-   * @return List<ColumnMapping> which contains the column mapping information by position
-   * @throws SerDeException
-   */
-  public static List<ColumnMapping> parseColumnsMapping(String columnsMappingSpec)
-      throws SerDeException {
-    return HBaseSerDe.parseColumnsMapping(columnsMappingSpec, true);
-  }
-
 }

Added: hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key.q Tue May 13 18:23:14 2014
@@ -0,0 +1,18 @@
+CREATE TABLE hbase_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory");
+
+CREATE EXTERNAL TABLE hbase_ck_2(key string, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string");
+
+from src tablesample (1 rows)
+insert into table hbase_ck_1 select struct('1000','2000','3000'),'value';
+
+select * from hbase_ck_1;
+select * from hbase_ck_2;

Added: hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/positive/hbase_custom_key2.q Tue May 13 18:23:14 2014
@@ -0,0 +1,35 @@
+CREATE TABLE hbase_ck_4(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom2",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2");
+
+from src tablesample (5 rows)
+insert into table hbase_ck_4 select
+struct(
+  cast(key as string),
+  cast(cast(key + 1000 as int) as string),
+  cast(cast(key + 2000 as int) as string)),
+value;
+
+set hive.fetch.task.conversion=more;
+
+-- 165,238,27,311,86
+select * from hbase_ck_4;
+
+-- 238
+explain
+select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238';
+select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238';
+
+-- 165,238
+explain
+select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27';
+select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27';
+
+-- 238,311
+explain
+select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238';
+select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238';
+

Added: hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out (added)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key.q.out Tue May 13 18:23:14 2014
@@ -0,0 +1,60 @@
+PREHOOK: query: CREATE TABLE hbase_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: CREATE TABLE hbase_ck_1(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_ck_1
+PREHOOK: query: CREATE EXTERNAL TABLE hbase_ck_2(key string, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: CREATE EXTERNAL TABLE hbase_ck_2(key string, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom",
+    "hbase.columns.mapping" = ":key,cf:string")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_ck_2
+PREHOOK: query: from src tablesample (1 rows)
+insert into table hbase_ck_1 select struct('1000','2000','3000'),'value'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_ck_1
+POSTHOOK: query: from src tablesample (1 rows)
+insert into table hbase_ck_1 select struct('1000','2000','3000'),'value'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_ck_1
+PREHOOK: query: select * from hbase_ck_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ck_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_1
+#### A masked pattern was here ####
+{"col1":"1000","col2":"2000","col3":"3000"}	value
+PREHOOK: query: select * from hbase_ck_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ck_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_2
+#### A masked pattern was here ####
+1000$$2000$$3000	value

Added: hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out?rev=1594315&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out (added)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_custom_key2.q.out Tue May 13 18:23:14 2014
@@ -0,0 +1,168 @@
+PREHOOK: query: CREATE TABLE hbase_ck_4(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom2",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: CREATE TABLE hbase_ck_4(key struct<col1:string,col2:string,col3:string>, value string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+    "hbase.table.name" = "hbase_custom2",
+    "hbase.columns.mapping" = ":key,cf:string",
+    "hbase.composite.key.factory"="org.apache.hadoop.hive.hbase.TestHBaseKeyFactory2")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_ck_4
+PREHOOK: query: from src tablesample (5 rows)
+insert into table hbase_ck_4 select
+struct(
+  cast(key as string),
+  cast(cast(key + 1000 as int) as string),
+  cast(cast(key + 2000 as int) as string)),
+value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_ck_4
+POSTHOOK: query: from src tablesample (5 rows)
+insert into table hbase_ck_4 select
+struct(
+  cast(key as string),
+  cast(cast(key + 1000 as int) as string),
+  cast(cast(key + 2000 as int) as string)),
+value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_ck_4
+PREHOOK: query: -- 165,238,27,311,86
+select * from hbase_ck_4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+POSTHOOK: query: -- 165,238,27,311,86
+select * from hbase_ck_4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+{"col1":"165","col2":"1165","col3":"2165"}	val_165
+{"col1":"238","col2":"1238","col3":"2238"}	val_238
+{"col1":"27","col2":"1027","col3":"2027"}	val_27
+{"col1":"311","col2":"1311","col3":"2311"}	val_311
+{"col1":"86","col2":"1086","col3":"2086"}	val_86
+PREHOOK: query: -- 238
+explain
+select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 238
+explain
+select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: hbase_ck_4
+          filterExpr: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean)
+          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          Filter Operator
+            predicate: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean)
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Select Operator
+              expressions: key (type: struct<col1:string,col2:string,col3:string>), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              ListSink
+
+PREHOOK: query: select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ck_4 where key.col1 = '238' AND key.col2 = '1238'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+{"col1":"238","col2":"1238","col3":"2238"}	val_238
+PREHOOK: query: -- 165,238
+explain
+select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 165,238
+explain
+select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: hbase_ck_4
+          filterExpr: ((key.col1 >= '165') and (key.col1 < '27')) (type: boolean)
+          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          Filter Operator
+            predicate: ((key.col1 >= '165') and (key.col1 < '27')) (type: boolean)
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Select Operator
+              expressions: key (type: struct<col1:string,col2:string,col3:string>), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              ListSink
+
+PREHOOK: query: select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ck_4 where key.col1 >= '165' AND key.col1 < '27'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+{"col1":"165","col2":"1165","col3":"2165"}	val_165
+{"col1":"238","col2":"1238","col3":"2238"}	val_238
+PREHOOK: query: -- 238,311
+explain
+select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- 238,311
+explain
+select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: hbase_ck_4
+          filterExpr: ((key.col1 > '100') and (key.col2 >= '1238')) (type: boolean)
+          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          Filter Operator
+            predicate: ((key.col1 > '100') and (key.col2 >= '1238')) (type: boolean)
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Select Operator
+              expressions: key (type: struct<col1:string,col2:string,col3:string>), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              ListSink
+
+PREHOOK: query: select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_ck_4 where key.col1 > '100' AND key.col2 >= '1238'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_ck_4
+#### A masked pattern was here ####
+{"col1":"238","col2":"1238","col3":"2238"}	val_238
+{"col1":"311","col2":"1311","col3":"2311"}	val_311

Modified: hive/trunk/itests/util/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/pom.xml?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/itests/util/pom.xml (original)
+++ hive/trunk/itests/util/pom.xml Tue May 13 18:23:14 2014
@@ -46,6 +46,12 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
+      <artifactId>hive-hbase-handler</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
       <artifactId>hive-metastore</artifactId>
       <version>${project.version}</version>
     </dependency>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue May 13 18:23:14 2014
@@ -662,11 +662,7 @@ public final class Utilities {
    * @return Bytes.
    */
   public static byte[] serializeExpressionToKryo(ExprNodeGenericFuncDesc expr) {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    Output output = new Output(baos);
-    runtimeSerializationKryo.get().writeObject(output, expr);
-    output.close();
-    return baos.toByteArray();
+    return serializeObjectToKryo(expr);
   }
 
   /**
@@ -675,11 +671,7 @@ public final class Utilities {
    * @return Expression; null if deserialization succeeded, but the result type is incorrect.
    */
   public static ExprNodeGenericFuncDesc deserializeExpressionFromKryo(byte[] bytes) {
-    Input inp = new Input(new ByteArrayInputStream(bytes));
-    ExprNodeGenericFuncDesc func = runtimeSerializationKryo.get().
-      readObject(inp,ExprNodeGenericFuncDesc.class);
-    inp.close();
-    return func;
+    return deserializeObjectFromKryo(bytes, ExprNodeGenericFuncDesc.class);
   }
 
   public static String serializeExpression(ExprNodeGenericFuncDesc expr) {
@@ -700,6 +692,37 @@ public final class Utilities {
     return deserializeExpressionFromKryo(bytes);
   }
 
+  private static byte[] serializeObjectToKryo(Serializable object) {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    Output output = new Output(baos);
+    runtimeSerializationKryo.get().writeObject(output, object);
+    output.close();
+    return baos.toByteArray();
+  }
+
+  private static <T extends Serializable> T deserializeObjectFromKryo(byte[] bytes, Class<T> clazz) {
+    Input inp = new Input(new ByteArrayInputStream(bytes));
+    T func = runtimeSerializationKryo.get().readObject(inp, clazz);
+    inp.close();
+    return func;
+  }
+
+  public static String serializeObject(Serializable expr) {
+    try {
+      return new String(Base64.encodeBase64(serializeObjectToKryo(expr)), "UTF-8");
+    } catch (UnsupportedEncodingException ex) {
+      throw new RuntimeException("UTF-8 support required", ex);
+    }
+  }
+
+  public static <T extends Serializable> T deserializeObject(String s, Class<T> clazz) {
+    try {
+      return deserializeObjectFromKryo(Base64.decodeBase64(s.getBytes("UTF-8")), clazz);
+    } catch (UnsupportedEncodingException ex) {
+      throw new RuntimeException("UTF-8 support required", ex);
+    }
+  }
+
   public static class CollectionPersistenceDelegate extends DefaultPersistenceDelegate {
     @Override
     protected Expression instantiate(Object oldInstance, Encoder out) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java Tue May 13 18:23:14 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
@@ -57,12 +58,19 @@ public class IndexPredicateAnalyzer {
 
   private final Set<String> udfNames;
   private final Set<String> allowedColumnNames;
+  private FieldValidator fieldValidator;
+
+  private boolean acceptsFields;
 
   public IndexPredicateAnalyzer() {
     udfNames = new HashSet<String>();
     allowedColumnNames = new HashSet<String>();
   }
 
+  public void setFieldValidator(FieldValidator fieldValidator) {
+    this.fieldValidator = fieldValidator;
+  }
+
   /**
    * Registers a comparison operator as one which can be satisfied
    * by an index search.  Unless this is called, analyzePredicate
@@ -175,11 +183,19 @@ public class IndexPredicateAnalyzer {
     ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
     ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
     ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
-    if (extracted == null) {
+    if (extracted == null || (extracted.length > 2 && !acceptsFields)) {
       return expr;
     }
-    if (extracted.length > 2) {
+
+    ExprNodeColumnDesc columnDesc;
+    ExprNodeConstantDesc constantDesc;
+    if (extracted[0] instanceof ExprNodeConstantDesc) {
       genericUDF = genericUDF.flip();
+      columnDesc = (ExprNodeColumnDesc) extracted[1];
+      constantDesc = (ExprNodeConstantDesc) extracted[0];
+    } else {
+      columnDesc = (ExprNodeColumnDesc) extracted[0];
+      constantDesc = (ExprNodeConstantDesc) extracted[1];
     }
 
     String udfName = genericUDF.getUdfName();
@@ -187,22 +203,34 @@ public class IndexPredicateAnalyzer {
       return expr;
     }
 
-    ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) extracted[0];
-    ExprNodeConstantDesc constantDesc = (ExprNodeConstantDesc) extracted[1];
     if (!allowedColumnNames.contains(columnDesc.getColumn())) {
       return expr;
     }
 
+    String[] fields = null;
+    if (extracted.length > 2) {
+      ExprNodeFieldDesc fieldDesc = (ExprNodeFieldDesc) extracted[2];
+      if (!isValidField(fieldDesc)) {
+        return expr;
+      }
+      fields = ExprNodeDescUtils.extractFields(fieldDesc);
+    }
+
     searchConditions.add(
       new IndexSearchCondition(
         columnDesc,
         udfName,
         constantDesc,
-        expr));
+        expr,
+        fields));
 
     // we converted the expression to a search condition, so
     // remove it from the residual predicate
-    return null;
+    return fields == null ? null : expr;
+  }
+
+  private boolean isValidField(ExprNodeFieldDesc field) {
+    return fieldValidator == null || fieldValidator.validate(field);
   }
 
   /**
@@ -232,4 +260,27 @@ public class IndexPredicateAnalyzer {
     }
     return expr;
   }
+
+  public void setAcceptsFields(boolean acceptsFields) {
+    this.acceptsFields = acceptsFields;
+  }
+
+  public static interface FieldValidator {
+    boolean validate(ExprNodeFieldDesc exprNodeDesc);
+  }
+
+  public static IndexPredicateAnalyzer createAnalyzer(boolean equalOnly) {
+
+    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
+    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual");
+    if (equalOnly) {
+      return analyzer;
+    }
+    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan");
+    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan");
+    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan");
+    analyzer.addComparisonOp("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan");
+
+    return analyzer;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexSearchCondition.java Tue May 13 18:23:14 2014
@@ -33,6 +33,16 @@ public class IndexSearchCondition
   private ExprNodeConstantDesc constantDesc;
   private ExprNodeGenericFuncDesc comparisonExpr;
 
+  private String[] fields;
+
+  public IndexSearchCondition(
+      ExprNodeColumnDesc columnDesc,
+      String comparisonOp,
+      ExprNodeConstantDesc constantDesc,
+      ExprNodeGenericFuncDesc comparisonExpr) {
+    this(columnDesc, comparisonOp, constantDesc, comparisonExpr, null);
+  }
+
   /**
    * Constructs a search condition, which takes the form
    * <pre>column-ref comparison-op constant-value</pre>.
@@ -50,12 +60,14 @@ public class IndexSearchCondition
     ExprNodeColumnDesc columnDesc,
     String comparisonOp,
     ExprNodeConstantDesc constantDesc,
-    ExprNodeGenericFuncDesc comparisonExpr) {
+    ExprNodeGenericFuncDesc comparisonExpr,
+    String[] fields) {
 
     this.columnDesc = columnDesc;
     this.comparisonOp = comparisonOp;
     this.constantDesc = constantDesc;
     this.comparisonExpr = comparisonExpr;
+    this.fields = fields;
   }
 
   public void setColumnDesc(ExprNodeColumnDesc columnDesc) {
@@ -90,6 +102,10 @@ public class IndexSearchCondition
     return comparisonExpr;
   }
 
+  public String[] getFields() {
+    return fields;
+  }
+
   @Override
   public String toString() {
     return comparisonExpr.getExprString();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Tue May 13 18:23:14 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.io;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -411,6 +412,13 @@ public class HiveInputFormat<K extends W
       return;
     }
 
+    Serializable filterObject = scanDesc.getFilterObject();
+    if (filterObject != null) {
+      jobConf.set(
+          TableScanDesc.FILTER_OBJECT_CONF_STR,
+          Utilities.serializeObject(filterObject));
+    }
+
     String filterText = filterExpr.getExprString();
     String filterExprSerialized = Utilities.serializeExpression(filterExpr);
     if (LOG.isDebugEnabled()) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java Tue May 13 18:23:14 2014
@@ -166,6 +166,11 @@ final public class OrcStruct implements 
     }
 
     @Override
+    public int getFieldID() {
+      return offset;
+    }
+
+    @Override
     public String getFieldComment() {
       return null;
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java Tue May 13 18:23:14 2014
@@ -234,5 +234,10 @@ public class ArrayWritableObjectInspecto
     public ObjectInspector getFieldObjectInspector() {
       return inspector;
     }
+
+    @Override
+    public int getFieldID() {
+      return index;
+    }
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java?rev=1594315&r1=1594314&r2=1594315&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStoragePredicateHandler.java Tue May 13 18:23:14 2014
@@ -23,6 +23,8 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.mapred.JobConf;
 
+import java.io.Serializable;
+
 /**
  * HiveStoragePredicateHandler is an optional companion to {@link
  * HiveStorageHandler}; it should only be implemented by handlers which
@@ -69,6 +71,11 @@ public interface HiveStoragePredicateHan
     public ExprNodeGenericFuncDesc pushedPredicate;
 
     /**
+     * Serialized format for filter
+     */
+    public Serializable pushedPredicateObject;
+
+    /**
      * Portion of predicate to be post-evaluated by Hive for any rows
      * which are returned by storage handler.
      */



Mime
View raw message