incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1244798 - in /incubator/hcatalog/trunk: ./ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/data/ src/test/org/apache/hcatalog/data/ src/test/org/apache/hcatalog/mapreduce/ src/test/org/apache/hcatalog/rcfile/
Date Thu, 16 Feb 2012 00:07:33 GMT
Author: khorgath
Date: Thu Feb 16 00:07:32 2012
New Revision: 1244798

URL: http://svn.apache.org/viewvc?rev=1244798&view=rev
Log:
Changes to HCatRecord to support switch from StorageDriver to SerDe (khorgath)

Added:
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Feb 16 00:07:32 2012
@@ -21,6 +21,9 @@ Apache HCatalog Change Log
 Trunk (unreleased changes)
 
   INCOMPATIBLE CHANGES
+  HCAT-241. Changes to HCatRecord to support switch from StorageDriver to SerDe (khorgath)
+
+  HCAT-240. Changes to HCatOutputFormat to make it use SerDes instead of StorageDriver (toffer)
 
   NEW FEATURES
   HCAT-255 Define hadoop properties on the hcat command line (ctdean via gates)
@@ -28,7 +31,6 @@ Trunk (unreleased changes)
   HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
 
   IMPROVEMENTS
-  HCAT-240. Changes to HCatOutputFormat to make it use SerDes instead of StorageDriver (toffer)
 
   HCAT-194. Better error messages for HCatalog access control errors (julienledem via hashutosh)
 
   

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Thu Feb 16
00:07:32 2012
@@ -47,6 +47,8 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
@@ -58,6 +60,9 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hcatalog.data.DataType;
+import org.apache.hcatalog.data.HCatRecord;
+import org.apache.hcatalog.data.HCatRecordSerDe;
 import org.apache.hcatalog.data.Pair;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
@@ -71,7 +76,7 @@ import org.apache.thrift.TException;
 
 public class HCatUtil {
 
-    // static final private Log LOG = LogFactory.getLog(HCatUtil.class);
+//     static final private Log LOG = LogFactory.getLog(HCatUtil.class);
 
     public static boolean checkJobContextIfRunningFromBackend(JobContext j) {
         if (j.getConfiguration().get("mapred.task.id", "").equals("")) {
@@ -395,7 +400,7 @@ public class HCatUtil {
     public static void logStackTrace(Log logger) {
         StackTraceElement[] stackTrace = new Exception().getStackTrace();
         for (int i = 1; i < stackTrace.length; i++) {
-            logger.info("\t" + stackTrace[i].toString());
+            logger.debug("\t" + stackTrace[i].toString());
         }
     }
 
@@ -412,9 +417,9 @@ public class HCatUtil {
 
     public static void logList(Log logger, String itemName,
             List<? extends Object> list) {
-        logger.info(itemName + ":");
+        logger.debug(itemName + ":");
         for (Object item : list) {
-            logger.info("\t[" + item + "]");
+            logger.debug("\t[" + item + "]");
         }
     }
 
@@ -520,6 +525,43 @@ public class HCatUtil {
             +"<databasename>.<table name> or <table name>. Got " + tableName);
       }
     }
+    
+    public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
+      return (compareRecords(first,second) == 0);
+    }
+
+    public static int compareRecords(HCatRecord first, HCatRecord second) {
+        return compareRecordContents(first.getAll(), second.getAll());
+    }
+
+    public static int compareRecordContents(List<Object> first, List<Object>
second) {
+      int mySz = first.size();
+      int urSz = second.size();
+      if(mySz != urSz) {
+        return mySz - urSz;
+      } else {
+        for (int i = 0; i < first.size(); i++) {
+          int c = DataType.compare(first.get(i), second.get(i));
+          if (c != 0) {
+            return c;
+          }
+        }
+        return 0;
+      }
+    }
+
+    public static ObjectInspector getObjectInspector(String serdeClassName, 
+        Configuration conf, Properties tbl) throws Exception {
+      SerDe s = (SerDe) Class.forName(serdeClassName).newInstance();
+      s.initialize(conf, tbl);
+      return s.getObjectInspector();
+    }
+
+    public static ObjectInspector getHCatRecordObjectInspector(HCatSchema hsch) throws Exception{
+      HCatRecordSerDe hrsd = new HCatRecordSerDe();
+      hrsd.initialize(hsch);
+      return hrsd.getObjectInspector();
+    }
 
     public static void configureOutputStorageHandler(HCatStorageHandler storageHandler,
                                                                               JobContext
context,
@@ -560,4 +602,5 @@ public class HCatUtil {
             dest.set(el.getKey(),el.getValue());
         }
     }
+
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/DefaultHCatRecord.java Thu
Feb 16 00:07:32 2012
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.schema.HCatSchema;
 
 public class DefaultHCatRecord extends HCatRecord {
@@ -92,34 +93,6 @@ public class DefaultHCatRecord extends H
     }
 
     @Override
-    public int compareTo(Object that) {
-
-        if(that instanceof HCatRecord) {
-            HCatRecord other = (HCatRecord)that;
-            int mySz = this.size();
-            int urSz = other.size();
-            if(mySz != urSz) {
-                return mySz - urSz;
-            } else{
-                for (int i = 0; i < mySz;i++) {
-                    int c = DataType.compare(get(i), other.get(i));
-                    if (c != 0) {
-                        return c;
-                    }
-                }
-            }
-            return 0;
-        } else {
-            return DataType.compare(this, that);
-        }
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        return (compareTo(other) == 0);
-    }
-
-    @Override
     public int hashCode() {
         int hash = 1;
         for (Object o : contents) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java Thu Feb 16
00:07:32 2012
@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.schema.HCatSchema;
 
 /**
@@ -134,4 +135,14 @@ public abstract class HCatRecord impleme
         set(fieldName,recordSchema,value);
     }
 
+//    @Override
+//    public int compareTo(Object that) {
+//      return HCatUtil.compareRecords(this,(HCatRecord)that);
+//    }
+//
+//    @Override
+//    public boolean equals(Object other) {
+//        return (compareTo(other) == 0);
+//    }
+
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
Thu Feb 16 00:07:32 2012
@@ -19,12 +19,19 @@ package org.apache.hcatalog.data;
 
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.common.HCatUtil;
 
 public class HCatRecordObjectInspector extends StandardStructObjectInspector {
 
+  public static final Log LOG = LogFactory
+      .getLog(HCatRecordObjectInspector.class.getName());
+
   protected HCatRecordObjectInspector(List<String> structFieldNames,
       List<ObjectInspector> structFieldObjectInspectors) {
     super(structFieldNames, structFieldObjectInspectors);
@@ -37,7 +44,9 @@ public class HCatRecordObjectInspector e
     }
     
     int fieldID = ((MyField) fieldRef).getFieldID();
-    assert (fieldID >= 0 && fieldID < fields.size());
+    if (!(fieldID >= 0 && fieldID < fields.size())){
+      throw new RuntimeException("Invalid field index ["+fieldID+"]");
+    }
     
     return ((HCatRecord) data).get(fieldID);
   }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java Thu Feb
16 00:07:32 2012
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.serde2.typ
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.Writable;
 import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.data.schema.HCatSchema;
 
 /**
  * SerDe class for serializing to and from HCatRecord
@@ -108,6 +109,18 @@ public class HCatRecordSerDe implements 
     cachedObjectInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo);
     
   }
+  
+  public void initialize(HCatSchema hsch) throws SerDeException {
+
+    if (LOG.isDebugEnabled()){
+      LOG.debug("Initializing HCatRecordSerDe through HCatSchema" + hsch.toString());
+    }
+    
+    rowTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(hsch.toString());
+    cachedObjectInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo);
+    
+  }
+  
 
   /**
    * The purpose of a deserialize method is to turn a data blob 
@@ -156,7 +169,7 @@ public class HCatRecordSerDe implements 
    * @param soi : StructObjectInspector 
    * @return HCatRecord
    */
-  private List<?> serializeStruct(Object obj, StructObjectInspector soi)
+  private static List<?> serializeStruct(Object obj, StructObjectInspector soi)
       throws SerDeException {
 
     List<? extends StructField> fields = soi.getAllStructFieldRefs();
@@ -181,7 +194,7 @@ public class HCatRecordSerDe implements 
    * Return underlying Java Object from an object-representation 
    * that is readable by a provided ObjectInspector.
    */
-  private Object serializeField(Object field,
+  public static Object serializeField(Object field,
       ObjectInspector fieldObjectInspector) throws SerDeException {
     Object res = null;
     if (fieldObjectInspector.getCategory() == Category.PRIMITIVE){
@@ -193,7 +206,7 @@ public class HCatRecordSerDe implements 
     } else if (fieldObjectInspector.getCategory() == Category.MAP){
       res = serializeMap(field,(MapObjectInspector)fieldObjectInspector);
     } else {
-      throw new SerDeException(getClass().toString() 
+      throw new SerDeException(HCatRecordSerDe.class.toString() 
           + " does not know what to do with fields of unknown category: "
           + fieldObjectInspector.getCategory() + " , type: " + fieldObjectInspector.getTypeName());
     }
@@ -205,7 +218,7 @@ public class HCatRecordSerDe implements 
    * an object-representation that is readable by a provided
    * MapObjectInspector
    */
-  private Map<?,?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException
{
+  private static Map<?,?> serializeMap(Object f, MapObjectInspector moi) throws SerDeException
{
     ObjectInspector koi = moi.getMapKeyObjectInspector();
     ObjectInspector voi = moi.getMapValueObjectInspector();
     Map<Object,Object> m = new TreeMap<Object, Object>();
@@ -221,7 +234,7 @@ public class HCatRecordSerDe implements 
     return m;
   }
 
-  private List<?> serializeList(Object f, ListObjectInspector loi) throws SerDeException
{
+  private static List<?> serializeList(Object f, ListObjectInspector loi) throws SerDeException
{
     List l = loi.getList(f);
     ObjectInspector eloi = loi.getListElementObjectInspector();
     if (eloi.getCategory() == Category.PRIMITIVE){
@@ -244,7 +257,7 @@ public class HCatRecordSerDe implements 
       }
       throw new SerDeException("HCatSerDe map type unimplemented");
     } else {
-      throw new SerDeException(getClass().toString() 
+      throw new SerDeException(HCatRecordSerDe.class.toString() 
           + " does not know what to do with fields of unknown category: "
           + eloi.getCategory() + " , type: " + eloi.getTypeName());
     }
@@ -274,4 +287,5 @@ public class HCatRecordSerDe implements 
     return null;
   }
 
+  
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordable.java Thu Feb
16 00:07:32 2012
@@ -19,17 +19,19 @@ package org.apache.hcatalog.data;
 
 import java.util.List;
 
-import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hcatalog.common.HCatException;
 
 /**
  * Interface that determines whether we can implement a HCatRecord on top of it
  */
-public interface HCatRecordable extends WritableComparable<Object> {
+public interface HCatRecordable extends Writable {
 
   /**
    * Gets the field at the specified index.
    * @param fieldNum the field number
    * @return the object at the specified index
+   * @throws HCatException 
    */
   Object get(int fieldNum);
 

Added: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java?rev=1244798&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java (added)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java Thu Feb
16 00:07:32 2012
@@ -0,0 +1,123 @@
+package org.apache.hcatalog.data;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hcatalog.common.HCatException;
+import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.data.schema.HCatSchema;
+
+public class LazyHCatRecord extends HCatRecord {
+
+  public static final Log LOG = LogFactory
+      .getLog(LazyHCatRecord.class.getName());
+
+  private Object o;
+  private StructObjectInspector soi;
+  private int size;
+  
+  @Override
+  public Object get(int fieldNum) {
+    try {
+      StructField fref = soi.getAllStructFieldRefs().get(fieldNum);
+      return deserialize(soi.getStructFieldData(o, fref),fref.getFieldObjectInspector());
+    } catch (Exception e) {
+      throw new RuntimeException("SerDe Exception deserializing",e);
+    }
+  }
+  
+  private Object deserialize(Object o, ObjectInspector objectInspector) throws Exception
{
+    return HCatRecordSerDe.serializeField(o, objectInspector);
+  }
+
+  @Override
+  public List<Object> getAll() {
+    
+    List<Object> r = new ArrayList<Object>(this.size);
+    for (int i = 0; i < this.size; i++){
+      r.set(i, get(i));
+    }
+    return r;
+  }
+
+  @Override
+  public void set(int fieldNum, Object value) {
+    throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+  }
+
+  @Override
+  public int size() {
+    return this.size;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    throw new RuntimeException("LazyHCatRecord is intended to wrap"
+        + " an object/object inspector as a HCatRecord "
+        + "- it does not need to be read from DataInput.");
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    throw new RuntimeException("LazyHCatRecord is intended to wrap"
+        + " an object/object inspector as a HCatRecord "
+        + "- it does not need to be written to a DataOutput.");
+  }
+
+  @Override
+  public Object get(String fieldName, HCatSchema recordSchema)
+      throws HCatException {
+    int idx = recordSchema.getPosition(fieldName);
+    return get(idx);
+  }
+
+  @Override
+  public void set(String fieldName, HCatSchema recordSchema, Object value)
+      throws HCatException {
+    throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+  }
+
+  @Override
+  public void remove(int idx) throws HCatException {
+    throw new RuntimeException("not allowed to run remove() on LazyHCatRecord");
+  }
+
+  @Override
+  public void copy(HCatRecord r) throws HCatException {
+    throw new RuntimeException("not allowed to run copy() on LazyHCatRecord");
+  }
+  
+  public LazyHCatRecord(Object o, ObjectInspector oi) throws Exception{
+
+    if (oi.getCategory() != Category.STRUCT) {
+      throw new SerDeException(getClass().toString()
+          + " can only make a lazy hcat record from objects of struct types, but we got:
"
+          + oi.getTypeName());
+    }
+
+    this.soi = (StructObjectInspector)oi;
+    this.o = o;
+    this.size = soi.getAllStructFieldRefs().size();
+
+  }
+
+  @Override
+  public String toString(){
+    StringBuilder sb = new StringBuilder();
+    for(int i = 0; i< size ; i++) {
+      sb.append(get(i)+"\t");
+    }
+    return sb.toString();
+  }
+
+}

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
Thu Feb 16 00:07:32 2012
@@ -32,6 +32,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.DefaultHCatRecord;
 import org.apache.hcatalog.data.HCatRecord;
 
@@ -63,7 +64,7 @@ public class TestDefaultHCatRecord exten
     for(int i =0; i < recs.length; i++){
       HCatRecord rec = new DefaultHCatRecord();
       rec.readFields(inpStream);
-      Assert.assertEquals(recs[i],rec);
+      Assert.assertTrue(HCatUtil.recordsEqual(recs[i],rec));
     }
 
     Assert.assertEquals(fInStream.available(), 0);
@@ -73,13 +74,13 @@ public class TestDefaultHCatRecord exten
 
   public void testCompareTo() {
     HCatRecord[] recs = getHCatRecords();
-    Assert.assertEquals(recs[0].compareTo(recs[1]),0);
+    Assert.assertTrue(HCatUtil.compareRecords(recs[0],recs[1]) == 0);
   }
 
   public void testEqualsObject() {
 
     HCatRecord[] recs = getHCatRecords();
-    Assert.assertTrue(recs[0].equals(recs[1]));
+    Assert.assertTrue(HCatUtil.recordsEqual(recs[0],recs[1]));
   }
 
   private HCatRecord[] getHCatRecords(){

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java Thu
Feb 16 00:07:32 2012
@@ -26,8 +26,10 @@ import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.io.Writable;
+import org.apache.hcatalog.common.HCatUtil;
 
 import junit.framework.Assert;
 import junit.framework.TestCase;
@@ -90,6 +92,8 @@ public class TestHCatRecordSerDe extends
     
     props.put(Constants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1");
     props.put(Constants.LIST_COLUMN_TYPES, typeString);
+//    props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
+//    props.put(Constants.SERIALIZATION_FORMAT, "1");
 
     data.put(props, new DefaultHCatRecord(rlist));
     return data;
@@ -112,7 +116,7 @@ public class TestHCatRecordSerDe extends
       System.out.println("ONE:"+s.toString());
 
       HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
-      Assert.assertTrue(r.equals(r2));
+      Assert.assertTrue(HCatUtil.recordsEqual(r,r2));
 
       // If it went through correctly, then s is also a HCatRecord, 
       // and also equal to the above, and a deepcopy, and this holds 
@@ -120,13 +124,13 @@ public class TestHCatRecordSerDe extends
 
       Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
       System.out.println("TWO:"+s2.toString());
-      Assert.assertTrue(r.equals((HCatRecord)s));
-      Assert.assertTrue(r.equals((HCatRecord)s2));
+      Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s));
+      Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s2));
       
       // serialize using another serde, and read out that object repr.
       LazySimpleSerDe testSD = new LazySimpleSerDe();
       testSD.initialize(conf, tblProps);
-      
+
       Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
       System.out.println("THREE:"+s3.toString());
       Object o3 = testSD.deserialize(s3);
@@ -135,8 +139,19 @@ public class TestHCatRecordSerDe extends
       // then serialize again using hrsd, and compare results
       HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
       System.out.println("FOUR:"+s4.toString());
-      Assert.assertFalse(r.equals(s4));
+
+      // Test LazyHCatRecord init and read
+      LazyHCatRecord s5 = new LazyHCatRecord(o3,testSD.getObjectInspector());
+      System.out.println("FIVE:"+s5.toString());
+
+      LazyHCatRecord s6 = new LazyHCatRecord(s4,hrsd.getObjectInspector());
+      System.out.println("SIX:"+s6.toString());
+      
+      DelimitedJSONSerDe jsde = new DelimitedJSONSerDe();
+      jsde.initialize(conf,tblProps);
       
+      Writable s7 = jsde.serialize(s6, hrsd.getObjectInspector());
+      System.out.println("SEVEN:"+s7);
     }
 
   }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Thu Feb 16 00:07:32 2012
@@ -133,27 +133,28 @@ public class TestHCatDynamicPartitioned 
   }
 
 //TODO 1.0 miniCluster is slow this test times out, make it work
-//  public void testHCatDynamicPartitionMaxPartitions() throws Exception {
-//    HiveConf hc = new HiveConf(this.getClass());
-//
-//    int maxParts = hiveConf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS);
-//    System.out.println("Max partitions allowed = " + maxParts);
-//
-//    IOException exc = null;
-//    try {
-//      generateWriteRecords(maxParts+5,maxParts+2,10);
-//      runMRCreate(null,dataColumns,writeRecords,maxParts+5,false);
-//    } catch(IOException e) {
-//      exc = e;
-//    }
-//
-//    if (HCatConstants.HCAT_IS_DYNAMIC_MAX_PTN_CHECK_ENABLED){
-//      assertTrue(exc != null);
-//      assertTrue(exc instanceof HCatException);
-//      assertEquals(ErrorType.ERROR_TOO_MANY_DYNAMIC_PTNS, ((HCatException) exc).getErrorType());
-//    }else{
-//      assertTrue(exc == null);
-//      runMRRead(maxParts+5);
-//    }
-//  }
+// renaming test to make test framework skip it
+  public void _testHCatDynamicPartitionMaxPartitions() throws Exception {
+    HiveConf hc = new HiveConf(this.getClass());
+
+    int maxParts = hiveConf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS);
+    System.out.println("Max partitions allowed = " + maxParts);
+
+    IOException exc = null;
+    try {
+      generateWriteRecords(maxParts+5,maxParts+2,10);
+      runMRCreate(null,dataColumns,writeRecords,maxParts+5,false);
+    } catch(IOException e) {
+      exc = e;
+    }
+
+    if (HCatConstants.HCAT_IS_DYNAMIC_MAX_PTN_CHECK_ENABLED){
+      assertTrue(exc != null);
+      assertTrue(exc instanceof HCatException);
+      assertEquals(ErrorType.ERROR_TOO_MANY_DYNAMIC_PTNS, ((HCatException) exc).getErrorType());
+    }else{
+      assertTrue(exc == null);
+      runMRRead(maxParts+5);
+    }
+  }
 }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java?rev=1244798&r1=1244797&r2=1244798&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Thu Feb 16 00:07:32 2012
@@ -119,7 +119,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(bytesArr[j], w);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(8, t.size());
-      Assert.assertEquals(t,tuples[j]);
+      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
     }
   }
 
@@ -147,7 +147,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(w.size(), 8);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(5, t.size());
-      Assert.assertEquals(t,tuples[j]);
+      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
     }
     assertFalse(rr.nextKeyValue());
   }
@@ -179,7 +179,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(w.size(), 8);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(7, t.size());
-      Assert.assertEquals(t,tuples[j]);
+      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
     }
     assertFalse(rr.nextKeyValue());
   }



Mime
View raw message