incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1245002 - in /incubator/hcatalog/trunk/src: java/org/apache/hcatalog/common/ java/org/apache/hcatalog/data/ test/org/apache/hcatalog/data/ test/org/apache/hcatalog/rcfile/
Date Thu, 16 Feb 2012 14:45:20 GMT
Author: khorgath
Date: Thu Feb 16 14:45:20 2012
New Revision: 1245002

URL: http://svn.apache.org/viewvc?rev=1245002&view=rev
Log:
HCATALOG-241 Changes to HCatRecord to support switch from StorageDriver to SerDe (khorgath)
-- additional changes per toffer

Modified:
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Thu Feb 16
14:45:20 2012
@@ -525,30 +525,6 @@ public class HCatUtil {
             +"<databasename>.<table name> or <table name>. Got " + tableName);
       }
     }
-    
-    public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
-      return (compareRecords(first,second) == 0);
-    }
-
-    public static int compareRecords(HCatRecord first, HCatRecord second) {
-        return compareRecordContents(first.getAll(), second.getAll());
-    }
-
-    public static int compareRecordContents(List<Object> first, List<Object>
second) {
-      int mySz = first.size();
-      int urSz = second.size();
-      if(mySz != urSz) {
-        return mySz - urSz;
-      } else {
-        for (int i = 0; i < first.size(); i++) {
-          int c = DataType.compare(first.get(i), second.get(i));
-          if (c != 0) {
-            return c;
-          }
-        }
-        return 0;
-      }
-    }
 
     public static ObjectInspector getObjectInspector(String serdeClassName, 
         Configuration conf, Properties tbl) throws Exception {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecord.java Thu Feb 16
14:45:20 2012
@@ -135,14 +135,4 @@ public abstract class HCatRecord impleme
         set(fieldName,recordSchema,value);
     }
 
-//    @Override
-//    public int compareTo(Object that) {
-//      return HCatUtil.compareRecords(this,(HCatRecord)that);
-//    }
-//
-//    @Override
-//    public boolean equals(Object other) {
-//        return (compareTo(other) == 0);
-//    }
-
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordObjectInspector.java
Thu Feb 16 14:45:20 2012
@@ -40,12 +40,12 @@ public class HCatRecordObjectInspector e
   @Override
   public Object getStructFieldData(Object data, StructField fieldRef) {
     if (data == null){
-      return null;
+      return new IllegalArgumentException("Data passed in to get field from was null!");
     }
     
     int fieldID = ((MyField) fieldRef).getFieldID();
     if (!(fieldID >= 0 && fieldID < fields.size())){
-      throw new RuntimeException("Invalid field index ["+fieldID+"]");
+      throw new IllegalArgumentException("Invalid field index ["+fieldID+"]");
     }
     
     return ((HCatRecord) data).get(fieldID);

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/LazyHCatRecord.java Thu Feb
16 14:45:20 2012
@@ -30,29 +30,28 @@ public class LazyHCatRecord extends HCat
   public Object get(int fieldNum) {
     try {
       StructField fref = soi.getAllStructFieldRefs().get(fieldNum);
-      return deserialize(soi.getStructFieldData(o, fref),fref.getFieldObjectInspector());
-    } catch (Exception e) {
-      throw new RuntimeException("SerDe Exception deserializing",e);
+      return HCatRecordSerDe.serializeField(
+          soi.getStructFieldData(o, fref),
+          fref.getFieldObjectInspector());
+    } catch (SerDeException e) {
+      throw new IllegalStateException("SerDe Exception deserializing",e);
     }
   }
   
-  private Object deserialize(Object o, ObjectInspector objectInspector) throws Exception
{
-    return HCatRecordSerDe.serializeField(o, objectInspector);
-  }
 
   @Override
   public List<Object> getAll() {
     
     List<Object> r = new ArrayList<Object>(this.size);
     for (int i = 0; i < this.size; i++){
-      r.set(i, get(i));
+      r.add(i, get(i));
     }
     return r;
   }
 
   @Override
   public void set(int fieldNum, Object value) {
-    throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+    throw new UnsupportedOperationException("not allowed to run set() on LazyHCatRecord");
   }
 
   @Override
@@ -62,14 +61,14 @@ public class LazyHCatRecord extends HCat
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    throw new RuntimeException("LazyHCatRecord is intended to wrap"
+    throw new UnsupportedOperationException("LazyHCatRecord is intended to wrap"
         + " an object/object inspector as a HCatRecord "
         + "- it does not need to be read from DataInput.");
   }
 
   @Override
   public void write(DataOutput out) throws IOException {
-    throw new RuntimeException("LazyHCatRecord is intended to wrap"
+    throw new UnsupportedOperationException("LazyHCatRecord is intended to wrap"
         + " an object/object inspector as a HCatRecord "
         + "- it does not need to be written to a DataOutput.");
   }
@@ -84,17 +83,17 @@ public class LazyHCatRecord extends HCat
   @Override
   public void set(String fieldName, HCatSchema recordSchema, Object value)
       throws HCatException {
-    throw new RuntimeException("not allowed to run set() on LazyHCatRecord");
+    throw new UnsupportedOperationException("not allowed to run set() on LazyHCatRecord");
   }
 
   @Override
   public void remove(int idx) throws HCatException {
-    throw new RuntimeException("not allowed to run remove() on LazyHCatRecord");
+    throw new UnsupportedOperationException("not allowed to run remove() on LazyHCatRecord");
   }
 
   @Override
   public void copy(HCatRecord r) throws HCatException {
-    throw new RuntimeException("not allowed to run copy() on LazyHCatRecord");
+    throw new UnsupportedOperationException("not allowed to run copy() on LazyHCatRecord");
   }
   
   public LazyHCatRecord(Object o, ObjectInspector oi) throws Exception{

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/HCatDataCheckUtil.java Thu
Feb 16 14:45:20 2012
@@ -19,6 +19,7 @@ package org.apache.hcatalog.data;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.List;
 import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
@@ -87,4 +88,30 @@ public class HCatDataCheckUtil {
     return src_values;
   }
 
+  
+  public static boolean recordsEqual(HCatRecord first, HCatRecord second) {
+    return (compareRecords(first,second) == 0);
+  }
+
+  public static int compareRecords(HCatRecord first, HCatRecord second) {
+    return compareRecordContents(first.getAll(), second.getAll());
+  }
+
+  public static int compareRecordContents(List<Object> first, List<Object> second)
{
+    int mySz = first.size();
+    int urSz = second.size();
+    if(mySz != urSz) {
+      return mySz - urSz;
+    } else {
+      for (int i = 0; i < first.size(); i++) {
+        int c = DataType.compare(first.get(i), second.get(i));
+        if (c != 0) {
+          return c;
+        }
+      }
+      return 0;
+    }
+  }
+
+
 }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestDefaultHCatRecord.java
Thu Feb 16 14:45:20 2012
@@ -64,7 +64,7 @@ public class TestDefaultHCatRecord exten
     for(int i =0; i < recs.length; i++){
       HCatRecord rec = new DefaultHCatRecord();
       rec.readFields(inpStream);
-      Assert.assertTrue(HCatUtil.recordsEqual(recs[i],rec));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(recs[i],rec));
     }
 
     Assert.assertEquals(fInStream.available(), 0);
@@ -74,13 +74,13 @@ public class TestDefaultHCatRecord exten
 
   public void testCompareTo() {
     HCatRecord[] recs = getHCatRecords();
-    Assert.assertTrue(HCatUtil.compareRecords(recs[0],recs[1]) == 0);
+    Assert.assertTrue(HCatDataCheckUtil.compareRecords(recs[0],recs[1]) == 0);
   }
 
   public void testEqualsObject() {
 
     HCatRecord[] recs = getHCatRecords();
-    Assert.assertTrue(HCatUtil.recordsEqual(recs[0],recs[1]));
+    Assert.assertTrue(HCatDataCheckUtil.recordsEqual(recs[0],recs[1]));
   }
 
   private HCatRecord[] getHCatRecords(){

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestHCatRecordSerDe.java Thu
Feb 16 14:45:20 2012
@@ -116,7 +116,7 @@ public class TestHCatRecordSerDe extends
       System.out.println("ONE:"+s.toString());
 
       HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
-      Assert.assertTrue(HCatUtil.recordsEqual(r,r2));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,r2));
 
       // If it went through correctly, then s is also a HCatRecord, 
       // and also equal to the above, and a deepcopy, and this holds 
@@ -124,8 +124,8 @@ public class TestHCatRecordSerDe extends
 
       Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
       System.out.println("TWO:"+s2.toString());
-      Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s));
-      Assert.assertTrue(HCatUtil.recordsEqual(r,(HCatRecord)s2));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s2));
       
       // serialize using another serde, and read out that object repr.
       LazySimpleSerDe testSD = new LazySimpleSerDe();
@@ -147,11 +147,6 @@ public class TestHCatRecordSerDe extends
       LazyHCatRecord s6 = new LazyHCatRecord(s4,hrsd.getObjectInspector());
       System.out.println("SIX:"+s6.toString());
       
-      DelimitedJSONSerDe jsde = new DelimitedJSONSerDe();
-      jsde.initialize(conf,tblProps);
-      
-      Writable s7 = jsde.serialize(s6, hrsd.getObjectInspector());
-      System.out.println("SEVEN:"+s7);
     }
 
   }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java?rev=1245002&r1=1245001&r2=1245002&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Thu Feb 16 14:45:20 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.TaskA
 import org.apache.hcatalog.common.HCatException;
 import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.DefaultHCatRecord;
+import org.apache.hcatalog.data.HCatDataCheckUtil;
 import org.apache.hcatalog.data.HCatRecord;
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.rcfile.RCFileInputDriver;
@@ -119,7 +120,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(bytesArr[j], w);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(8, t.size());
-      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
     }
   }
 
@@ -147,7 +148,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(w.size(), 8);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(5, t.size());
-      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
     }
     assertFalse(rr.nextKeyValue());
   }
@@ -179,7 +180,7 @@ public class TestRCFileInputStorageDrive
       Assert.assertEquals(w.size(), 8);
       HCatRecord t = sd.convertToHCatRecord(null,w);
       Assert.assertEquals(7, t.size());
-      Assert.assertTrue(HCatUtil.recordsEqual(t,tuples[j]));
+      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(t,tuples[j]));
     }
     assertFalse(rr.nextKeyValue());
   }



Mime
View raw message