incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1294477 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/data/HCatRecordSerDe.java src/java/org/apache/hcatalog/data/JsonSerDe.java src/test/org/apache/hcatalog/data/TestJsonSerDe.java
Date Tue, 28 Feb 2012 06:18:49 GMT
Author: gates
Date: Tue Feb 28 06:18:49 2012
New Revision: 1294477

URL: http://svn.apache.org/viewvc?rev=1294477&view=rev
Log:
HCATALOG-275 JSON SerDe issues

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue Feb 28 06:18:49 2012
@@ -52,6 +52,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-275 JSON SerDe issues (khorgath via gates)
+
   HCAT-278 When outputSchema doesn't match table schema wrong columns are returned to the
user (gates)
 
   HCAT-276 After merging in HCATALOG-237 related changes Pig scripts with more than one store
fail (daijy via gates)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/HCatRecordSerDe.java Tue Feb
28 06:18:49 2012
@@ -174,6 +174,10 @@ public class HCatRecordSerDe implements 
 
     List<? extends StructField> fields = soi.getAllStructFieldRefs();
     List<Object> list = soi.getStructFieldsDataAsList(obj);
+    
+    if (list == null){
+      return null;
+    }
 
     List<Object> l = new ArrayList<Object>(fields.size());
 
@@ -182,7 +186,7 @@ public class HCatRecordSerDe implements 
 
         // Get the field objectInspector and the field object.
         ObjectInspector foi = fields.get(i).getFieldObjectInspector();
-        Object f = (list == null ? null : list.get(i));
+        Object f = list.get(i);
         Object res = serializeField(f, foi);
         l.add(i, res);
       }
@@ -236,6 +240,10 @@ public class HCatRecordSerDe implements 
 
   private static List<?> serializeList(Object f, ListObjectInspector loi) throws SerDeException
{
     List l = loi.getList(f);
+    if (l == null){
+      return null;
+    }
+
     ObjectInspector eloi = loi.getListElementObjectInspector();
     if (eloi.getCategory() == Category.PRIMITIVE){
       List<Object> list = new ArrayList<Object>(l.size());

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/JsonSerDe.java Tue Feb 28 06:18:49
2012
@@ -228,7 +228,11 @@ public class JsonSerDe implements SerDe 
       break;
     case BOOLEAN:
       String bval = (valueToken == JsonToken.VALUE_NULL)?null:p.getText();
-      val = (bval.equalsIgnoreCase("true"));
+      if (bval != null){
+        val = Boolean.valueOf(bval);
+      } else {
+        val = null;
+      }
       break;
     case FLOAT:
       val = (valueToken == JsonToken.VALUE_NULL)?null:p.getFloatValue();
@@ -242,6 +246,10 @@ public class JsonSerDe implements SerDe 
     case BINARY:
       throw new IOException("JsonSerDe does not support BINARY type");
     case ARRAY:
+      if (valueToken == JsonToken.VALUE_NULL){
+        val = null; 
+        break;
+      }
       if (valueToken != JsonToken.START_ARRAY){
         throw new IOException("Start of Array expected");
       }
@@ -252,6 +260,10 @@ public class JsonSerDe implements SerDe 
       val = arr;
       break;
     case MAP:
+      if (valueToken == JsonToken.VALUE_NULL){
+        val = null; 
+        break;
+      }
       if (valueToken != JsonToken.START_OBJECT){
         throw new IOException("Start of Object expected");
       }
@@ -272,6 +284,10 @@ public class JsonSerDe implements SerDe 
       val = map;
       break;
     case STRUCT:
+      if (valueToken == JsonToken.VALUE_NULL){
+        val = null; 
+        break;
+      }
       if (valueToken != JsonToken.START_OBJECT){
         throw new IOException("Start of Object expected");
       }
@@ -321,7 +337,28 @@ public class JsonSerDe implements SerDe 
       throws SerDeException {
     StringBuilder sb = new StringBuilder();
     try {
-      buildJSONString(sb, obj, objInspector);
+
+      StructObjectInspector soi = (StructObjectInspector) objInspector;
+      List<? extends StructField> structFields = soi.getAllStructFieldRefs();
+      assert (columnNames.size() == structFields.size());
+      if (obj == null) {
+        sb.append("null");
+      } else {
+        sb.append(SerDeUtils.LBRACE);
+        for (int i = 0; i < structFields.size(); i++) {
+          if (i > 0) {
+            sb.append(SerDeUtils.COMMA);
+          }
+          sb.append(SerDeUtils.QUOTE);
+          sb.append(columnNames.get(i));
+          sb.append(SerDeUtils.QUOTE);
+          sb.append(SerDeUtils.COLON);
+          buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
+              structFields.get(i).getFieldObjectInspector());
+        }
+        sb.append(SerDeUtils.RBRACE);
+      }
+
     } catch (IOException e) {
       LOG.warn("Error ["+ e.getMessage()+"] generating json text from object");
       throw new SerDeException(e);

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java?rev=1294477&r1=1294476&r2=1294477&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestJsonSerDe.java Tue Feb
28 06:18:49 2012
@@ -26,6 +26,7 @@ import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 
 import junit.framework.Assert;
@@ -33,10 +34,10 @@ import junit.framework.TestCase;
 
 public class TestJsonSerDe extends TestCase{
 
-  public Map<Properties,HCatRecord> getData(){
-    Map<Properties,HCatRecord> data = new HashMap<Properties,HCatRecord>();
+  public List<Pair<Properties,HCatRecord>> getData(){
+    List<Pair<Properties,HCatRecord>> data = new ArrayList<Pair<Properties,HCatRecord>>();
 
-    List<Object> rlist = new ArrayList<Object>(11);
+    List<Object> rlist = new ArrayList<Object>(13);
     rlist.add(new Byte("123"));
     rlist.add(new Short("456"));
     rlist.add(new Integer(789));
@@ -80,6 +81,21 @@ public class TestJsonSerDe extends TestC
       c1_1.add(i2);
       c1.add(c1_1);
     rlist.add(c1);
+    
+    List<Object> nlist = new ArrayList<Object>(13);
+    nlist.add(null); // tinyint
+    nlist.add(null); // smallint
+    nlist.add(null); // int
+    nlist.add(null); // bigint
+    nlist.add(null); // double
+    nlist.add(null); // float
+    nlist.add(null); // string
+    nlist.add(null); // string
+    nlist.add(null); // struct
+    nlist.add(null); // array
+    nlist.add(null); // map
+    nlist.add(null); // bool
+    nlist.add(null); // complex
 
     String typeString = 
         "tinyint,smallint,int,bigint,double,float,string,string,"
@@ -92,7 +108,8 @@ public class TestJsonSerDe extends TestC
 //    props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
 //    props.put(Constants.SERIALIZATION_FORMAT, "1");
 
-    data.put(props, new DefaultHCatRecord(rlist));
+    data.add(new Pair(props, new DefaultHCatRecord(rlist)));
+    data.add(new Pair(props, new DefaultHCatRecord(nlist)));
     return data;
   }
 
@@ -100,9 +117,9 @@ public class TestJsonSerDe extends TestC
 
     Configuration conf = new Configuration();
 
-    for (Entry<Properties,HCatRecord> e : getData().entrySet()){
-      Properties tblProps = e.getKey();
-      HCatRecord r = e.getValue();
+    for (Pair<Properties,HCatRecord> e : getData()){
+      Properties tblProps = e.first;
+      HCatRecord r = e.second;
       
       HCatRecordSerDe hrsd = new HCatRecordSerDe();
       hrsd.initialize(conf, tblProps);



Mime
View raw message