carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [09/15] carbondata git commit: [CARBONDATA-1307] Fix bug for TableInfo serialization problem in cluster mode
Date Tue, 25 Jul 2017 17:37:46 GMT
[CARBONDATA-1307] Fix bug for TableInfo serialization problem in cluster mode

TableInfo serialization has problem when there are scala list in ColumnSchema, this PR fixed
this issue

This closes #1172


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a3a12f0c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a3a12f0c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a3a12f0c

Branch: refs/heads/master
Commit: a3a12f0ceb9ab2a7bd3690961035fd7c186c2216
Parents: 7d1cde5
Author: jackylk <jacky.likun@huawei.com>
Authored: Fri Jul 14 16:25:12 2017 +0800
Committer: Raghunandan S <carbondatacontributions@gmail.com>
Committed: Tue Jul 25 17:35:42 2017 +0800

----------------------------------------------------------------------
 .../carbondata/hadoop/CarbonInputFormat.java    | 15 +++++++++++++--
 .../hadoop/util/ObjectSerializationUtil.java    | 20 ++++++++++++--------
 2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a3a12f0c/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 6d14424..b81fdf4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -16,6 +16,8 @@
  */
 package org.apache.carbondata.hadoop;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
@@ -112,7 +114,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void,
T> {
   public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
       throws IOException {
     if (null != tableInfo) {
-      configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
+      configuration.set(TABLE_INFO, ObjectSerializationUtil.encodeToString(tableInfo.serialize()));
     }
   }
 
@@ -121,7 +123,16 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void,
T> {
    */
   private TableInfo getTableInfo(Configuration configuration) throws IOException {
     String tableInfoStr = configuration.get(TABLE_INFO);
-    return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
+    if (tableInfoStr == null) {
+      return null;
+    } else {
+      TableInfo output = new TableInfo();
+      output.readFields(
+          new DataInputStream(
+              new ByteArrayInputStream(
+                  ObjectSerializationUtil.decodeStringToBytes(tableInfoStr))));
+      return output;
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a3a12f0c/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
index fab729e..ea3246f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
@@ -16,11 +16,7 @@
  */
 package org.apache.carbondata.hadoop.util;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.*;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -70,7 +66,11 @@ public class ObjectSerializationUtil {
       }
     }
 
-    return new String(Base64.encodeBase64(baos.toByteArray()),
+    return encodeToString(baos.toByteArray());
+  }
+
+  public static String encodeToString(byte[] bytes) throws UnsupportedEncodingException {
+    return new String(Base64.encodeBase64(bytes),
             CarbonCommonConstants.DEFAULT_CHARSET);
   }
 
@@ -86,8 +86,7 @@ public class ObjectSerializationUtil {
       return null;
     }
 
-    byte[] bytes =
-            Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+    byte[] bytes = decodeStringToBytes(objectString);
 
     ByteArrayInputStream bais = null;
     GZIPInputStream gis = null;
@@ -116,4 +115,9 @@ public class ObjectSerializationUtil {
       }
     }
   }
+
+  public static byte[] decodeStringToBytes(String objectString)
+    throws UnsupportedEncodingException {
+    return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+  }
 }


Mime
View raw message