carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [2/2] carbondata git commit: [CARBONDATA-1443] Schema storage has issue while converting table object to json string
Date Fri, 01 Sep 2017 07:16:31 GMT
[CARBONDATA-1443] Schema storage has issue while converting table object to json string

Every table object converts to json and splits to small splits as there is limit in hive metastore
of 4000. While splitting the json string code has problem so it could not get table and throws
NPE

This closes #1314


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/baca6f92
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/baca6f92
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/baca6f92

Branch: refs/heads/branch-1.2
Commit: baca6f925ad973864ff432dbaccd6a99df5ea363
Parents: bb0b347
Author: Ravindra Pesala <ravi.pesala@gmail.com>
Authored: Fri Sep 1 10:51:04 2017 +0530
Committer: chenliang613 <chenliang613@apache.org>
Committed: Fri Sep 1 14:25:00 2017 +0800

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java | 51 +++++++++++++++++---
 .../carbondata/core/util/CarbonUtilTest.java    | 47 +++++++++++++++++-
 2 files changed, 91 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 67c1d81..8b6e44a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1803,10 +1803,31 @@ public final class CarbonUtil {
     return !(null == badRecordsLocation || badRecordsLocation.length() == 0);
   }
 
+  /**
+   * Converts Tableinfo object to json multi string objects of size 4000
+   * @param tableInfo
+   * @param seperator separator between each string
+   * @param quote Quote to be used for string
+   * @param prefix Prefix to be added before generated string
+   * @return
+   */
   public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator,
       String quote, String prefix) {
     Gson gson = new Gson();
     String schemaString = gson.toJson(tableInfo);
+    return splitSchemaStringToMultiString(seperator, quote, prefix, schemaString);
+  }
+
+  /**
+   * Converts Json String to multi string objects of size 4000
+   * @param schemaString Json string
+   * @param seperator separator between each string
+   * @param quote Quote to be used for string
+   * @param prefix Prefix to be added before generated string
+   * @return
+   */
+  public static String splitSchemaStringToMultiString(String seperator, String quote,
+      String prefix, String schemaString) {
     int schemaLen = schemaString.length();
     int splitLen = 4000;
     int parts = schemaLen / splitLen;
@@ -1817,10 +1838,12 @@ public final class CarbonUtil {
         new StringBuilder(prefix).append(quote).append("carbonSchemaPartsNo").append(quote)
             .append(seperator).append("'").append(parts).append("',");
     int runningLen = 0;
-    int endLen = splitLen;
+    int endLen = schemaLen > splitLen ? splitLen : schemaLen;
     for (int i = 0; i < parts; i++) {
       if (i == parts - 1) {
-        endLen = schemaLen % splitLen;
+        if (schemaLen % splitLen > 0) {
+          endLen = schemaLen % splitLen;
+        }
       }
       builder.append(quote).append("carbonSchema").append(i).append(quote).append(seperator);
       builder.append("'").append(schemaString.substring(runningLen, runningLen + endLen))
@@ -1833,10 +1856,25 @@ public final class CarbonUtil {
     return builder.toString();
   }
 
+  /**
+   * Converts Tableinfo object to json multi string objects  of size 4000 and stored in map
+   * @param tableInfo
+   * @return
+   */
   public static Map<String, String> convertToMultiStringMap(TableInfo tableInfo) {
-    Map<String, String> map = new HashMap<>();
     Gson gson = new Gson();
     String schemaString = gson.toJson(tableInfo);
+    return splitSchemaStringToMap(schemaString);
+  }
+
+  /**
+   * Converts Json string to multi string objects  of size 4000 and stored in map
+   *
+   * @param schemaString
+   * @return
+   */
+  public static Map<String, String> splitSchemaStringToMap(String schemaString) {
+    Map<String, String> map = new HashMap<>();
     int schemaLen = schemaString.length();
     int splitLen = 4000;
     int parts = schemaLen / splitLen;
@@ -1845,10 +1883,12 @@ public final class CarbonUtil {
     }
     map.put("carbonSchemaPartsNo", parts + "");
     int runningLen = 0;
-    int endLen = splitLen;
+    int endLen = schemaLen > splitLen ? splitLen : schemaLen;
     for (int i = 0; i < parts; i++) {
       if (i == parts - 1) {
-        endLen = schemaLen % splitLen;
+        if (schemaLen % splitLen > 0) {
+          endLen = schemaLen % splitLen;
+        }
       }
       map.put("carbonSchema" + i, schemaString.substring(runningLen, runningLen + endLen));
       runningLen += splitLen;
@@ -1856,7 +1896,6 @@ public final class CarbonUtil {
     return map;
   }
 
-
   public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
     Gson gson = new Gson();
     String partsNo = properties.get("carbonSchemaPartsNo");

http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 776059f..4a8483f 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -27,6 +27,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.carbondata.core.datastore.block.TableBlockInfo;
 import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
@@ -1002,6 +1003,50 @@ public class CarbonUtilTest {
 
   }
 
+  @Test
+  public void testSplitSchemaStringToMapWithLessThanSplitLen() {
+    String schema = generateString(399);
+    Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+    assert (map.size() == 2);
+    String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+    assert (schemaString.length() > schema.length());
+  }
+
+  @Test
+  public void testSplitSchemaStringToMapWithEqualThanSplitLen() {
+    String schema = generateString(4000);
+    Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+    assert (map.size() == 2);
+    String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+    assert (schemaString.length() > schema.length());
+  }
+
+  @Test
+  public void testSplitSchemaStringToMapWithMoreThanSplitLen() {
+    String schema = generateString(7999);
+    Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+    assert (map.size() == 3);
+    String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+    assert (schemaString.length() > schema.length());
+  }
+
+  @Test
+  public void testSplitSchemaStringToMapWithMultiplesOfSplitLen() {
+    String schema = generateString(12000);
+    Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+    assert (map.size() == 4);
+    String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+    assert (schemaString.length() > schema.length());
+  }
+
+  private String generateString(int length) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < length; i++) {
+      builder.append("a");
+    }
+    return builder.toString();
+  }
+
   private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
       FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[]
expectRangeIndex) {
     int[] range;
@@ -1013,7 +1058,7 @@ public class CarbonUtilTest {
     int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1,
keyWord);
     assertTrue(expectRangeIndex[0] <= index && index <= range[1]);
   }
- 	
+
  	
   @AfterClass public static void testcleanUp() {
     new File("../core/src/test/resources/testFile.txt").deleteOnExit();


Mime
View raw message