carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [22/50] [abbrv] carbondata git commit: [CARBONDATA-2729][file-format] Schema Compatibility problem between version 1.3.0 and 1.4.0
Date Wed, 18 Jul 2018 02:19:55 GMT
[CARBONDATA-2729][file-format] Schema Compatibility problem between version 1.3.0 and 1.4.0

Problem:
In TableSchema the field Name schemaEvaluation is changed to schemaEvoluation and in DataMapSchema
field name className is changed to providerName.
Due to this current Carbon version & Version 1.4.0 is not able to properly de-serialize
the schema created with older Carbon Version.
Solution
To de-serialize old schema having different field name the multiple values for SerializedName
annotation can be used.
For refrence please refer the gson PR#699.

This closes #2498


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bc12de00
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bc12de00
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bc12de00

Branch: refs/heads/carbonstore
Commit: bc12de004b654d3f61c836672f3ff857bb3e9a97
Parents: 3df2fd0
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Mon Jul 2 19:51:03 2018 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Sat Jul 14 14:50:08 2018 +0800

----------------------------------------------------------------------
 core/pom.xml                                    |   2 +-
 .../metadata/schema/table/DataMapSchema.java    |   4 +
 .../core/metadata/schema/table/TableSchema.java |   5 +
 .../metadata/schema/table/TableInfoTest.java    | 203 +++++++++++++++++--
 integration/spark-common/pom.xml                |   2 +-
 5 files changed, 193 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc12de00/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 71fa66c..decb5d0 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -48,7 +48,7 @@
     <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
-      <version>2.3.1</version>
+      <version>2.4</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc12de00/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
index e373fae..a48b03c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty;
 import static org.apache.carbondata.core.constants.CarbonCommonConstants.INDEX_COLUMNS;
 
 import com.google.gson.Gson;
+import com.google.gson.annotations.SerializedName;
 import org.apache.commons.lang.StringUtils;
 
 /**
@@ -49,6 +50,9 @@ public class DataMapSchema implements Serializable, Writable {
    * 1. Index DataMap: provider name is class name of implementation class of DataMapFactory
    * 2. OLAP DataMap: provider name is one of the {@link DataMapClassProvider#shortName}
    */
+  // the old version the field name for providerName was className, so to de-serialization
+  // old schema provided the old field name in the alternate filed using annotation
+  @SerializedName(value = "providerName", alternate = "className")
   protected String providerName;
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc12de00/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index b53a9d8..4425697 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -34,6 +34,8 @@ import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.util.CarbonUtil;
 
+import com.google.gson.annotations.SerializedName;
+
 /**
  * Persisting the table information
  */
@@ -62,6 +64,9 @@ public class TableSchema implements Serializable, Writable {
   /**
    * History of schema evolution of this table
    */
+  // the old version the field name for schemaEvolution was schemaEvalution, so to de-serialization
+  // old schema provided the old field name in the alternate filed using annotation
+  @SerializedName(value = "schemaEvolution", alternate = "schemaEvalution")
   private SchemaEvolution schemaEvolution;
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc12de00/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableInfoTest.java
b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableInfoTest.java
index 811ce00..df8fecc 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableInfoTest.java
@@ -16,6 +16,13 @@
  */
 package org.apache.carbondata.core.metadata.schema.table;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
+import org.apache.carbondata.core.util.CarbonUtil;
+
 import junit.framework.TestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -23,30 +30,184 @@ import org.junit.Test;
 
 public class TableInfoTest extends TestCase {
 
-    private TableInfo tableInfo;
+  private TableInfo tableInfo;
 
-    @BeforeClass public void setUp() {
-        tableInfo = getTableInfo("tableInfoTestDatabase", "equalsTableInfoTestTable");
-    }
+  @BeforeClass public void setUp() {
+    tableInfo = getTableInfo("tableInfoTestDatabase", "equalsTableInfoTestTable");
+  }
 
-    @AfterClass public void tearDown() {
-        tableInfo = null;
-    }
+  @AfterClass public void tearDown() {
+    tableInfo = null;
+  }
 
-    @Test public void testTableInfoEquals() {
-        TableInfo cmpEqualsTableInfo =
-            getTableInfo("tableInfoTestDatabase", "equalsTableInfoTestTable");
-        TableInfo cmpNotEqualsTableInfo =
-            getTableInfo("tableInfoTestDatabase", "notEqualsTableInfoTestTable");
-        assertTrue(tableInfo.equals(cmpEqualsTableInfo));
-        assertTrue(!(tableInfo.equals(cmpNotEqualsTableInfo)));
-    }
+  @Test public void testTableInfoEquals() {
+    TableInfo cmpEqualsTableInfo =
+        getTableInfo("tableInfoTestDatabase", "equalsTableInfoTestTable");
+    TableInfo cmpNotEqualsTableInfo =
+        getTableInfo("tableInfoTestDatabase", "notEqualsTableInfoTestTable");
+    assertTrue(tableInfo.equals(cmpEqualsTableInfo));
+    assertTrue(!(tableInfo.equals(cmpNotEqualsTableInfo)));
+  }
+
+  private TableInfo getTableInfo(String databaseName, String tableName) {
+    TableInfo info = new TableInfo();
+    info.setDatabaseName("tableInfoTestDatabase");
+    info.setLastUpdatedTime(1000L);
+    info.setTableUniqueName(CarbonTable.buildUniqueName(databaseName, tableName));
+    return info;
+  }
+
+  /**
+   * test with field name schemaEvalution
+   */
+  @Test public void testTableInfoDeserializationWithSchemaEvalution() {
+    Map<String, String> properties = new HashMap<>();
+    properties.put("carbonSchemaPartsNo", "1");
+    properties.put("carbonSchema0", "{\"databaseName\":\"carbonversion_1_1\",\"tableUniqueName\":"
+        + "\"carbonversion_1_1_testinttype1\",\"factTable\":{\"tableId\":"
+        + "\"8ef75d32-b5b7-4c6b-9b1c-16059b4f5f26\",\"tableName\":\"testinttype1\","
+        + "\"listOfColumns\":[{\"dataType\":{\"id\":0,\"precedenceOrder\":0,\"name\":"
+        + "\"STRING\",\"sizeInBytes\":-1},\"columnName\":\"c1\",\"columnUniqueId\":"
+        + "\"5347ee48-d3ca-43a4-bd88-a818fa42a2a3\",\"columnReferenceId\":"
+        + "\"5347ee48-d3ca-43a4-bd88-a818fa42a2a3\",\"isColumnar\":true,"
+        + "\"encodingList\":[\"INVERTED_INDEX\"],\"isDimensionColumn\":true,"
+        + "\"columnGroupId\":-1,\"scale\":0,\"precision\":0,\"schemaOrdinal\":0,"
+        + "\"numberOfChild\":0,\"invisible\":false,\"isSortColumn\":true,"
+        + "\"aggFunction\":\"\",\"timeSeriesFunction\":\"\",\"isLocalDictColumn\":true},"
+        + "{\"dataType\":{\"id\":5,\"precedenceOrder\":3,\"name\":\"INT\","
+        + "\"sizeInBytes\":4},\"columnName\":\"c2\",\"columnUniqueId\":"
+        + "\"9c38b74d-5b56-4554-adbf-b6c7ead63ee2\",\"columnReferenceId\":"
+        + "\"9c38b74d-5b56-4554-adbf-b6c7ead63ee2\",\"isColumnar\":true,"
+        + "\"encodingList\":[],\"isDimensionColumn\":false,\"columnGroupId\":-1,"
+        + "\"scale\":0,\"precision\":0,\"schemaOrdinal\":1,\"numberOfChild\":0,"
+        + "\"invisible\":false,\"isSortColumn\":false,\"aggFunction\":\"\","
+        + "\"timeSeriesFunction\":\"\",\"isLocalDictColumn\":false}],\"schemaEvalution\":"
+        + "{\"schemaEvolutionEntryList\":[{\"timeStamp\":1530534235537}]},"
+        + "\"tableProperties\":{\"sort_columns\":\"c1\",\"comment\":\"\","
+        + "\"local_dictionary_enable\":\"true\"}},\"lastUpdatedTime\":1530534235537,"
+        + "\"tablePath\":\"/store/carbonversion_1_1/testinttype1\","
+        + "\"isTransactionalTable\":true,\"dataMapSchemaList\":[],"
+        + "\"parentRelationIdentifiers\":[],\"isSchemaModified\":false}");
+    TableInfo tableInfo = CarbonUtil.convertGsonToTableInfo(properties);
+    // the schema evolution should not be null
+    assertTrue(null != tableInfo.getFactTable().getSchemaEvolution());
+    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
+    ThriftWrapperSchemaConverterImpl schemaConverter = new ThriftWrapperSchemaConverterImpl();
+    org.apache.carbondata.format.TableInfo thriftTable = schemaConverter
+        .fromWrapperToExternalTableInfo(carbonTable.getTableInfo(), carbonTable.getDatabaseName(),
+            carbonTable.getTableName());
+    assertTrue(null != thriftTable);
+  }
+
+  /**
+   * test with field name schemaEvolution
+   */
+  @Test public void testTableInfoDeserializationWithSchemaEvolution() {
+    Map<String, String> properties = new HashMap<>();
+    properties.put("carbonSchemaPartsNo", "1");
+    properties.put("carbonSchema0", "{\"databaseName\":\"carbonversion_1_1\",\"tableUniqueName\":"
+        + "\"carbonversion_1_1_testinttype1\",\"factTable\":{\"tableId\":"
+        + "\"8ef75d32-b5b7-4c6b-9b1c-16059b4f5f26\",\"tableName\":\"testinttype1\","
+        + "\"listOfColumns\":[{\"dataType\":{\"id\":0,\"precedenceOrder\":0,\"name\":"
+        + "\"STRING\",\"sizeInBytes\":-1},\"columnName\":\"c1\",\"columnUniqueId\":"
+        + "\"5347ee48-d3ca-43a4-bd88-a818fa42a2a3\",\"columnReferenceId\":"
+        + "\"5347ee48-d3ca-43a4-bd88-a818fa42a2a3\",\"isColumnar\":true,"
+        + "\"encodingList\":[\"INVERTED_INDEX\"],\"isDimensionColumn\":true,"
+        + "\"columnGroupId\":-1,\"scale\":0,\"precision\":0,\"schemaOrdinal\":0,"
+        + "\"numberOfChild\":0,\"invisible\":false,\"isSortColumn\":true,"
+        + "\"aggFunction\":\"\",\"timeSeriesFunction\":\"\",\"isLocalDictColumn\":true},"
+        + "{\"dataType\":{\"id\":5,\"precedenceOrder\":3,\"name\":\"INT\","
+        + "\"sizeInBytes\":4},\"columnName\":\"c2\",\"columnUniqueId\":"
+        + "\"9c38b74d-5b56-4554-adbf-b6c7ead63ee2\",\"columnReferenceId\":"
+        + "\"9c38b74d-5b56-4554-adbf-b6c7ead63ee2\",\"isColumnar\":true,"
+        + "\"encodingList\":[],\"isDimensionColumn\":false,\"columnGroupId\":-1,"
+        + "\"scale\":0,\"precision\":0,\"schemaOrdinal\":1,\"numberOfChild\":0,"
+        + "\"invisible\":false,\"isSortColumn\":false,\"aggFunction\":\"\","
+        + "\"timeSeriesFunction\":\"\",\"isLocalDictColumn\":false}],\"schemaEvolution\":"
+        + "{\"schemaEvolutionEntryList\":[{\"timeStamp\":1530534235537}]},"
+        + "\"tableProperties\":{\"sort_columns\":\"c1\",\"comment\":\"\","
+        + "\"local_dictionary_enable\":\"true\"}},\"lastUpdatedTime\":1530534235537,"
+        + "\"tablePath\":\"/store/carbonversion_1_1/testinttype1\","
+        + "\"isTransactionalTable\":true,\"dataMapSchemaList\":[],"
+        + "\"parentRelationIdentifiers\":[],\"isSchemaModified\":false}");
+    TableInfo tableInfo = CarbonUtil.convertGsonToTableInfo(properties);
+    // the schema evolution should not be null
+    assertTrue(null != tableInfo.getFactTable().getSchemaEvolution());
+    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
+    ThriftWrapperSchemaConverterImpl schemaConverter = new ThriftWrapperSchemaConverterImpl();
+    org.apache.carbondata.format.TableInfo thriftTable = schemaConverter
+        .fromWrapperToExternalTableInfo(carbonTable.getTableInfo(), carbonTable.getDatabaseName(),
+            carbonTable.getTableName());
+    assertTrue(null != thriftTable);
+  }
 
-    private TableInfo getTableInfo(String databaseName, String tableName) {
-        TableInfo info = new TableInfo();
-        info.setDatabaseName("tableInfoTestDatabase");
-        info.setLastUpdatedTime(1000L);
-        info.setTableUniqueName(CarbonTable.buildUniqueName(databaseName, tableName));
-        return info;
+  @Test public void testDataMapSchemaDesrializationWithClassName() {
+    Map<String, String> properties = new HashMap<>();
+    properties.put("carbonSchemaPartsNo", "1");
+    properties.put("carbonSchema0", "{\"databaseName\":\"carbonversion_1_3\",\"tableUniqueName\":"
+        + "\"carbonversion_1_3_testinttype3\",\"factTable\":{\"tableId\":"
+        + "\"453fa0dd-721d-41b7-9378-f6d6122daf36\",\"tableName\":\"testinttype3\","
+        + "\"listOfColumns\":[{\"dataType\":{\"id\":0,\"precedenceOrder\":0,\"name\":"
+        + "\"STRING\",\"sizeInBytes\":-1},\"columnName\":\"c1\",\"columnUniqueId\":"
+        + "\"c84e7e3b-5682-4b46-8c72-0f2f341a0a49\",\"columnReferenceId\":"
+        + "\"c84e7e3b-5682-4b46-8c72-0f2f341a0a49\",\"isColumnar\":true,\"encodingList"
+        + "\":[\"INVERTED_INDEX\"],\"isDimensionColumn\":true,\"columnGroupId\":-1,"
+        + "\"scale\":-1,\"precision\":-1,\"schemaOrdinal\":0,\"numberOfChild\":0,"
+        + "\"invisible\":false,\"isSortColumn\":true,\"aggFunction\":\"\","
+        + "\"timeSeriesFunction\":\"\"},{\"dataType\":{\"id\":5,\"precedenceOrder\":3,"
+        + "\"name\":\"INT\",\"sizeInBytes\":4},\"columnName\":\"c2\",\"columnUniqueId\":"
+        + "\"008dc283-beca-4a3e-ad40-b7916aa67795\",\"columnReferenceId\":"
+        + "\"008dc283-beca-4a3e-ad40-b7916aa67795\",\"isColumnar\":true,"
+        + "\"encodingList\":[],\"isDimensionColumn\":false,\"columnGroupId\":-1,"
+        + "\"scale\":-1,\"precision\":-1,\"schemaOrdinal\":1,\"numberOfChild\":0,"
+        + "\"invisible\":false,\"isSortColumn\":false,\"aggFunction\":\"\","
+        + "\"timeSeriesFunction\":\"\"}],\"schemaEvalution\":{\"schemaEvolutionEntryList"
+        + "\":[{\"timeStamp\":1531389794988,\"added\":[],\"removed\":[]}]},"
+        + "\"tableProperties\":{\"sort_columns\":\"c1\",\"comment\":\"\"}},"
+        + "\"lastUpdatedTime\":1531389794988,\"tablePath\":"
+        + "\"/opt/store/carbonversion_1_3/testinttype3\",\"dataMapSchemaList\":"
+        + "[{\"dataMapName\":\"dm1\",\"className\":"
+        + "\"org.apache.carbondata.core.datamap.AggregateDataMap\","
+        + "\"relationIdentifier\":{\"databaseName\":\"carbonversion_1_3\","
+        + "\"tableName\":\"testinttype3_dm1\",\"tableId\":"
+        + "\"97ccae02-c821-4601-a782-69e715671419\"},\"childSchema\":"
+        + "{\"tableId\":\"97ccae02-c821-4601-a782-69e715671419\",\"tableName\":"
+        + "\"testinttype3_dm1\",\"listOfColumns\":[{\"dataType\":{\"id\":0,"
+        + "\"precedenceOrder\":0,\"name\":\"STRING\",\"sizeInBytes\":-1},"
+        + "\"columnName\":\"testinttype3_c1\",\"columnUniqueId\":"
+        + "\"e72ec46b-f41d-43e8-82d9-9b44714a3f36\",\"columnReferenceId\":"
+        + "\"e72ec46b-f41d-43e8-82d9-9b44714a3f36\",\"isColumnar\":true,\"encodingList"
+        + "\":[\"INVERTED_INDEX\"],\"isDimensionColumn\":true,\"columnGroupId\":-1,"
+        + "\"scale\":-1,\"precision\":-1,\"schemaOrdinal\":0,\"numberOfChild\":0,"
+        + "\"invisible\":false,\"isSortColumn\":true,\"aggFunction\":\"\","
+        + "\"parentColumnTableRelations\":[{\"relationIdentifier\":{\"databaseName\":"
+        + "\"carbonversion_1_3\",\"tableName\":\"testinttype3\",\"tableId\":"
+        + "\"453fa0dd-721d-41b7-9378-f6d6122daf36\"},\"columnId\":"
+        + "\"c84e7e3b-5682-4b46-8c72-0f2f341a0a49\",\"columnName\":\"c1\"}],"
+        + "\"timeSeriesFunction\":\"\"},{\"dataType\":{\"id\":7,\"precedenceOrder\":5,"
+        + "\"name\":\"LONG\",\"sizeInBytes\":8},\"columnName\":\"testinttype3_c2_sum\","
+        + "\"columnUniqueId\":\"4d77c528-c830-4f8b-943b-bac9ee9f9af7\",\"columnReferenceId"
+        + "\":\"4d77c528-c830-4f8b-943b-bac9ee9f9af7\",\"isColumnar\":true,\"encodingList"
+        + "\":[],\"isDimensionColumn\":false,\"columnGroupId\":-1,\"scale\":-1,\"precision"
+        + "\":-1,\"schemaOrdinal\":1,\"numberOfChild\":0,\"invisible\":false,\"isSortColumn"
+        + "\":false,\"aggFunction\":\"sum\",\"parentColumnTableRelations\":"
+        + "[{\"relationIdentifier\":{\"databaseName\":\"carbonversion_1_3\",\"tableName"
+        + "\":\"testinttype3\",\"tableId\":\"453fa0dd-721d-41b7-9378-f6d6122daf36\"},"
+        + "\"columnId\":\"008dc283-beca-4a3e-ad40-b7916aa67795\",\"columnName\":\"c2\"}],"
+        + "\"timeSeriesFunction\":\"\"}],\"schemaEvalution\":{\"schemaEvolutionEntryList\":"
+        + "[{\"timeStamp\":1531389797829,\"added\":[],\"removed\":[]}]},\"tableProperties\""
+        + ":{\"sort_scope\":\"LOCAL_SORT\",\"sort_columns\":\"testinttype3_c1\","
+        + "\"table_blocksize\":\"1024\",\"comment\":\"\"}},\"properties\":"
+        + "{\"QUERYTYPE\":\"AGGREGATION\",\"CHILD_SELECT QUERY\":"
+        + "\"c2VsZWN0IGMxLCBzdW0oYzIpIGZyb20gdGVzdGludHR5cGUzIGdyb3VwIGJ5IGMx\"}}],"
+        + "\"parentRelationIdentifiers\":[],\"isSchemaModified\":false}");
+    TableInfo tableInfo = CarbonUtil.convertGsonToTableInfo(properties);
+    // the schema evolution should not be null
+    assertTrue(null != tableInfo.getDataMapSchemaList());
+    List<DataMapSchema> dataMapSchemaList = tableInfo.getDataMapSchemaList();
+    for (DataMapSchema dataMapSchema : dataMapSchemaList) {
+      String providerName = dataMapSchema.getProviderName();
+      assertTrue("org.apache.carbondata.core.datamap.AggregateDataMap".equals(providerName));
     }
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc12de00/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 0bac9f7..6077d0e 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -136,7 +136,7 @@
                 <artifactItem>
                   <groupId>com.google.code.gson</groupId>
                   <artifactId>gson</artifactId>
-                  <version>2.3.1</version>
+                  <version>2.4</version>
                 </artifactItem>
                 <artifactItem>
                   <groupId>org.xerial.snappy</groupId>


Mime
View raw message