carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2716][DataMap] Add validation for datamap writer listener during data loading
Date Wed, 11 Jul 2018 08:00:49 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 641ec098f -> 57b457153


[CARBONDATA-2716][DataMap] Add validation for datamap writer listener during data loading

In some scenarios, while doing data loading, the loading will use the datamap writer listener
that does not belong to this table. Here we will validate
the tableName to make sure that the loading will only use the
corresponding listerners.

This closes #2471


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/57b45715
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/57b45715
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/57b45715

Branch: refs/heads/master
Commit: 57b457153090de145d3b323a877139fbd65baa5a
Parents: 641ec09
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Tue Jul 10 16:22:24 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Wed Jul 11 16:00:29 2018 +0800

----------------------------------------------------------------------
 .../carbondata/processing/datamap/DataMapWriterListener.java | 8 ++++++++
 .../store/writer/v3/CarbonFactDataWriterImplV3.java          | 8 ++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/57b45715/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 24fcf61..4207253 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
 import org.apache.carbondata.core.datamap.dev.DataMapWriter;
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.processing.store.TablePage;
@@ -48,6 +49,12 @@ public class DataMapWriterListener {
 
   // list indexed column -> list of data map writer
   private Map<List<CarbonColumn>, List<DataMapWriter>> registry = new ConcurrentHashMap<>();
+  // table for this listener
+  private CarbonTableIdentifier tblIdentifier;
+
+  public CarbonTableIdentifier getTblIdentifier() {
+    return tblIdentifier;
+  }
 
   /**
    * register all datamap writer for specified table and segment
@@ -62,6 +69,7 @@ public class DataMapWriterListener {
       throw new RuntimeException(e);
     }
     if (tableIndices != null) {
+      tblIdentifier = carbonTable.getCarbonTableIdentifier();
       for (TableDataMap tableDataMap : tableIndices) {
         // register it only if it is not lazy datamap, for lazy datamap, user
         // will rebuild the datamap manually

http://git-wip-us.apache.org/repos/asf/carbondata/blob/57b45715/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index e562f26..dc6e443 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -151,7 +151,9 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter
{
 
   private void addPageData(TablePage tablePage) throws IOException {
     blockletDataHolder.addPage(tablePage);
-    if (listener != null) {
+    if (listener != null &&
+        model.getDatabaseName().equalsIgnoreCase(listener.getTblIdentifier().getDatabaseName())
&&
+        model.getTableName().equalsIgnoreCase(listener.getTblIdentifier().getTableName()))
{
       if (pageId == 0) {
         listener.onBlockletStart(blockletId);
       }
@@ -185,7 +187,9 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter
{
         writeHeaderToFile();
       }
       writeBlockletToFile(dataChunkBytes);
-      if (listener != null) {
+      if (listener != null &&
+          model.getDatabaseName().equalsIgnoreCase(listener.getTblIdentifier().getDatabaseName())
&&
+          model.getTableName().equalsIgnoreCase(listener.getTblIdentifier().getTableName()))
{
         listener.onBlockletEnd(blockletId++);
       }
       pageId = 0;


Mime
View raw message