carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [29/50] [abbrv] incubator-carbondata git commit: Use map to store SegmentProperties and reuse SegmentProperties.
Date Tue, 02 May 2017 14:12:53 GMT
Use map to store SegmentProperties and reuse SegmentProperties.

Add comment.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/33a3858a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/33a3858a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/33a3858a

Branch: refs/heads/12-dev
Commit: 33a3858a8c28551dd37f4a530cb52d97d888f810
Parents: 58be73f
Author: Yadong Qi <qiyadong2010@gmail.com>
Authored: Fri Apr 21 16:21:04 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Apr 27 22:15:44 2017 -0700

----------------------------------------------------------------------
 .../core/datastore/SegmentTaskIndexStore.java   | 61 +++++++++++++++++++-
 .../core/datastore/block/SegmentTaskIndex.java  | 10 ++--
 .../schema/table/column/ColumnSchema.java       | 10 +++-
 .../datastore/block/SegmentTaskIndexTest.java   |  4 +-
 4 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 862455e..b1eaa84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastore;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -31,12 +32,14 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.cache.Cache;
 import org.apache.carbondata.core.cache.CarbonLRUCache;
 import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
 import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
 import org.apache.carbondata.core.datastore.block.TableBlockInfo;
 import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.mutate.UpdateVO;
 import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
 import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,6 +71,9 @@ public class SegmentTaskIndexStore
    */
   private Map<String, Object> segmentLockMap;
 
+  private Map<SegmentPropertiesWrapper, SegmentProperties> segmentProperties =
+      new HashMap<SegmentPropertiesWrapper, SegmentProperties>();
+
   /**
    * constructor to initialize the SegmentTaskIndexStore
    *
@@ -345,7 +351,26 @@ public class SegmentTaskIndexStore
     List<DataFileFooter> footerList = CarbonUtil
         .readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber,
             tableBlockInfoList, tableIdentifier);
-    AbstractIndex segment = new SegmentTaskIndex();
+
+    // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality
are
+    // the same.
+    List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
+    int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
+    SegmentPropertiesWrapper segmentPropertiesWrapper =
+        new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
+    SegmentProperties segmentProperties;
+    if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
+      segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
+    } else {
+      // create a metadata details
+      // this will be useful in query handling
+      // all the data file metadata will have common segment properties we
+      // can use first one to get create the segment properties
+      segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
+      this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
+    }
+
+    AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
     // file path of only first block is passed as it all table block info path of
     // same task id will be same
     segment.buildIndex(footerList);
@@ -396,4 +421,38 @@ public class SegmentTaskIndexStore
       return result;
     }
   }
+
+  /**
+   * This class wraps tableIdentifier, columnsInTable and columnCardinality as a key to determine
+   * whether the SegmentProperties object can be reused.
+   */
+  public static class SegmentPropertiesWrapper {
+    private AbsoluteTableIdentifier tableIdentifier;
+    private List<ColumnSchema> columnsInTable;
+    private int[] columnCardinality;
+
+    public SegmentPropertiesWrapper(AbsoluteTableIdentifier tableIdentifier,
+        List<ColumnSchema> columnsInTable, int[] columnCardinality) {
+      this.tableIdentifier = tableIdentifier;
+      this.columnsInTable = columnsInTable;
+      this.columnCardinality = columnCardinality;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof SegmentPropertiesWrapper)) {
+        return false;
+      }
+      SegmentPropertiesWrapper other = (SegmentPropertiesWrapper) obj;
+      return tableIdentifier.equals(other.tableIdentifier)
+        && columnsInTable.equals(other.columnsInTable)
+        && Arrays.equals(columnCardinality, other.columnCardinality);
+    }
+
+    @Override
+    public int hashCode() {
+      return tableIdentifier.hashCode()
+        + columnsInTable.hashCode() + Arrays.hashCode(columnCardinality);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
index 7293323..f577ede 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
@@ -29,17 +29,15 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
  */
 public class SegmentTaskIndex extends AbstractIndex {
 
+  public SegmentTaskIndex(SegmentProperties segmentProperties) {
+    this.segmentProperties = segmentProperties;
+  }
+
   /**
    * Below method is store the blocks in some data structure
    *
    */
   public void buildIndex(List<DataFileFooter> footerList) {
-    // create a metadata details
-    // this will be useful in query handling
-    // all the data file metadata will have common segment properties we
-    // can use first one to get create the segment properties
-    segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(),
-        footerList.get(0).getSegmentInfo().getColumnCardinality());
     // create a segment builder info
     // in case of segment create we do not need any file path and each column value size
     // as Btree will be build as per min max and start key

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 7aa0900..abe065c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -263,7 +263,8 @@ public class ColumnSchema implements Serializable {
   @Override public int hashCode() {
     final int prime = 31;
     int result = 1;
-    result = prime * result + ((columnName == null) ? 0 : columnName.hashCode());
+    result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()) +
+      ((dataType == null) ? 0 : dataType.hashCode());
     return result;
   }
 
@@ -288,6 +289,13 @@ public class ColumnSchema implements Serializable {
     } else if (!columnName.equals(other.columnName)) {
       return false;
     }
+    if (dataType == null) {
+      if (other.dataType != null) {
+        return false;
+      }
+    } else if (!dataType.equals(other.dataType)) {
+      return false;
+    }
     return true;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
index ac2cfaa..4c08c6b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
@@ -58,7 +58,6 @@ public class SegmentTaskIndexTest {
       @Mock public void build(BTreeBuilderInfo segmentBuilderInfos) {}
     };
     long numberOfRows = 100;
-    SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex();
     columnSchema.setColumnName("employeeName");
     columnSchemaList.add(new ColumnSchema());
 
@@ -68,6 +67,9 @@ public class SegmentTaskIndexTest {
     footer.setNumberOfRows(numberOfRows);
     footerList.add(footer);
 
+    SegmentProperties properties = new SegmentProperties(footerList.get(0).getColumnInTable(),
+        footerList.get(0).getSegmentInfo().getColumnCardinality());
+    SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex(properties);
     segmentTaskIndex.buildIndex(footerList);
     assertEquals(footerList.get(0).getNumberOfRows(), numberOfRows);
 


Mime
View raw message