carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/9] carbondata git commit: add EncodingStrategy
Date Tue, 13 Jun 2017 17:04:50 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 3a63bd249 -> eb9dbfa67


http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
index 8ee08c4..56ee762 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
@@ -18,6 +18,7 @@
 package org.apache.carbondata.processing.store.writer;
 
 import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedData;
 import org.apache.carbondata.core.util.NodeHolder;
 import org.apache.carbondata.processing.store.TablePageKey;
 import org.apache.carbondata.processing.store.TablePageStatistics;
@@ -28,7 +29,7 @@ public interface CarbonFactDataWriter<T> {
    * This method will be used to create NodeHolder for a table page
    */
 
-  NodeHolder buildDataNodeHolder(Encoder.EncodedData encoded, TablePageStatistics stats,
+  NodeHolder buildDataNodeHolder(EncodedData encoded, TablePageStatistics stats,
       TablePageKey key) throws CarbonDataWriterException;
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/processing/src/main/java/org/apache/carbondata/processing/store/writer/Encoder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/Encoder.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/Encoder.java
deleted file mode 100644
index c2d0214..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/Encoder.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.store.writer;
-
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
-import org.apache.carbondata.processing.store.TablePage;
-
-public interface Encoder {
-
-  EncodedData encode(TablePage tablePage);
-
-  // result result of all columns
-  class EncodedData {
-    // dimension data that include rowid (index)
-    public IndexStorage[] indexStorages;
-
-    // encoded and compressed dimension data
-    public byte[][] dimensions;
-
-    // encoded and compressed measure data
-    public byte[][] measures;
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index 389af4e..bf35cd9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.columnar.ColGroupBlockStorage;
 import org.apache.carbondata.core.datastore.columnar.IndexStorage;
 import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedData;
 import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
 import org.apache.carbondata.core.util.CarbonMetadataUtil;
 import org.apache.carbondata.core.util.NodeHolder;
@@ -36,7 +37,6 @@ import org.apache.carbondata.processing.store.TablePageKey;
 import org.apache.carbondata.processing.store.TablePageStatistics;
 import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
 import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
-import org.apache.carbondata.processing.store.writer.Encoder;
 
 public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
 
@@ -48,7 +48,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]>
{
   }
 
   @Override
-  public NodeHolder buildDataNodeHolder(Encoder.EncodedData encoded,
+  public NodeHolder buildDataNodeHolder(EncodedData encoded,
       TablePageStatistics stats, TablePageKey key)
       throws CarbonDataWriterException {
     // if there are no NO-Dictionary column present in the table then

http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index df9523a..b4bc040 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
 import org.apache.carbondata.core.datastore.columnar.ColGroupBlockStorage;
 import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedData;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
 import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
@@ -42,7 +43,6 @@ import org.apache.carbondata.processing.store.TablePageKey;
 import org.apache.carbondata.processing.store.TablePageStatistics;
 import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
 import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
-import org.apache.carbondata.processing.store.writer.Encoder;
 
 /**
  * Below class will be used to write the data in V3 format
@@ -82,7 +82,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
    * This node holder object will be used to persist data which will
    * be written in carbon data file
    */
-  @Override public NodeHolder buildDataNodeHolder(Encoder.EncodedData encoded,
+  @Override public NodeHolder buildDataNodeHolder(EncodedData encoded,
       TablePageStatistics stats, TablePageKey key) throws CarbonDataWriterException {
     // if there are no NO-Dictionary column present in the table then
     // set the empty byte array
@@ -111,6 +111,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
     for (int i = 0; i < numDimensions; i++) {
       isSortedData[i] = encoded.indexStorages[i].isAlreadySorted();
       keyLengths[i] = encoded.dimensions[i].length;
+      totalKeySize += keyLengths[i];
       if (!isSortedData[i]) {
         dataAfterCompression[i] =
             getByteArray((short[])encoded.indexStorages[i].getDataAfterComp());


Mime
View raw message