carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-1872][API] Clean up unused constant in CarbonCommonConstant
Date Tue, 19 Dec 2017 17:39:16 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master b8a02f391 -> b0d4de94e


[CARBONDATA-1872][API] Clean up unused constant in CarbonCommonConstant

This closes #1631


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b0d4de94
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b0d4de94
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b0d4de94

Branch: refs/heads/master
Commit: b0d4de94ebf6bd9f5851c14f0d4a11bf2d74da23
Parents: b8a02f3
Author: Jacky Li <jacky.likun@qq.com>
Authored: Thu Dec 7 19:40:45 2017 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Tue Dec 19 23:08:58 2017 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   | 161 +------------------
 .../core/mutate/CarbonUpdateUtil.java           |   1 -
 .../core/statusmanager/LoadMetadataDetails.java |  19 ---
 .../sql/test/Spark2TestQueryExecutor.scala      |   6 +-
 .../store/CarbonFactDataHandlerColumnar.java    |  53 ------
 5 files changed, 6 insertions(+), 234 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f67b0c5..b8a998c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -46,14 +46,6 @@ public final class CarbonCommonConstants {
    * byte to KB conversion factor
    */
   public static final int BYTE_TO_KB_CONVERSION_FACTOR = 1024;
-  /**
-   * BYTE_ENCODING
-   */
-  public static final String BYTE_ENCODING = "ISO-8859-1";
-  /**
-   * measure meta data file name
-   */
-  public static final String MEASURE_METADATA_FILE_NAME = "/msrMetaData_";
 
   /**
    * set the segment ids to query from the table
@@ -179,22 +171,6 @@ public final class CarbonCommonConstants {
 
   public static final int BYTEBUFFER_SIZE = 24 * 1024;
   /**
-   * Average constant
-   */
-  public static final String AVERAGE = "avg";
-  /**
-   * Count constant
-   */
-  public static final String COUNT = "count";
-  /**
-   * SUM
-   */
-  public static final String SUM = "sum";
-  /**
-   * DUMMY aggregation function
-   */
-  public static final String DUMMY = "dummy";
-  /**
    * MEMBER_DEFAULT_VAL
    */
   public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
@@ -234,18 +210,10 @@ public final class CarbonCommonConstants {
    */
   public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL = "";
   /**
-   * HIERARCHY_FILE_EXTENSION
-   */
-  public static final String HIERARCHY_FILE_EXTENSION = ".hierarchy";
-  /**
    * SORT_TEMP_FILE_LOCATION
    */
   public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
   /**
-   * CARBON_RESULT_SIZE_DEFAULT
-   */
-  public static final String LEVEL_FILE_EXTENSION = ".level";
-  /**
    * FACT_FILE_EXT
    */
   public static final String FACT_FILE_EXT = ".carbondata";
@@ -260,35 +228,6 @@ public final class CarbonCommonConstants {
    */
   public static final String UPDATE_DELTA_FILE_EXT = FACT_FILE_EXT;
   /**
-   * MEASUREMETADATA_FILE_EXT
-   */
-  public static final String MEASUREMETADATA_FILE_EXT = ".msrmetadata";
-  /**
-   * GRAPH_ROWSET_SIZE
-   */
-  @CarbonProperty
-  public static final String GRAPH_ROWSET_SIZE = "carbon.graph.rowset.size";
-  /**
-   * GRAPH_ROWSET_SIZE_DEFAULT
-   */
-  public static final String GRAPH_ROWSET_SIZE_DEFAULT = "500";
-  /**
-   * Comment for <code>TYPE_MYSQL</code>
-   */
-  public static final String TYPE_MYSQL = "MYSQL";
-  /**
-   * Comment for <code>TYPE_MSSQL</code>
-   */
-  public static final String TYPE_MSSQL = "MSSQL";
-  /**
-   * Comment for <code>TYPE_ORACLE</code>
-   */
-  public static final String TYPE_ORACLE = "ORACLE";
-  /**
-   * Comment for <code>TYPE_SYBASE</code>
-   */
-  public static final String TYPE_SYBASE = "SYBASE";
-  /**
    * SORT_INTERMEDIATE_FILES_LIMIT
    */
   @CarbonProperty
@@ -298,10 +237,6 @@ public final class CarbonCommonConstants {
    */
   public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
   /**
-   * BAD_RECORD_KEY_VALUE
-   */
-  public static final String BAD_RECORD_KEY = "BADRECORD";
-  /**
    * MERGERD_EXTENSION
    */
   public static final String MERGERD_EXTENSION = ".merge";
@@ -311,15 +246,6 @@ public final class CarbonCommonConstants {
   @CarbonProperty
   public static final String SORT_FILE_BUFFER_SIZE = "carbon.sort.file.buffer.size";
   /**
-   * no.of records after which counter to be printed
-   */
-  @CarbonProperty
-  public static final String DATA_LOAD_LOG_COUNTER = "carbon.load.log.counter";
-  /**
-   * DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER
-   */
-  public static final String DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER = "500000";
-  /**
    * SORT_FILE_WRITE_BUFFER_SIZE
    */
   @CarbonProperty
@@ -405,18 +331,6 @@ public final class CarbonCommonConstants {
    */
   public static final String SEMICOLON_SPC_CHARACTER = ";#!@:SEMIC:@!#;";
   /**
-   * AMPERSAND_SPC_CHARATER
-   */
-  public static final String AMPERSAND_SPC_CHARACTER = "&#!@:AMPER:@!#&";
-  /**
-   * ATTHERATE_SPC_CHARATER
-   */
-  public static final String COMA_SPC_CHARACTER = ",#!:COMA:!#,";
-  /**
-   * HYPHEN_SPC_CHARACTER
-   */
-  public static final String HYPHEN_SPC_CHARACTER = "-#!:HYPHEN:!#-";
-  /**
    * SORT_TEMP_FILE_EXT
    */
   public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
@@ -485,27 +399,10 @@ public final class CarbonCommonConstants {
   @CarbonProperty
   public static final String STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location";
   /**
-   * IS_COLUMNAR_STORAGE_DEFAULTVALUE
-   */
-  public static final String IS_COLUMNAR_STORAGE_DEFAULTVALUE = "true";
-  /**
-   * DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE
-   */
-  public static final String DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE = "1";
-  /**
    * IS_FULLY_FILLED_BITS_DEFAULT_VALUE
    */
   public static final String IS_FULLY_FILLED_BITS_DEFAULT_VALUE = "true";
   /**
-   * IS_INT_BASED_INDEXER
-   */
-  @CarbonProperty
-  public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK = "aggregate.columnar.keyblock";
-  /**
-   * IS_INT_BASED_INDEXER_DEFAULTVALUE
-   */
-  public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE = "true";
-  /**
    * ENABLE_QUERY_STATISTICS
    */
   @CarbonProperty
@@ -541,10 +438,6 @@ public final class CarbonCommonConstants {
    */
   public static final String UNDERSCORE = "_";
   /**
-   * DASH
-   */
-  public static final String DASH = "-";
-  /**
    * POINT
    */
   public static final String POINT = ".";
@@ -621,16 +514,6 @@ public final class CarbonCommonConstants {
   public static final String TIMESTAMP = "Timestamp";
   public static final String ARRAY = "array";
   public static final String STRUCT = "struct";
-  public static final String FROM = "from";
-  /**
-   * FACT_UPDATE_EXTENSION.
-   */
-  public static final String FACT_UPDATE_EXTENSION = ".carbondata_update";
-  public static final String FACT_DELETE_EXTENSION = "_delete";
-  /**
-   * MARKED_FOR_UPDATION
-   */
-  public static final String FACT_FILE_UPDATED = "update";
   /**
    * MAX_QUERY_EXECUTION_TIME
    */
@@ -663,17 +546,11 @@ public final class CarbonCommonConstants {
    */
   @CarbonProperty
   public static final String CARBON_MAX_DRIVER_LRU_CACHE_SIZE = "carbon.max.driver.lru.cache.size";
-  public static final String POSITION_REFERENCE = "positionReference";
   /**
    * implicit column which will be added to each carbon table
    */
   public static final String POSITION_ID = "positionId";
   /**
-   * max driver lru cache size upto which lru cache will be loaded in memory
-   */
-  @CarbonProperty
-  public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size";
-  /**
    * max executor lru cache size upto which lru cache will be loaded in memory
    */
   @CarbonProperty
@@ -1027,16 +904,6 @@ public final class CarbonCommonConstants {
   public static final String UPDATE_INDEX_FILE_EXT = ".carbonindex";
 
   /**
-   * Key word for true
-   */
-  public static final String KEYWORD_TRUE = "TRUE";
-
-  /**
-   * Key word for false
-   */
-  public static final String KEYWORD_FALSE = "FALSE";
-
-  /**
    * hyphen
    */
   public static final String HYPHEN = "-";
@@ -1047,11 +914,6 @@ public final class CarbonCommonConstants {
   public static final String UPDATED_COL_EXTENSION = "-updatedColumn";
 
   /**
-   * appending the key to differentiate the update flow with insert flow.
-   */
-  public static final String RDDUTIL_UPDATE_KEY = "UPDATE_";
-
-  /**
    * to determine to use the rdd persist or not.
    */
   @CarbonProperty
@@ -1099,10 +961,6 @@ public final class CarbonCommonConstants {
   public static final String CARBON_DATA_FILE_DEFAULT_VERSION = "V3";
 
   /**
-   * data file version header
-   */
-  public static final String CARBON_DATA_VERSION_HEADER = "CARBONDATAVERSION#";
-  /**
    * Maximum no of column supported
    */
   public static final int DEFAULT_MAX_NUMBER_OF_COLUMNS = 20000;
@@ -1200,12 +1058,13 @@ public final class CarbonCommonConstants {
   @CarbonProperty
   public static final String LOAD_BATCH_SORT_SIZE_INMB = "carbon.load.batch.sort.size.inmb";
   public static final String LOAD_BATCH_SORT_SIZE_INMB_DEFAULT = "0";
-  @CarbonProperty
+
   /**
    * The Number of partitions to use when shuffling data for sort. If user don't configurate
or
    * configurate it less than 1, it uses the number of map tasks as reduce tasks. In general,
we
    * recommend 2-3 tasks per CPU core in your cluster.
    */
+  @CarbonProperty
   public static final String LOAD_GLOBAL_SORT_PARTITIONS = "carbon.load.global.sort.partitions";
 
   public static final String LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT = "0";
@@ -1232,11 +1091,6 @@ public final class CarbonCommonConstants {
   public static final String IS_DRIVER_INSTANCE = "is.driver.instance";
 
   /**
-   * maximum length of column
-   */
-  public static final int DEFAULT_COLUMN_LENGTH = 100000;
-
-  /**
    * property for enabling unsafe based query processing
    */
   @CarbonProperty
@@ -1248,17 +1102,6 @@ public final class CarbonCommonConstants {
   public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false";
 
   /**
-   * property for offheap based processing
-   */
-  @CarbonProperty
-  public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing";
-
-  /**
-   * default value of offheap based processing
-   */
-  public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true";
-
-  /**
    * whether to prefetch data while loading.
    */
   @CarbonProperty

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index f4566ac..aeaa205 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -231,7 +231,6 @@ public class CarbonUpdateUtil {
               // if this call is coming from the delete delta flow then the time stamp
               // String will come empty then no need to write into table status file.
               if (isTimestampUpdationRequired) {
-                loadMetadata.setIsDeleted(CarbonCommonConstants.KEYWORD_TRUE);
                 // if in case of update flow.
                 if (loadMetadata.getUpdateDeltaStartTimestamp().isEmpty()) {
                   // this means for first time it is getting updated .

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 85602bc..e2bcc8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -63,7 +63,6 @@ public class LoadMetadataDetails implements Serializable {
   // partition count of this segment
   private String partitionCount;
 
-  private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE;
   private String dataSize;
   private String indexSize;
 
@@ -339,24 +338,6 @@ public class LoadMetadataDetails implements Serializable {
   }
 
   /**
-   * To get isDeleted property.
-   *
-   * @return isDeleted
-   */
-  public String getIsDeleted() {
-    return isDeleted;
-  }
-
-  /**
-   * To set isDeleted property.
-   *
-   * @param isDeleted
-   */
-  public void setIsDeleted(String isDeleted) {
-    this.isDeleted = isDeleted;
-  }
-
-  /**
    * To get the update delta end timestamp
    *
    * @return updateDeltaEndTimestamp

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
index 3fed15d..e668de7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
@@ -40,11 +40,13 @@ class Spark2TestQueryExecutor extends TestQueryExecutorRegister {
 }
 
 object Spark2TestQueryExecutor {
+
+  private val STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location"
+
   private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
   LOGGER.info("use TestQueryExecutorImplV2")
   CarbonProperties.getInstance()
-    .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
-      System.getProperty("java.io.tmpdir"))
+    .addProperty(STORE_LOCATION_TEMP_PATH, System.getProperty("java.io.tmpdir"))
     .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FORCE")
 
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 75fcea3..ac31fcb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -38,8 +38,6 @@ import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
 import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
 import org.apache.carbondata.core.datastore.row.CarbonRow;
 import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator;
 import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -47,7 +45,6 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonThreadFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.processing.datatypes.GenericDataType;
 import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
 import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
@@ -371,22 +368,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler
{
     }
     LOGGER.info("Number of rows per column blocklet " + pageSize);
     dataRows = new ArrayList<>(this.pageSize);
-    int dimSet =
-        Integer.parseInt(CarbonCommonConstants.DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE);
-    // if at least one dimension is present then initialize column splitter otherwise null
-    int noOfColStore = colGrpModel.getNoOfColumnStore();
-    int[] keyBlockSize = new int[noOfColStore + getExpandedComplexColsCount()];
-
-    if (model.getDimLens().length > 0) {
-      //Using Variable length variable split generator
-      //This will help in splitting mdkey to columns. variable split is required because
all
-      // columns which are part of
-      //row store will be in single column store
-      //e.g if {0,1,2,3,4,5} is dimension and {0,1,2) is row store dimension
-      //than below splitter will return column as {0,1,2}{3}{4}{5}
-      ColumnarSplitter columnarSplitter = model.getSegmentProperties().getFixedLengthKeySplitter();
-      System.arraycopy(columnarSplitter.getBlockKeySize(), 0, keyBlockSize, 0, noOfColStore);
-    }
 
     // agg type
     List<Integer> otherMeasureIndexList =
@@ -411,46 +392,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler
{
       customMeasureIndex[i] = customMeasureIndexList.get(i);
     }
     setComplexMapSurrogateIndex(model.getDimensionCount());
-    int[] blockKeySize = getBlockKeySizeWithComplexTypes(new MultiDimKeyVarLengthEquiSplitGenerator(
-        CarbonUtil.getIncrementedCardinalityFullyFilled(model.getDimLens().clone()), (byte)
dimSet)
-        .getBlockKeySize());
-    System.arraycopy(blockKeySize, noOfColStore, keyBlockSize, noOfColStore,
-        blockKeySize.length - noOfColStore);
     this.dataWriter = getFactDataWriter();
     // initialize the channel;
     this.dataWriter.initializeWriter();
   }
 
   /**
-   * This method combines primitive dimensions with complex metadata columns
-   *
-   * @param primitiveBlockKeySize
-   * @return all dimensions cardinality including complex dimension metadata column
-   */
-  private int[] getBlockKeySizeWithComplexTypes(int[] primitiveBlockKeySize) {
-    int allColsCount = getExpandedComplexColsCount();
-    int[] blockKeySizeWithComplexTypes =
-        new int[this.colGrpModel.getNoOfColumnStore() + allColsCount];
-
-    List<Integer> blockKeySizeWithComplex =
-        new ArrayList<Integer>(blockKeySizeWithComplexTypes.length);
-    int dictDimensionCount = model.getDimensionCount();
-    for (int i = 0; i < dictDimensionCount; i++) {
-      GenericDataType complexDataType = model.getComplexIndexMap().get(i);
-      if (complexDataType != null) {
-        complexDataType.fillBlockKeySize(blockKeySizeWithComplex, primitiveBlockKeySize);
-      } else {
-        blockKeySizeWithComplex.add(primitiveBlockKeySize[i]);
-      }
-    }
-    for (int i = 0; i < blockKeySizeWithComplexTypes.length; i++) {
-      blockKeySizeWithComplexTypes[i] = blockKeySizeWithComplex.get(i);
-    }
-
-    return blockKeySizeWithComplexTypes;
-  }
-
-  /**
    * Below method will be used to get the fact data writer instance
    *
    * @return data writer instance


Mime
View raw message