carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [14/50] [abbrv] incubator-carbondata git commit: [Bug] Added comments for carbon.properties.template and Carbonindex file cleanup fix (#749)
Date Thu, 30 Jun 2016 17:42:01 GMT
[Bug] Added comments for carbon.properties.template and Carbonindex file cleanup fix (#749)

* Added comments for carbon.properties.template
* clean files not cleaning .carbonindex files

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/720e8d61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/720e8d61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/720e8d61

Branch: refs/heads/master
Commit: 720e8d6152e91be7276be5a7c1317d17caf5b30b
Parents: 24c47c2
Author: nareshpr <prnaresh.naresh@gmail.com>
Authored: Sat Jun 25 19:53:02 2016 +0530
Committer: Venkata Ramana G <g.ramana.v@gmail.com>
Committed: Sat Jun 25 19:53:02 2016 +0530

----------------------------------------------------------------------
 conf/carbon.properties.template                 | 42 ++++++++++----------
 .../core/carbon/path/CarbonTablePath.java       | 14 +++++++
 .../core/constants/CarbonCommonConstants.java   | 12 +++---
 .../spark/load/DeleteLoadFolders.java           |  5 ++-
 .../store/SingleThreadFinalSortFilesMerger.java |  6 +--
 5 files changed, 47 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/conf/carbon.properties.template
----------------------------------------------------------------------
diff --git a/conf/carbon.properties.template b/conf/carbon.properties.template
index 314320c..8dd6598 100644
--- a/conf/carbon.properties.template
+++ b/conf/carbon.properties.template
@@ -10,13 +10,13 @@ carbon.kettle.home=$<SPARK_HOME>/carbonlib/carbonplugins
 
 #################### Performance Configuration ##################
 ######## DataLoading Configuration ########
-#File read buffer size used during sorting:MIN=:MAX=
+#File read buffer size used during sorting(in MB) :MIN=1:MAX=100
 carbon.sort.file.buffer.size=20
-#Rowset size exchanged between data load graph steps.:MIN=:MAX=
+#Rowset size exchanged between data load graph steps :MIN=500:MAX=1000000
 carbon.graph.rowset.size=100000
-#Number of cores to be used while data loading:MIN=:MAX=
+#Number of cores to be used while data loading
 carbon.number.of.cores.while.loading=6
-#CARBON sort size.:MIN=:MAX=
+#Record count to sort and write to temp intermediate files
 carbon.sort.size=500000
 #Algorithm for hashmap for hashkey calculation
 carbon.enableXXHash=true
@@ -27,43 +27,45 @@ carbon.enableXXHash=true
 #enable prefetch of data during merge sort while reading data from sort temp files in data
loading
 #carbon.merge.sort.prefetch=true
 ######## Compaction Configuration ########
-#Number of cores to be used while compacting:MIN=:MAX=
+#Number of cores to be used while compacting
 carbon.number.of.cores.while.compacting=2
-#default minor compaction in MBs
-carbon.minor.compaction.size=256
-#default major compaction in MBs
+#For minor compaction, Number of segments to be merged in stage 1, number of compacted segments
to be merged in stage 2.
+carbon.compaction.level.threshold=4,3
+#default size (in MB) for major compaction to be triggered
 carbon.major.compaction.size=1024
 ######## Query Configuration ########
-#Number of cores to be used.:MIN=:MAX=
+#Number of cores to be used while querying
 carbon.number.of.cores=4
-#Carbon Inmemory record size:MIN=:MAX=
-carbon.inmemory.record.size=100000
+#Number of records to be in memory while querying :MIN=100000:MAX=240000
+carbon.inmemory.record.size=120000
 #Improves the performance of filter query
 carbon.enable.quick.filter=false
+##number of core to load the blocks in driver
+#no.of.cores.to.load.blocks.in.driver=10
 
 #################### Extra Configuration ##################
 ##Timestamp format of input data used for timestamp data type.
 #carbon.timestamp.format=yyyy-MM-dd HH:mm:ss
 ######## Dataload Configuration ########
-######File write buffer size used during sorting.
+##File write buffer size used during sorting.
 #carbon.sort.file.write.buffer.size=10485760
+##Locking mechanism for data loading on a table
+#carbon.lock.type=LOCALLOCK
 ##Minimum no of intermediate files after which sort merged to be started.
 #carbon.sort.intermediate.files.limit=20
 ##space reserved in percentage for writing block meta data in carbon data file
 #carbon.block.meta.size.reserved.percentage=10
 ##csv reading buffer size.
 #carbon.csv.read.buffersize.byte=1048576
-##space reserved in percentage for writing block meta data in carbon data file
-#carbon.block.meta.size.reserved.percentage=10
-##High Cardinality value
+##To identify and apply compression for non-high cardinality columns
 #high.cardinality.value=100000
-##CARBON maximum no of threads used for sorting.
-#carbon.max.thread.for.sorting=3
+##maximum no of threads used for reading intermediate files for final merging.
+#carbon.merge.sort.reader.thread=3
 ##Carbon blocklet size. Note: this configuration cannot be change once store is generated
 #carbon.blocklet.size=120000
-##How to times retry to get the lock
+##number of retries to get the metadata lock for loading data to table
 #carbon.load.metadata.lock.retries=3
-##Maximum number of blocklets written in a single file.:Min=1:Max=1000
+##Maximum number of blocklets written in a single file :Min=1:Max=1000
 #carbon.max.file.size=100
 ##Interval between the retries to get the lock
 #carbon.load.metadata.lock.retry.timeout.sec=5
@@ -83,8 +85,6 @@ carbon.enable.quick.filter=false
 #max.query.execution.time=60
 ##Min max is feature added to enhance query performance. To disable this feature, make it
false.
 #carbon.enableMinMax=true
-##number of core to load the blocks in driver
-#no.of.cores.to.load.blocks.in.driver=10
 ######## Global Dictionary Configurations ########
 ##To enable/disable identify high cardinality during first data loading
 #high.cardinality.identify.enable=true

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
index 8dcd207..5f862bc 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
@@ -98,6 +98,20 @@ public class CarbonTablePath extends Path {
   }
 
   /**
+   * check if it is carbon index file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonIndexFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
+    }
+    return false;
+  }
+
+  /**
    * gets table path
    */
   public String getPath() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index a120d00..473e764 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -250,7 +250,7 @@ public final class CarbonCommonConstants {
   /**
    * SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
    */
-  public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "10";
+  public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
   /**
    * MERGERD_EXTENSION
    */
@@ -349,13 +349,13 @@ public final class CarbonCommonConstants {
    */
   public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
   /**
-   * CARBON_MAX_THREAD_FOR_SORTING
+   * CARBON_MERGE_SORT_READER_THREAD
    */
-  public static final String CARBON_MAX_THREAD_FOR_SORTING = "carbon.max.thread.for.sorting";
+  public static final String CARBON_MERGE_SORT_READER_THREAD = "carbon.merge.sort.reader.thread";
   /**
-   * CARBON_MAX_THREAD_FOR_SORTING
+   * CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE
    */
-  public static final String CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE = "2";
+  public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE = "3";
   /**
    * IS_SORT_TEMP_FILE_COMPRESSION_ENABLED
    */
@@ -645,7 +645,7 @@ public final class CarbonCommonConstants {
   /**
    * xxhash algorithm property for hashmap. Default value false
    */
-  public static final String ENABLE_XXHASH_DEFAULT = "false";
+  public static final String ENABLE_XXHASH_DEFAULT = "true";
 
   /**
    * default charset to be used for reading and writing

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
b/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
index 1bb48d4..01d785a 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
@@ -38,6 +38,7 @@ import java.util.List;
 import org.carbondata.common.logging.LogService;
 import org.carbondata.common.logging.LogServiceFactory;
 import org.carbondata.core.carbon.path.CarbonStorePath;
+import org.carbondata.core.carbon.path.CarbonTablePath;
 import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
 import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
@@ -180,8 +181,8 @@ public final class DeleteLoadFolders {
         CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
 
           @Override public boolean accept(CarbonFile file) {
-            return (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) || file.getName()
-                .endsWith(CarbonCommonConstants.MEASUREMETADATA_FILE_EXT));
+            return (CarbonTablePath.isCarbonDataFile(file.getName())
+                || CarbonTablePath.isCarbonIndexFile(file.getName()));
           }
         });
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
b/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
index 9096b07..85f7c54 100644
--- a/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
@@ -162,11 +162,11 @@ public class SingleThreadFinalSortFilesMerger {
     int maxThreadForSorting = 0;
     try {
       maxThreadForSorting = Integer.parseInt(CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING,
-              CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE));
+          .getProperty(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD,
+              CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE));
     } catch (NumberFormatException e) {
       maxThreadForSorting =
-          Integer.parseInt(CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE);
+          Integer.parseInt(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE);
     }
     ExecutorService service = Executors.newFixedThreadPool(maxThreadForSorting);
 


Mime
View raw message