carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject carbondata git commit: [CARBONDATA-2295] Added property for UnsafeMemory configuration
Date Sat, 31 Mar 2018 08:44:21 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master f187856e3 -> fca960e37


[CARBONDATA-2295] Added property for UnsafeMemory configuration

Added property for setting up the unsafe memory limit in MB

This closes #2114


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fca960e3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fca960e3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fca960e3

Branch: refs/heads/master
Commit: fca960e372981e9dbb0cd048a4b55f3d2c9839dc
Parents: f187856
Author: Bhavya <bhavya@knoldus.com>
Authored: Thu Mar 29 20:33:34 2018 +0530
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Sat Mar 31 16:44:01 2018 +0800

----------------------------------------------------------------------
 integration/presto/README.md                             |  4 ++++
 .../apache/carbondata/presto/impl/CarbonTableConfig.java | 11 +++++++++++
 .../apache/carbondata/presto/impl/CarbonTableReader.java |  7 ++++++-
 .../apache/carbondata/presto/server/PrestoServer.scala   |  3 ++-
 4 files changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/README.md
----------------------------------------------------------------------
diff --git a/integration/presto/README.md b/integration/presto/README.md
index d264954..e1d4631 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -73,10 +73,14 @@ Please follow the below steps to query carbondata in presto
   ```
   connector.name=carbondata
   carbondata-store={schema-store-path}
+  carbon.unsafe.working.memory.in.mb={value}
   ```
   Replace the schema-store-path with the absolute path of the parent directory of the schema.
   For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/,
   Then set carbondata-store=hdfs://namenode:9000/test/carbondata
+  
+  carbon.unsafe.working.memory.in.mb property defines the limit for Unsafe Memory usage in
Mega Bytes. Replace the value for it 
+  as per your need, if your tables are big you can increase the unsafe memory. The default
value is 512 MB.
 
   If you updated the jar balls or configuration files, make sure you have dispatched them
    to all the presto nodes and restarted the presto servers on the nodes. The updates will
not take effect before restarting.

http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
index 677cefd..73f9d2c 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
@@ -30,6 +30,7 @@ public class CarbonTableConfig {
   private String dbPath;
   private String tablePath;
   private String storePath;
+  private String unsafeMemoryInMb;
 
   @NotNull public String getDbPath() {
     return dbPath;
@@ -57,4 +58,14 @@ public class CarbonTableConfig {
     this.storePath = storePath;
     return this;
   }
+
+  public String getUnsafeMemoryInMb() {
+    return unsafeMemoryInMb;
+  }
+
+  @Config("carbon.unsafe.working.memory.in.mb")
+  public CarbonTableConfig setUnsafeMemoryInMb(String unsafeMemoryInMb) {
+    this.unsafeMemoryInMb = unsafeMemoryInMb;
+    return this;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 09389f8..78dbbec 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datamap.DataMapStoreManager;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -42,6 +43,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
 import org.apache.carbondata.core.reader.ThriftReader;
 import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.hadoop.CarbonInputSplit;
 import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -366,7 +368,10 @@ public class CarbonTableReader {
   public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel,
                                                      Expression filters)  {
     List<CarbonLocalInputSplit> result = new ArrayList<>();
-
+    if(config.getUnsafeMemoryInMb() != null) {
+      CarbonProperties.getInstance().addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+          config.getUnsafeMemoryInMb());
+    }
     CarbonTable carbonTable = tableCacheModel.carbonTable;
     TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
     Configuration config = new Configuration();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index d6d3617..127c4c9 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -69,7 +69,8 @@ object PrestoServer {
     Try {
       queryRunner.installPlugin(new CarbondataPlugin)
       val carbonProperties = ImmutableMap.builder[String, String]
-        .put("carbondata-store", carbonStorePath).build
+        .put("carbondata-store", carbonStorePath)
+        .put("carbon.unsafe.working.memory.in.mb", "512").build
 
       // CreateCatalog will create a catalog for CarbonData in etc/catalog.
       queryRunner.createCatalog(CARBONDATA_CATALOG, CARBONDATA_CONNECTOR, carbonProperties)


Mime
View raw message