From commits-return-10265-archive-asf-public=cust-asf.ponee.io@carbondata.apache.org Sat Mar 31 10:44:23 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id D94D218064A for ; Sat, 31 Mar 2018 10:44:22 +0200 (CEST) Received: (qmail 39874 invoked by uid 500); 31 Mar 2018 08:44:21 -0000 Mailing-List: contact commits-help@carbondata.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.apache.org Delivered-To: mailing list commits@carbondata.apache.org Received: (qmail 39865 invoked by uid 99); 31 Mar 2018 08:44:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 31 Mar 2018 08:44:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BBCC1EB4F4; Sat, 31 Mar 2018 08:44:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chenliang613@apache.org To: commits@carbondata.apache.org Message-Id: <5e0eca254d1d4616826672e5d506de36@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: carbondata git commit: [CARBONDATA-2295] Added property for UnsafeMemory configuration Date: Sat, 31 Mar 2018 08:44:21 +0000 (UTC) Repository: carbondata Updated Branches: refs/heads/master f187856e3 -> fca960e37 [CARBONDATA-2295] Added property for UnsafeMemory configuration Added property for setting up the unsafe memory limit in MB This closes #2114 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fca960e3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fca960e3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fca960e3 Branch: refs/heads/master Commit: fca960e372981e9dbb0cd048a4b55f3d2c9839dc Parents: f187856 Author: Bhavya Authored: Thu Mar 29 20:33:34 2018 +0530 Committer: chenliang613 Committed: Sat Mar 31 16:44:01 2018 +0800 ---------------------------------------------------------------------- integration/presto/README.md | 4 ++++ .../apache/carbondata/presto/impl/CarbonTableConfig.java | 11 +++++++++++ .../apache/carbondata/presto/impl/CarbonTableReader.java | 7 ++++++- .../apache/carbondata/presto/server/PrestoServer.scala | 3 ++- 4 files changed, 23 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/README.md ---------------------------------------------------------------------- diff --git a/integration/presto/README.md b/integration/presto/README.md index d264954..e1d4631 100644 --- a/integration/presto/README.md +++ b/integration/presto/README.md @@ -73,10 +73,14 @@ Please follow the below steps to query carbondata in presto ``` connector.name=carbondata carbondata-store={schema-store-path} + carbon.unsafe.working.memory.in.mb={value} ``` Replace the schema-store-path with the absolute path of the parent directory of the schema. For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/, Then set carbondata-store=hdfs://namenode:9000/test/carbondata + + carbon.unsafe.working.memory.in.mb property defines the limit for Unsafe Memory usage in Mega Bytes. Replace the value for it + as per your need, if your tables are big you can increase the unsafe memory. The default value is 512 MB. If you updated the jar balls or configuration files, make sure you have dispatched them to all the presto nodes and restarted the presto servers on the nodes. The updates will not take effect before restarting. http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java index 677cefd..73f9d2c 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java @@ -30,6 +30,7 @@ public class CarbonTableConfig { private String dbPath; private String tablePath; private String storePath; + private String unsafeMemoryInMb; @NotNull public String getDbPath() { return dbPath; @@ -57,4 +58,14 @@ public class CarbonTableConfig { this.storePath = storePath; return this; } + + public String getUnsafeMemoryInMb() { + return unsafeMemoryInMb; + } + + @Config("carbon.unsafe.working.memory.in.mb") + public CarbonTableConfig setUnsafeMemoryInMb(String unsafeMemoryInMb) { + this.unsafeMemoryInMb = unsafeMemoryInMb; + return this; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java index 09389f8..78dbbec 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java @@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datamap.DataMapStoreManager; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; @@ -42,6 +43,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.TableInfo; import org.apache.carbondata.core.reader.ThriftReader; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.hadoop.CarbonInputSplit; import org.apache.carbondata.hadoop.api.CarbonTableInputFormat; @@ -366,7 +368,10 @@ public class CarbonTableReader { public List getInputSplits2(CarbonTableCacheModel tableCacheModel, Expression filters) { List result = new ArrayList<>(); - + if(config.getUnsafeMemoryInMb() != null) { + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, + config.getUnsafeMemoryInMb()); + } CarbonTable carbonTable = tableCacheModel.carbonTable; TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo(); Configuration config = new Configuration(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/fca960e3/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala index d6d3617..127c4c9 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala @@ -69,7 +69,8 @@ object PrestoServer { Try { queryRunner.installPlugin(new CarbondataPlugin) val carbonProperties = ImmutableMap.builder[String, String] - .put("carbondata-store", carbonStorePath).build + .put("carbondata-store", carbonStorePath) + .put("carbon.unsafe.working.memory.in.mb", "512").build // CreateCatalog will create a catalog for CarbonData in etc/catalog. queryRunner.createCatalog(CARBONDATA_CATALOG, CARBONDATA_CONNECTOR, carbonProperties)