asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wangs...@apache.org
Subject asterixdb git commit: ASTERIXDB-1569: the config file now supports unit, comma, and decimal.
Date Fri, 12 Aug 2016 02:16:41 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 95f5b3fb0 -> bddc81106


ASTERIXDB-1569: the config file now supports unit, comma, and decimal.

 - This patch fixes an issue that the config file only supports byte
   value for the byte required parameters. Now, we can use unit (KB, MB, GB ...),
   comma, and decimal.

Change-Id: I763d1b06e93f75775c0294da0d609c2bf3e102c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1064
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
Reviewed-by: Till Westmann <tillw@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/bddc8110
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/bddc8110
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/bddc8110

Branch: refs/heads/master
Commit: bddc811068f42795c3b07c20cf027270bdd83762
Parents: 95f5b3f
Author: Taewoo Kim <wangsaeu@yahoo.com>
Authored: Thu Aug 11 16:33:46 2016 -0700
Committer: Taewoo Kim <wangsaeu@yahoo.com>
Committed: Thu Aug 11 19:16:21 2016 -0700

----------------------------------------------------------------------
 .../resources/asterix-build-configuration.xml   |  12 +-
 .../config/AsterixCompilerProperties.java       |   8 +-
 .../common/config/AsterixFeedProperties.java    |   2 +-
 .../config/AsterixReplicationProperties.java    |   4 +-
 .../common/config/AsterixStorageProperties.java |   8 +-
 .../config/AsterixTransactionProperties.java    |   6 +-
 .../common/config/PropertyInterpreters.java     |  28 ++++
 .../docker/asterix-configuration.xml            |  18 +--
 .../configs/asterix-configuration.xml           |  20 +--
 .../resources/conf/asterix-configuration.xml    |  24 ++--
 .../integrationts/asterix-configuration.xml     |  30 ++--
 .../resources/base-asterix-configuration.xml    |  22 +--
 .../configs/base-asterix-configuration.xml      |  22 +--
 .../org/apache/hyracks/util/StorageUtil.java    | 142 ++++++++++++++++---
 14 files changed, 242 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
index 1d693f9..cbd5c93 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -52,19 +52,19 @@
   </property>
   <property>
     <name>compiler.framesize</name>
-    <value>32768</value>
+    <value>32KB</value>
   </property>
   <property>
     <name>compiler.sortmemory</name>
-    <value>327680</value>
+    <value>320KB</value>
   </property>
   <property>
     <name>compiler.groupmemory</name>
-    <value>163840</value>
+    <value>160KB</value>
   </property>
   <property>
     <name>compiler.joinmemory</name>
-    <value>163840</value>
+    <value>160KB</value>
   </property>
   <property>
     <name>compiler.pregelix.home</name>
@@ -72,14 +72,14 @@
   </property>
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>32768</value>
+    <value>32KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "32768" // 32KB)
     </description>
   </property>
   <property>
     <name>storage.buffercache.size</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "33554432" // 32MB)

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
index 42f612a..5bfb5f8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java
@@ -40,22 +40,22 @@ public class AsterixCompilerProperties extends AbstractAsterixProperties
{
 
     public long getSortMemorySize() {
         return accessor.getProperty(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public long getJoinMemorySize() {
         return accessor.getProperty(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public long getGroupMemorySize() {
         return accessor.getProperty(COMPILER_GROUPMEMORY_KEY, COMPILER_GROUPMEMORY_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public int getFrameSize() {
         return accessor.getProperty(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 
     public String getPregelixHome() {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
index e4d2913..0f3951e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixFeedProperties.java
@@ -41,7 +41,7 @@ public class AsterixFeedProperties extends AbstractAsterixProperties {
 
     public long getMemoryComponentGlobalBudget() {
         return accessor.getProperty(FEED_MEMORY_GLOBALBUDGET_KEY, FEED_MEMORY_GLOBALBUDGET_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public long getMemoryAvailableWaitTimeout() {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
index 7f51bbd..eec4947 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixReplicationProperties.java
@@ -260,7 +260,7 @@ public class AsterixReplicationProperties extends AbstractAsterixProperties
{
 
     public int getLogBufferPageSize() {
         return accessor.getProperty(REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY, REPLICATION_LOG_BUFFER_PAGE_SIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 
     public int getLogBufferNumOfPages() {
@@ -270,6 +270,6 @@ public class AsterixReplicationProperties extends AbstractAsterixProperties
{
 
     public int getLogBatchSize() {
         return accessor.getProperty(REPLICATION_LOG_BATCH_SIZE_KEY, REPLICATION_LOG_BATCH_SIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
index 9fb4e94..5fdbc1c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
@@ -55,12 +55,12 @@ public class AsterixStorageProperties extends AbstractAsterixProperties
{
 
     public int getBufferCachePageSize() {
         return accessor.getProperty(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 
     public long getBufferCacheSize() {
         return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public int getBufferCacheNumPages() {
@@ -74,7 +74,7 @@ public class AsterixStorageProperties extends AbstractAsterixProperties
{
 
     public int getMemoryComponentPageSize() {
         return accessor.getProperty(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 
     public int getMemoryComponentNumPages() {
@@ -96,7 +96,7 @@ public class AsterixStorageProperties extends AbstractAsterixProperties
{
 
     public long getMemoryComponentGlobalBudget() {
         return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY,
-                STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongPropertyInterpreter());
+                STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public double getBloomFilterFalsePositiveRate() {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
index 356dad3..00bcdc9 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixTransactionProperties.java
@@ -81,12 +81,12 @@ public class AsterixTransactionProperties extends AbstractAsterixProperties
{
 
     public int getLogBufferPageSize() {
         return accessor.getProperty(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT,
-                PropertyInterpreters.getIntegerPropertyInterpreter());
+                PropertyInterpreters.getIntegerBytePropertyInterpreter());
     }
 
     public long getLogPartitionSize() {
         return accessor.getProperty(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 
     public int getCheckpointLSNThreshold() {
@@ -131,6 +131,6 @@ public class AsterixTransactionProperties extends AbstractAsterixProperties
{
 
     public long getJobRecoveryMemorySize() {
         return accessor.getProperty(TXN_JOB_RECOVERY_MEMORY_SIZE_KEY, TXN_JOB_RECOVERY_MEMORY_SIZE_DEFAULT,
-                PropertyInterpreters.getLongPropertyInterpreter());
+                PropertyInterpreters.getLongBytePropertyInterpreter());
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
index b54bcc3..4dd07d8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertyInterpreters.java
@@ -20,6 +20,8 @@ package org.apache.asterix.common.config;
 
 import java.util.logging.Level;
 
+import org.apache.hyracks.util.StorageUtil;
+
 public class PropertyInterpreters {
 
     public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() {
@@ -88,4 +90,30 @@ public class PropertyInterpreters {
         };
     }
 
+    public static IPropertyInterpreter<Long> getLongBytePropertyInterpreter() {
+        return new IPropertyInterpreter<Long>() {
+            @Override
+            public Long interpret(String s) throws IllegalArgumentException {
+                try {
+                    return StorageUtil.getByteValue(s);
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException(e);
+                }
+            }
+        };
+    }
+
+    public static IPropertyInterpreter<Integer> getIntegerBytePropertyInterpreter()
{
+        return new IPropertyInterpreter<Integer>() {
+            @Override
+            public Integer interpret(String s) throws IllegalArgumentException {
+                try {
+                    return (int) StorageUtil.getByteValue(s);
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException(e);
+                }
+            }
+        };
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-docker/docker/asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-docker/docker/asterix-configuration.xml b/asterixdb/asterix-docker/docker/asterix-configuration.xml
index dd8d571..ea43da9 100644
--- a/asterixdb/asterix-docker/docker/asterix-configuration.xml
+++ b/asterixdb/asterix-docker/docker/asterix-configuration.xml
@@ -66,7 +66,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -74,7 +74,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -91,7 +91,7 @@
 
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -122,7 +122,7 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
@@ -146,7 +146,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>524288</value>
+    <value>512KB</value>
     <description>The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -154,7 +154,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -221,7 +221,7 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to sort operations.
       (Default = "33554432" // 32mb)
     </description>
@@ -229,7 +229,7 @@
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to join operations.
       (Default = "33554432" // 32mb)
     </description>
@@ -237,7 +237,7 @@
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
index af37d0b..c642cbb 100644
--- a/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
+++ b/asterixdb/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
@@ -37,7 +37,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -45,7 +45,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>3221225472</value>
+    <value>3GB</value>
     <description>[3GB] The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -63,7 +63,7 @@
   <!-- Buffer size per dataset for in-memory components: 1GB -->
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -95,8 +95,8 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>4399824896</value>
-    <description>[4GB + 8MB]The total size of memory in bytes that the sum of all
+    <value>4196MB</value>
+    <description>[4GB + 100MB]The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
     </description>
@@ -119,7 +119,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>8388608</value>
+    <value>8MB</value>
     <description>[8MB]The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -127,7 +127,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -194,7 +194,7 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>67108864</value>
+    <value>64MB</value>
     <description>[64MB]The amount of memory in bytes given to sort operations.
       (Default = "33554432" // 32mb)
     </description>
@@ -202,7 +202,7 @@
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>67108864</value>
+    <value>64MB</value>
     <description>[64MB]The amount of memory in bytes given to join operations.
       (Default = "33554432" // 32mb)
     </description>
@@ -210,7 +210,7 @@
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
index f6da341..a7c5c6d 100644
--- a/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
+++ b/asterixdb/asterix-installer/src/main/resources/conf/asterix-configuration.xml
@@ -41,7 +41,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -49,7 +49,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -66,7 +66,7 @@
 
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -98,7 +98,7 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>1073741824</value>
+    <value>1GB</value>
     <description>The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>524288</value>
+    <value>512KB</value>
     <description>The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -130,7 +130,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -197,7 +197,7 @@
 
   <property>
     <name>txn.job.recovery.memorysize</name>
-    <value>67108864</value>
+    <value>64MB</value>
     <description>The memory allocated per job during recovery.
      (Default = "67108864" // 64MB)
     </description>
@@ -205,23 +205,23 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to sort operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to join operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
index 4cd7fff..e36c33b 100644
--- a/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/asterix-configuration.xml
@@ -41,7 +41,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -49,7 +49,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -66,7 +66,7 @@
 
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -98,7 +98,7 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>1073741824</value>
+    <value>1GB</value>
     <description>The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>524288</value>
+    <value>512KB</value>
     <description>The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -130,7 +130,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -197,23 +197,31 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to sort operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to join operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
+    </description>
+  </property>
+
+  <property>
+    <name>compiler.groupmemory</name>
+    <value>32MB</value>
+    <description>The amount of memory in bytes given to group-by operations.
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
index 778e51f..788cb76 100644
--- a/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
+++ b/asterixdb/asterix-yarn/src/main/resources/base-asterix-configuration.xml
@@ -41,7 +41,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -49,7 +49,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -66,7 +66,7 @@
 
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -98,7 +98,7 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>524288</value>
+    <value>512KB</value>
     <description>The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -130,7 +130,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -197,23 +197,23 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to sort operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to join operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
index 778e51f..788cb76 100644
--- a/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
+++ b/asterixdb/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml
@@ -41,7 +41,7 @@
 
   <property>
     <name>storage.buffercache.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages in the buffer cache.
       (Default = "131072" // 128KB)
     </description>
@@ -49,7 +49,7 @@
 
   <property>
     <name>storage.buffercache.size</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The size of memory allocated to the disk buffer cache.
       The value should be a multiple of the buffer cache page size(Default
       = "536870912" // 512MB)
@@ -66,7 +66,7 @@
 
   <property>
     <name>storage.memorycomponent.pagesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The page size in bytes for pages allocated to memory
       components. (Default = "131072" // 128KB)
     </description>
@@ -98,7 +98,7 @@
 
   <property>
     <name>storage.memorycomponent.globalbudget</name>
-    <value>536870912</value>
+    <value>512MB</value>
     <description>The total size of memory in bytes that the sum of all
       open memory
       components cannot exceed. (Default = "536870192" // 512MB)
@@ -122,7 +122,7 @@
 
   <property>
     <name>txn.log.buffer.pagesize</name>
-    <value>524288</value>
+    <value>512KB</value>
     <description>The size of pages in the in-memory log buffer. (Default =
       "524288" // 512KB)
     </description>
@@ -130,7 +130,7 @@
 
   <property>
     <name>txn.log.partitionsize</name>
-    <value>2147483648</value>
+    <value>2GB</value>
     <description>The maximum size of a log file partition allowed before
       rotating the log to the next partition. (Default = "2147483648" //
       2GB)
@@ -197,23 +197,23 @@
 
   <property>
     <name>compiler.sortmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to sort operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.joinmemory</name>
-    <value>33554432</value>
+    <value>32MB</value>
     <description>The amount of memory in bytes given to join operations.
-      (Default = "33554432" // 32mb)
+      (Default = "33554432" // 32MB)
     </description>
   </property>
 
   <property>
     <name>compiler.framesize</name>
-    <value>131072</value>
+    <value>128KB</value>
     <description>The Hyracks frame size that the compiler configures per
       job. (Default = "131072" // 128KB)
     </description>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bddc8110/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
index 4607214..7025f15 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/StorageUtil.java
@@ -23,12 +23,23 @@ public class StorageUtil {
     private static final int BASE = 1024, KB = BASE, MB = KB * BASE, GB = MB * BASE, TB =
GB * BASE, PB = TB * BASE;
 
     public enum StorageUnit {
-        BYTE,
-        KILOBYTE,
-        MEGABYTE,
-        GIGABYTE,
-        TERABYTE,
-        PETABYTE
+        BYTE("B"),
+        KILOBYTE("KB"),
+        MEGABYTE("MB"),
+        GIGABYTE("GB"),
+        TERABYTE("TB"),
+        PETABYTE("PB");
+
+        private final String unitTypeInLetter;
+
+        private StorageUnit(String unitTypeInLetter) {
+            this.unitTypeInLetter = unitTypeInLetter;
+        }
+
+        @Override
+        public String toString() {
+            return this.unitTypeInLetter;
+        }
     }
 
     private StorageUtil() {
@@ -36,23 +47,24 @@ public class StorageUtil {
     }
 
     public static int getSizeInBytes(final int size, final StorageUnit unit) {
-        switch (unit) {
-            case BYTE:
-                return size;
-            case KILOBYTE:
-                return size * KB;
-            case MEGABYTE:
-                return size * MB;
-            case GIGABYTE:
-                return size * GB;
-            case TERABYTE:
-                return size * TB;
-            default:
-                throw new IllegalStateException("Unsupported unti: " + unit);
+        double result = getSizeInBytes((double) size, unit);
+        if (result > Integer.MAX_VALUE) {
+            throw new IllegalStateException("The given value:" + result + " is not within
the integer range.");
+        } else {
+            return (int) result;
         }
     }
 
     public static long getSizeInBytes(final long size, final StorageUnit unit) {
+        double result = getSizeInBytes((double) size, unit);
+        if (result > Long.MAX_VALUE) {
+            throw new IllegalStateException("The given value:" + result + " is not within
the long range.");
+        } else {
+            return (long) result;
+        }
+    }
+
+    public static double getSizeInBytes(final double size, final StorageUnit unit) {
         switch (unit) {
             case BYTE:
                 return size;
@@ -67,7 +79,87 @@ public class StorageUtil {
             case PETABYTE:
                 return size * PB;
             default:
-                throw new IllegalStateException("Unsupported unti: " + unit);
+                throw new IllegalStateException("Unsupported unit: " + unit);
+        }
+    }
+
+    /**
+     * Helper method to parse a byte unit string to its double value and unit
+     * (e.g., 10,345.8MB becomes Pair<10345.8, StorageUnit.MB>.)
+     *
+     * @throws HyracksException
+     */
+    public static ByteValueStringInfo parseByteUnitString(String s) {
+        String sSpaceRemoved = s.replaceAll(" ", "");
+        String sUpper = sSpaceRemoved.toUpperCase();
+        ByteValueStringInfo valueAndUnitType = new ByteValueStringInfo();
+
+        // Default type
+        StorageUtil.StorageUnit unitType = StorageUnit.BYTE;
+
+        // If the length is 1, it should only contain a digit number.
+        if (sUpper.length() == 1) {
+            if (Character.isDigit(sUpper.charAt(0))) {
+                unitType = StorageUnit.BYTE;
+            } else {
+                throw new IllegalStateException(
+                        "The given string: " + s + " is not a byte unit string (e.g., 320KB
or 1024).");
+            }
+        } else if (sUpper.length() > 1) {
+            String checkStr = sUpper.substring(sUpper.length() - 2);
+            boolean found = false;
+            for (StorageUnit unit : StorageUnit.values()) {
+                if (checkStr.equals(unit.toString())) {
+                    unitType = unit;
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found) {
+                // The last suffix should be at least "B" or a digit to be qualified as byte
unit string.
+                char lastChar = sUpper.charAt(sUpper.length() - 1);
+                if (sUpper.substring(sUpper.length() - 1).equals(StorageUnit.BYTE.toString())
+                        || Character.isDigit(lastChar)) {
+                    unitType = StorageUnit.BYTE;
+                } else {
+                    throw new IllegalStateException(
+                            "The given string: " + s + " is not a byte unit string (e.g.,
320KB or 1024).");
+                }
+            }
+        } else {
+            // String length is zero. We can't parse this string.
+            throw new IllegalStateException(
+                    "The given string: " + s + " is not a byte unit string (e.g., 320KB or
1024).");
+        }
+
+        // Strip all unit suffixes such as KB, MB ...
+        String sFinalVal = sUpper.replaceAll("[^\\.0123456789]", "");
+
+        // Return the digit and its unit type.
+        valueAndUnitType.value = Double.parseDouble(sFinalVal);
+        valueAndUnitType.unitType = unitType;
+
+        return valueAndUnitType;
+    }
+
+    /**
+     * Return byte value for the given string (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...)
+     *
+     * @throws HyracksException
+     */
+    public static long getByteValue(String s) {
+        try {
+            ByteValueStringInfo valueAndUnitType = parseByteUnitString(s);
+
+            double result = getSizeInBytes(valueAndUnitType.value, valueAndUnitType.unitType);
+            if (result > Long.MAX_VALUE) {
+                throw new IllegalStateException("The given value:" + result + " is not within
the long range.");
+            } else {
+                return (long) result;
+            }
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException(e);
         }
     }
 
@@ -85,4 +177,14 @@ public class StorageUtil {
         final int baseValue = (63 - Long.numberOfLeadingZeros(bytes)) / 10;
         return String.format("%.2f %sB", (double) bytes / (1L << (baseValue * 10)),
" kMGTPE".charAt(baseValue));
     }
+
+    private static class ByteValueStringInfo {
+        double value;
+        StorageUnit unitType;
+
+        ByteValueStringInfo() {
+            value = 0.0;
+            unitType = StorageUnit.BYTE;
+        }
+    }
 }


Mime
View raw message