asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject incubator-asterixdb git commit: ASTERIXDB-1455: Scale CharRecord Buffer Expansions
Date Wed, 18 May 2016 15:54:41 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master d7f4f0482 -> 5e1226637


ASTERIXDB-1455: Scale CharRecord Buffer Expansions

Reading a 30 MB record requires > 1500 buffer reallocations when increasing by
a fixed amount.  Updated to scale up the buffer increases to reduce the number
of reallocations in case of large records.

Change-Id: I4822b2a78e1f411de94d2c717523df6b2b4a381e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/861
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/5e122663
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/5e122663
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/5e122663

Branch: refs/heads/master
Commit: 5e12266374b24227bb726870ff93b44e07382443
Parents: d7f4f04
Author: Michael Blow <mblow@apache.org>
Authored: Tue May 17 23:30:03 2016 -0700
Committer: Michael Blow <michael.blow@couchbase.com>
Committed: Wed May 18 08:54:36 2016 -0700

----------------------------------------------------------------------
 .../apache/asterix/external/input/record/CharArrayRecord.java  | 6 ++----
 .../record/reader/hdfs/AbstractCharRecordLookupReader.java     | 2 +-
 .../apache/asterix/external/util/ExternalDataConstants.java    | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5e122663/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
index 6b8e10d..f174962 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/CharArrayRecord.java
@@ -69,10 +69,8 @@ public class CharArrayRecord implements IRawRecord<char[]> {
                 throw new IOException(
                         "Record is too large!. Maximum record size is " + ExternalDataConstants.MAX_RECORD_SIZE);
             }
-            int newSize = len + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT;
-            if (newSize > ExternalDataConstants.MAX_RECORD_SIZE) {
-                newSize = ExternalDataConstants.MAX_RECORD_SIZE;
-            }
+            int newSize = Math.min((int)(len * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR),
+                    ExternalDataConstants.MAX_RECORD_SIZE);
             value = Arrays.copyOf(value, newSize);
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5e122663/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
index 45d01e9..f9ec114 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/AbstractCharRecordLookupReader.java
@@ -64,7 +64,7 @@ public abstract class AbstractCharRecordLookupReader extends AbstractHDFSLookupR
         reusableByteBuffer.clear();
         if (reusableByteBuffer.remaining() < value.getLength()) {
             reusableByteBuffer = ByteBuffer
-                    .allocateDirect(value.getLength() + ExternalDataConstants.DEFAULT_BUFFER_INCREMENT);
+                    .allocateDirect((int)(value.getLength() * ExternalDataConstants.DEFAULT_BUFFER_INCREMENT_FACTOR));
         }
         reusableByteBuffer.put(value.getBytes(), 0, value.getLength());
         reusableByteBuffer.flip();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5e122663/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 0fd76cc..cf2b7f3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -206,7 +206,7 @@ public class ExternalDataConstants {
      * Size default values
      */
     public static final int DEFAULT_BUFFER_SIZE = 4096;
-    public static final int DEFAULT_BUFFER_INCREMENT = 2048;
+    public static final float DEFAULT_BUFFER_INCREMENT_FACTOR = 1.5F;
     public static final int DEFAULT_QUEUE_SIZE = 64;
     public static final int MAX_RECORD_SIZE = 32000000;
 


Mime
View raw message