accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [4/9] accumulo git commit: ACCUMULO-1055: made default for max MINC merge file size 0
Date Fri, 26 Aug 2016 19:37:10 GMT
ACCUMULO-1055: made default for max MINC merge file size 0


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/792277e3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/792277e3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/792277e3

Branch: refs/heads/master
Commit: 792277e361e24bdbc417da1522206352b1a8a61d
Parents: 3ca2fde
Author: milleruntime <michaelpmiller@gmail.com>
Authored: Wed Aug 24 13:53:48 2016 -0400
Committer: milleruntime <michaelpmiller@gmail.com>
Committed: Wed Aug 24 13:53:48 2016 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java |  4 ++--
 .../tserver/tablet/DatafileManager.java         |  6 +++++-
 .../tserver/tablet/DatafileManagerTest.java     | 22 +++++++++++++++++---
 3 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9cb331a..ce725fb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,8 +444,8 @@ public enum Property {
   TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
       "After a tablet has been idle (no mutations) for this time period it may have its "
           + "in-memory map flushed to disk in a minor compaction. There is no guarantee an
idle " + "tablet will be compacted."),
-  TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
-      "The max file size used for a merging minor compaction."),
+  TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "0", PropertyType.MEMORY,
+      "The max file size used for a merging minor compaction. The default value of 0 disables
a max file size."),
   TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
       "The maximum amount of memory that will be used to cache results of a client query/scan.
"
           + "Once this limit is reached, the buffered data is sent to the client."),

http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index cef1482..60ad194 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -298,7 +298,11 @@ class DatafileManager {
     if (datafileSizes.size() >= maxFiles) {
       // find the smallest file
 
-      long min = maxMergingMinorCompactionFileSize;
+      long maxFileSize = Long.MAX_VALUE;
+      if (maxMergingMinorCompactionFileSize > 0) {
+        maxFileSize = maxMergingMinorCompactionFileSize;
+      }
+      long min = maxFileSize;
       FileRef minName = null;
 
       for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/792277e3/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index 0e9dee9..bb1d6e8 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -65,7 +65,7 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
-    Long maxMergeFileSize = Long.valueOf(1000);
+    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
 
     DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -81,7 +81,7 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException
{
-    Long maxMergeFileSize = Long.valueOf(1000);
+    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M");
 
     DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -97,7 +97,7 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile() throws IOException {
-    Long maxMergeFileSize = Long.valueOf(1000);
+    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
 
     DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
@@ -108,4 +108,20 @@ public class DatafileManagerTest {
     assertEquals("smallfile", mergeFile.path().getName());
   }
 
+  /*
+   * Test disabled max file size for merging minor compaction
+   */
+  @Test
+  public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
+    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+    SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+    EasyMock.verify(tablet, tableConf);
+
+    assertEquals("smallishfile", mergeFile.path().getName());
+  }
+
 }


Mime
View raw message