accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [6/9] accumulo git commit: ACCUMULO-1055: changes for DFM to read TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method
Date Fri, 26 Aug 2016 19:37:12 GMT
ACCUMULO-1055: changes for DFM to read TABLE_MINC_MAX_MERGE_FILE_SIZE in reserve method


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff9a6035
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff9a6035
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff9a6035

Branch: refs/heads/master
Commit: ff9a603526aff49eb5d860fea1e263432b8636dc
Parents: 792277e
Author: Mike Miller <michaelpmiller@gmail.com>
Authored: Thu Aug 25 15:39:53 2016 -0400
Committer: Mike Miller <michaelpmiller@gmail.com>
Committed: Thu Aug 25 15:39:53 2016 -0400

----------------------------------------------------------------------
 .../tserver/tablet/DatafileManager.java         |  7 +++--
 .../apache/accumulo/tserver/tablet/Tablet.java  |  3 +-
 .../tserver/tablet/DatafileManagerTest.java     | 32 ++++++++++++++------
 3 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 60ad194..ac5243a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -31,6 +31,7 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -58,17 +59,16 @@ class DatafileManager {
   // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
   private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new
TreeMap<FileRef,DataFileValue>());
   private final Tablet tablet;
-  private final Long maxMergingMinorCompactionFileSize;
+  private Long maxMergingMinorCompactionFileSize;
 
   // ensure we only have one reader/writer of our bulk file notes at at time
   private final Object bulkFileImportLock = new Object();
 
-  DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long
maxMergCompFileSize) {
+  DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
     for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
       this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
     }
     this.tablet = tablet;
-    this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
   }
 
   private FileRef mergingMinorCompactionFile = null;
@@ -299,6 +299,7 @@ class DatafileManager {
       // find the smallest file
 
       long maxFileSize = Long.MAX_VALUE;
+      maxMergingMinorCompactionFileSize = AccumuloConfiguration.getMemoryInBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
       if (maxMergingMinorCompactionFileSize > 0) {
         maxFileSize = maxMergingMinorCompactionFileSize;
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 4377b64..6637521 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,8 +484,7 @@ public class Tablet implements TabletCommitter {
 
     // do this last after tablet is completely setup because it
     // could cause major compaction to start
-    Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
-    datafileManager = new DatafileManager(this, datafiles, maxFileSize);
+    datafileManager = new DatafileManager(this, datafiles);
 
     computeNumEntries();
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff9a6035/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index bb1d6e8..990d24d 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.impl.KeyExtent;
 import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.server.conf.TableConfiguration;
@@ -56,8 +57,6 @@ public class DatafileManagerTest {
     EasyMock.expect(tablet.getExtent()).andReturn(extent);
     EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
     EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
-
-    EasyMock.replay(tablet, tableConf);
   }
 
   /*
@@ -65,10 +64,14 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
-    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+    String maxMergeFileSize = "1000B";
+    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+    EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+    EasyMock.replay(tablet, tableConf);
+
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
 
-    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    DatafileManager dfm = new DatafileManager(tablet, testFiles);
     FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
 
     EasyMock.verify(tablet, tableConf);
@@ -81,10 +84,11 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException
{
-    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+    EasyMock.replay(tablet, tableConf);
+
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M");
 
-    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    DatafileManager dfm = new DatafileManager(tablet, testFiles);
     FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
 
     EasyMock.verify(tablet, tableConf);
@@ -97,10 +101,14 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFile() throws IOException {
-    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("1000B");
+    String maxMergeFileSize = "1000B";
+    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+    EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+    EasyMock.replay(tablet, tableConf);
+
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
 
-    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    DatafileManager dfm = new DatafileManager(tablet, testFiles);
     FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
 
     EasyMock.verify(tablet, tableConf);
@@ -113,10 +121,14 @@ public class DatafileManagerTest {
    */
   @Test
   public void testReserveMergingMinorCompactionFileDisabled() throws IOException {
-    Long maxMergeFileSize = AccumuloConfiguration.getMemoryInBytes("0");
+    String maxMergeFileSize = "0";
+    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+    EasyMock.expect(tableConf.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)).andReturn(maxMergeFileSize);
+    EasyMock.replay(tablet, tableConf);
+
     SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallishfile", "10M",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
 
-    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    DatafileManager dfm = new DatafileManager(tablet, testFiles);
     FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
 
     EasyMock.verify(tablet, tableConf);


Mime
View raw message