accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [1/9] accumulo git commit: ACCUMULO-1055: created TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest
Date Fri, 26 Aug 2016 19:37:07 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.8 b08c4b911 -> a8881acfb
  refs/heads/master ee1062f2b -> 90e2db545


ACCUMULO-1055: created TABLE_MINC_MAX_MERGE_FILE_SIZE and DatafileManagerTest


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3ca2fde1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3ca2fde1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3ca2fde1

Branch: refs/heads/1.8
Commit: 3ca2fde1feaeff9b1a83fd3505fd4d90b133a26e
Parents: 271800c
Author: milleruntime <michaelpmiller@gmail.com>
Authored: Thu Aug 18 08:19:21 2016 -0400
Committer: milleruntime <michaelpmiller@gmail.com>
Committed: Tue Aug 23 12:05:59 2016 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/core/conf/Property.java |   2 +
 .../tserver/tablet/DatafileManager.java         |   8 +-
 .../apache/accumulo/tserver/tablet/Tablet.java  |   3 +-
 .../tserver/tablet/DatafileManagerTest.java     | 111 +++++++++++++++++++
 4 files changed, 120 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 5f4f715..9cb331a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -444,6 +444,8 @@ public enum Property {
   TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
       "After a tablet has been idle (no mutations) for this time period it may have its "
           + "in-memory map flushed to disk in a minor compaction. There is no guarantee an
idle " + "tablet will be compacted."),
+  TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "50M", PropertyType.MEMORY,
+      "The max file size used for a merging minor compaction."),
   TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
       "The maximum amount of memory that will be used to cache results of a client query/scan.
"
           + "Once this limit is reached, the buffered data is sent to the client."),

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index b488e13..cef1482 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -58,15 +58,17 @@ class DatafileManager {
   // access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
   private final Map<FileRef,DataFileValue> datafileSizes = Collections.synchronizedMap(new
TreeMap<FileRef,DataFileValue>());
   private final Tablet tablet;
+  private final Long maxMergingMinorCompactionFileSize;
 
   // ensure we only have one reader/writer of our bulk file notes at at time
   private final Object bulkFileImportLock = new Object();
 
-  DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes) {
+  DatafileManager(Tablet tablet, SortedMap<FileRef,DataFileValue> datafileSizes, Long
maxMergCompFileSize) {
     for (Entry<FileRef,DataFileValue> datafiles : datafileSizes.entrySet()) {
       this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
     }
     this.tablet = tablet;
+    this.maxMergingMinorCompactionFileSize = maxMergCompFileSize;
   }
 
   private FileRef mergingMinorCompactionFile = null;
@@ -296,11 +298,11 @@ class DatafileManager {
     if (datafileSizes.size() >= maxFiles) {
       // find the smallest file
 
-      long min = Long.MAX_VALUE;
+      long min = maxMergingMinorCompactionFileSize;
       FileRef minName = null;
 
       for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) {
-        if (entry.getValue().getSize() < min && !majorCompactingFiles.contains(entry.getKey()))
{
+        if (entry.getValue().getSize() <= min && !majorCompactingFiles.contains(entry.getKey()))
{
           min = entry.getValue().getSize();
           minName = entry.getKey();
         }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 6637521..4377b64 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -484,7 +484,8 @@ public class Tablet implements TabletCommitter {
 
     // do this last after tablet is completely setup because it
     // could cause major compaction to start
-    datafileManager = new DatafileManager(this, datafiles);
+    Long maxFileSize = AccumuloConfiguration.getMemoryInBytes(tableConfiguration.get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE));
+    datafileManager = new DatafileManager(this, datafiles, maxFileSize);
 
     computeNumEntries();
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ca2fde1/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
new file mode 100644
index 0000000..0e9dee9
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.tablet;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.fs.FileRef;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for org.apache.accumulo.tserver.tablet.DatafileManager
+ */
+public class DatafileManagerTest {
+  private Tablet tablet;
+  private KeyExtent extent;
+  private TableConfiguration tableConf;
+
+  private SortedMap<FileRef,DataFileValue> createFileMap(String... sa) {
+    SortedMap<FileRef,DataFileValue> ret = new TreeMap<>();
+    for (int i = 0; i < sa.length; i += 2) {
+      ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i
+ 1]), 1));
+    }
+    return ret;
+  }
+
+  @Before
+  public void setupMockClasses() {
+    tablet = EasyMock.createMock(Tablet.class);
+    extent = EasyMock.createMock(KeyExtent.class);
+    tableConf = EasyMock.createMock(TableConfiguration.class);
+
+    EasyMock.expect(tablet.getExtent()).andReturn(extent);
+    EasyMock.expect(tablet.getTableConfiguration()).andReturn(tableConf);
+    EasyMock.expect(tableConf.getMaxFilesPerTablet()).andReturn(5);
+
+    EasyMock.replay(tablet, tableConf);
+  }
+
+  /*
+   * Test max file size (table.compaction.minor.merge.file.size.max) exceeded when calling
reserveMergingMinorCompactionFile
+   */
+  @Test
+  public void testReserveMergingMinorCompactionFile_MaxExceeded() throws IOException {
+    Long maxMergeFileSize = Long.valueOf(1000);
+    SortedMap<FileRef,DataFileValue> testFiles = createFileMap("largefile", "10M",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+    EasyMock.verify(tablet, tableConf);
+
+    assertEquals(null, mergeFile);
+  }
+
+  /*
+   * Test max files not reached (table.file.max) when calling reserveMergingMinorCompactionFile
+   */
+  @Test
+  public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() throws IOException
{
+    Long maxMergeFileSize = Long.valueOf(1000);
+    SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M");
+
+    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+    EasyMock.verify(tablet, tableConf);
+
+    assertEquals(null, mergeFile);
+  }
+
+  /*
+   * Test the smallest file is chosen for merging minor compaction
+   */
+  @Test
+  public void testReserveMergingMinorCompactionFile() throws IOException {
+    Long maxMergeFileSize = Long.valueOf(1000);
+    SortedMap<FileRef,DataFileValue> testFiles = createFileMap("smallfile", "100B",
"file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+
+    DatafileManager dfm = new DatafileManager(tablet, testFiles, maxMergeFileSize);
+    FileRef mergeFile = dfm.reserveMergingMinorCompactionFile();
+
+    EasyMock.verify(tablet, tableConf);
+
+    assertEquals("smallfile", mergeFile.path().getName());
+  }
+
+}


Mime
View raw message