hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zhang...@apache.org
Subject [11/41] hbase git commit: HBASE-19528 Major Compaction Tool; ADDENDUM
Date Fri, 02 Feb 2018 08:53:18 GMT
HBASE-19528 Major Compaction Tool; ADDENDUM

Add @ClassRule to new tests and rename those added here so they
align w/ how we name other Test classes.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/60827fc1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/60827fc1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/60827fc1

Branch: refs/heads/HBASE-19064
Commit: 60827fc1ea93ce3c93ea9b86e618e419babed42f
Parents: 4b3b627
Author: Michael Stack <stack@apache.org>
Authored: Wed Jan 31 14:18:20 2018 -0800
Committer: Michael Stack <stack@apache.org>
Committed: Wed Jan 31 14:18:24 2018 -0800

----------------------------------------------------------------------
 .../compaction/MajorCompactionRequestTest.java  | 166 ------------------
 .../util/compaction/MajorCompactorTest.java     |  81 ---------
 .../compaction/TestMajorCompactionRequest.java  | 170 +++++++++++++++++++
 .../util/compaction/TestMajorCompactor.java     |  86 ++++++++++
 4 files changed, 256 insertions(+), 247 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
deleted file mode 100644
index c5ce4e3..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.util.compaction;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.RegionInfoBuilder;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-@Category({SmallTests.class})
-public class MajorCompactionRequestTest {
-
-  private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility();
-  private static final String FAMILY = "a";
-  private Path rootRegionDir;
-  private Path regionStoreDir;
-
-  @Before public void setUp() throws Exception {
-    rootRegionDir = UTILITY.getDataTestDirOnTestFS("MajorCompactionRequestTest");
-    regionStoreDir = new Path(rootRegionDir, FAMILY);
-  }
-
-  @Test public void testStoresNeedingCompaction() throws Exception {
-    // store files older than timestamp
-    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
-    MajorCompactionRequest request = makeMockRequest(100, storeFiles, false);
-    Optional<MajorCompactionRequest> result =
-        request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
-    assertTrue(result.isPresent());
-
-    // store files newer than timestamp
-    storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
-    request = makeMockRequest(100, storeFiles, false);
-    result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
-    assertFalse(result.isPresent());
-  }
-
-  @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
-    // this tests that reference files that are new, but have older timestamps for the files
-    // they reference still will get compacted.
-    TableName table = TableName.valueOf("MajorCompactorTest");
-    TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
-    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
-    HRegion region =
-        HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(),
htd);
-
-    Configuration configuration = mock(Configuration.class);
-    // the reference file timestamp is newer
-    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
-    List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
-    // the files that are referenced are older, thus we still compact.
-    HRegionFileSystem fileSystem =
-        mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
-    MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
-        region.getRegionInfo(), Sets.newHashSet(FAMILY), 100));
-    doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
-    doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
-        any(Path.class));
-    doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
-    Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"));
-    assertEquals(FAMILY, Iterables.getOnlyElement(result));
-  }
-
-  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
-      List<StoreFileInfo> storeFiles) throws IOException {
-    long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
-    return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
-  }
-
-  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
-      List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException
{
-    FileSystem fileSystem = mock(FileSystem.class);
-    if (hasReferenceFiles) {
-      FileStatus fileStatus = mock(FileStatus.class);
-      doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
-      doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
-    }
-    HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
-    doReturn(info).when(mockSystem).getRegionInfo();
-    doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
-    doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
-    doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
-    doReturn(fileSystem).when(mockSystem).getFileSystem();
-    return mockSystem;
-  }
-
-  private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long
timestamp)
-      throws IOException {
-    List<StoreFileInfo> infos = Lists.newArrayList();
-    int i = 0;
-    while (i < howMany) {
-      StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
-      doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
-      doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
-          .getPath();
-      infos.add(storeFileInfo);
-      i++;
-    }
-    return infos;
-  }
-
-  private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo>
storeFiles,
-      boolean references) throws IOException {
-    Configuration configuration = mock(Configuration.class);
-    RegionInfo regionInfo = mock(RegionInfo.class);
-    when(regionInfo.getEncodedName()).thenReturn("HBase");
-    when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
-    MajorCompactionRequest request =
-        new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp);
-    MajorCompactionRequest spy = spy(request);
-    HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
-    doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
-    doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
-    return spy;
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
deleted file mode 100644
index 3fb37ec..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util.compaction;
-
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.junit.After;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-
-@Category({ MiscTests.class, MediumTests.class })
-public class MajorCompactorTest {
-
-  public static final byte[] FAMILY = Bytes.toBytes("a");
-  private HBaseTestingUtility utility;
-
-  @Before public void setUp() throws Exception {
-    utility = new HBaseTestingUtility();
-    utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10);
-    utility.startMiniCluster();
-  }
-
-  @After public void tearDown() throws Exception {
-    utility.shutdownMiniCluster();
-  }
-
-  @Test public void testCompactingATable() throws Exception {
-    TableName tableName = TableName.valueOf("MajorCompactorTest");
-    utility.createMultiRegionTable(tableName, FAMILY, 5);
-    utility.waitTableAvailable(tableName);
-    Connection connection = utility.getConnection();
-    Table table = connection.getTable(tableName);
-    // write data and flush multiple store files:
-    for (int i = 0; i < 5; i++) {
-      utility.loadRandomRows(table, FAMILY, 50, 100);
-      utility.flush(tableName);
-    }
-    table.close();
-    int numberOfRegions = utility.getAdmin().getRegions(tableName).size();
-    int numHFiles = utility.getNumHFiles(tableName, FAMILY);
-    // we should have a table with more store files than we would before we major compacted.
-    assertTrue(numberOfRegions < numHFiles);
-
-    MajorCompactor compactor =
-        new MajorCompactor(utility.getConfiguration(), tableName,
-            Sets.newHashSet(Bytes.toString(FAMILY)), 1, System.currentTimeMillis(), 200);
-    compactor.initializeWorkQueues();
-    compactor.compactAllRegions();
-    compactor.shutdown();
-
-    // verify that the store has been completely major compacted.
-    numberOfRegions = utility.getAdmin().getRegions(tableName).size();
-    numHFiles = utility.getNumHFiles(tableName, FAMILY);
-    assertEquals(numHFiles, numberOfRegions);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
new file mode 100644
index 0000000..b626481
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.compaction;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@Category({SmallTests.class})
+public class TestMajorCompactionRequest {
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestMajorCompactionRequest.class);
+
+  private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility();
+  private static final String FAMILY = "a";
+  private Path rootRegionDir;
+  private Path regionStoreDir;
+
+  @Before public void setUp() throws Exception {
+    rootRegionDir = UTILITY.getDataTestDirOnTestFS("TestMajorCompactionRequest");
+    regionStoreDir = new Path(rootRegionDir, FAMILY);
+  }
+
+  @Test public void testStoresNeedingCompaction() throws Exception {
+    // store files older than timestamp
+    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
+    MajorCompactionRequest request = makeMockRequest(100, storeFiles, false);
+    Optional<MajorCompactionRequest> result =
+        request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
+    assertTrue(result.isPresent());
+
+    // store files newer than timestamp
+    storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
+    request = makeMockRequest(100, storeFiles, false);
+    result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
+    assertFalse(result.isPresent());
+  }
+
+  @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
+    // this tests that reference files that are new, but have older timestamps for the files
+    // they reference still will get compacted.
+    TableName table = TableName.valueOf("TestMajorCompactor");
+    TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
+    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
+    HRegion region =
+        HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(),
htd);
+
+    Configuration configuration = mock(Configuration.class);
+    // the reference file timestamp is newer
+    List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
+    List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
+    // the files that are referenced are older, thus we still compact.
+    HRegionFileSystem fileSystem =
+        mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
+    MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
+        region.getRegionInfo(), Sets.newHashSet(FAMILY), 100));
+    doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
+    doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
+        any(Path.class));
+    doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
+    Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"));
+    assertEquals(FAMILY, Iterables.getOnlyElement(result));
+  }
+
+  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
+      List<StoreFileInfo> storeFiles) throws IOException {
+    long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
+    return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
+  }
+
+  private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
+      List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException
{
+    FileSystem fileSystem = mock(FileSystem.class);
+    if (hasReferenceFiles) {
+      FileStatus fileStatus = mock(FileStatus.class);
+      doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
+      doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
+    }
+    HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
+    doReturn(info).when(mockSystem).getRegionInfo();
+    doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
+    doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
+    doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
+    doReturn(fileSystem).when(mockSystem).getFileSystem();
+    return mockSystem;
+  }
+
+  private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long
timestamp)
+      throws IOException {
+    List<StoreFileInfo> infos = Lists.newArrayList();
+    int i = 0;
+    while (i < howMany) {
+      StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
+      doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
+      doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
+          .getPath();
+      infos.add(storeFileInfo);
+      i++;
+    }
+    return infos;
+  }
+
+  private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo>
storeFiles,
+      boolean references) throws IOException {
+    Configuration configuration = mock(Configuration.class);
+    RegionInfo regionInfo = mock(RegionInfo.class);
+    when(regionInfo.getEncodedName()).thenReturn("HBase");
+    when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
+    MajorCompactionRequest request =
+        new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp);
+    MajorCompactionRequest spy = spy(request);
+    HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
+    doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
+    doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
+    return spy;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
new file mode 100644
index 0000000..ccf0146
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.compaction;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestMajorCompactor {
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestMajorCompactor.class);
+
+  public static final byte[] FAMILY = Bytes.toBytes("a");
+  private HBaseTestingUtility utility;
+
+  @Before public void setUp() throws Exception {
+    utility = new HBaseTestingUtility();
+    utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10);
+    utility.startMiniCluster();
+  }
+
+  @After public void tearDown() throws Exception {
+    utility.shutdownMiniCluster();
+  }
+
+  @Test public void testCompactingATable() throws Exception {
+    TableName tableName = TableName.valueOf("TestMajorCompactor");
+    utility.createMultiRegionTable(tableName, FAMILY, 5);
+    utility.waitTableAvailable(tableName);
+    Connection connection = utility.getConnection();
+    Table table = connection.getTable(tableName);
+    // write data and flush multiple store files:
+    for (int i = 0; i < 5; i++) {
+      utility.loadRandomRows(table, FAMILY, 50, 100);
+      utility.flush(tableName);
+    }
+    table.close();
+    int numberOfRegions = utility.getAdmin().getRegions(tableName).size();
+    int numHFiles = utility.getNumHFiles(tableName, FAMILY);
+    // we should have a table with more store files than we would before we major compacted.
+    assertTrue(numberOfRegions < numHFiles);
+
+    MajorCompactor compactor =
+        new MajorCompactor(utility.getConfiguration(), tableName,
+            Sets.newHashSet(Bytes.toString(FAMILY)), 1, System.currentTimeMillis(), 200);
+    compactor.initializeWorkQueues();
+    compactor.compactAllRegions();
+    compactor.shutdown();
+
+    // verify that the store has been completely major compacted.
+    numberOfRegions = utility.getAdmin().getRegions(tableName).size();
+    numHFiles = utility.getNumHFiles(tableName, FAMILY);
+    assertEquals(numHFiles, numberOfRegions);
+  }
+}
\ No newline at end of file


Mime
View raw message