hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject [1/2] hbase git commit: HBASE-15400 Use DateTieredCompactor for Date Tiered Compaction (Clara Xiong)
Date Sat, 09 Apr 2016 16:48:14 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 c480b2a4e -> 940bde52b


http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java
index 8afe621..21480d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompaction.java
@@ -1,5 +1,4 @@
 /**
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,9 +25,13 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
 import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionRequest;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -37,6 +40,10 @@ import org.junit.experimental.categories.Category;
 public class TestDateTieredCompaction extends TestCompactionPolicy {
   ArrayList<StoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[]
sizes)
       throws IOException {
+    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
+    EnvironmentEdgeManager.injectEdge(timeMachine);
+    // Has to be  > 0 and < now.
+    timeMachine.setValue(1);
     ArrayList<Long> ageInDisk = new ArrayList<Long>();
     for (int i = 0; i < sizes.length; i++) {
       ageInDisk.add(0L);
@@ -57,29 +64,47 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     super.config();
 
     // Set up policy
-    conf.setLong(CompactionConfiguration.MAX_AGE_KEY, 100);
+    conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
+      "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+    conf.setLong(CompactionConfiguration.MAX_AGE_MILLIS_KEY, 100);
     conf.setLong(CompactionConfiguration.INCOMING_WINDOW_MIN_KEY, 3);
     conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, 6);
     conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 4);
-    conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
-      DateTieredCompactionPolicy.class.getName());
+    conf.setBoolean(CompactionConfiguration.SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY, false);
 
     // Special settings for compaction policy per window
-    this.conf.setInt(CompactionConfiguration.MIN_KEY, 2);
-    this.conf.setInt(CompactionConfiguration.MAX_KEY, 12);
-    this.conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F);
+    conf.setInt(CompactionConfiguration.MIN_KEY, 2);
+    conf.setInt(CompactionConfiguration.MAX_KEY, 12);
+    conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F);
+    
+    conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
+    conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
   }
 
-  void compactEquals(long now, ArrayList<StoreFile> candidates, long... expected)
-      throws IOException {
-    Assert.assertTrue(((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
-        .needsCompaction(candidates, ImmutableList.<StoreFile> of(), now));
-
-    List<StoreFile> actual =
-        ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
-            .applyCompactionPolicy(candidates, false, false, now);
-
-    Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
+  void compactEquals(long now, ArrayList<StoreFile> candidates, long[] expectedFileSizes,
+      long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException {
+    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
+    EnvironmentEdgeManager.injectEdge(timeMachine);
+    timeMachine.setValue(now);
+    DateTieredCompactionRequest request;
+    if (isMajor) {
+      for (StoreFile file : candidates) {
+        ((MockStoreFile)file).setIsMajor(true);
+      }
+      Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
+        .shouldPerformMajorCompaction(candidates));      
+      request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
+          .getCompactionPolicy()).selectMajorCompaction(candidates);
+    } else {
+      Assert.assertEquals(toCompact, ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
+          .needsCompaction(candidates, ImmutableList.<StoreFile> of()));
+      request = (DateTieredCompactionRequest) ((DateTieredCompactionPolicy) store.storeEngine
+          .getCompactionPolicy()).selectMinorCompaction(candidates, false, false);
+    }
+    List<StoreFile> actual = Lists.newArrayList(request.getFiles());
+    Assert.assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual)));
+    Assert.assertEquals(Arrays.toString(expectedBoundaries),
+    Arrays.toString(request.getBoundaries().toArray()));
   }
 
   /**
@@ -92,7 +117,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
};
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13
};
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11,
12, 13 },
+      new long[] { Long.MIN_VALUE, 12 }, false, true);
   }
 
   /**
@@ -105,7 +131,22 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 25, 24, 23, 22, 21,
20);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21,
22, 23,
+        24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
+  }
+
+  /**
+   * Test for file on the upper bound of incoming window
+   * @throws IOException with error
+   */
+  @Test
+  public void OnUpperBoundOfIncomingWindow() throws IOException {
+    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18
};
+    long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13
};
+
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11,
12, 13 },
+      new long[] { Long.MIN_VALUE, 12 }, false, true);
   }
 
   /**
@@ -115,23 +156,25 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
   @Test
   public void NewerThanIncomingWindow() throws IOException {
     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18
};
+    long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19
};
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13
};
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11,
12, 13 },
+      new long[] { Long.MIN_VALUE, 12}, false, true);
   }
 
   /**
-   * If there is no T1 window, we don't build 2
+   * If there is no T1 window, we don't build T2
    * @throws IOException with error
    */
   @Test
   public void NoT2() throws IOException {
     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-    long[] maxTimestamps = new long[] { 44, 60, 61, 92, 95, 100 };
+    long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
     long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
 
-    compactEquals(100, sfCreate(minTimestamps, maxTimestamps, sizes), 23, 22);
+    compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23
},
+      new long[] { Long.MIN_VALUE, 96}, false, true);
   }
 
   @Test
@@ -140,7 +183,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157
};
     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
 
-    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 32, 31, 30);
+    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31,
32 },
+      new long[] { Long.MIN_VALUE, 120 }, false, true);
   }
 
   /**
@@ -153,7 +197,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 22, 21, 20);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21,
22 },
+      new long[] { Long.MIN_VALUE }, false, true);
   }
 
   /**
@@ -166,7 +211,8 @@ public class TestDateTieredCompaction extends TestCompactionPolicy {
     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157
};
     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
 
-    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30);
+    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31
},
+      new long[] { Long.MIN_VALUE }, false, true);
   }
 
   /**
@@ -179,33 +225,101 @@ public class TestDateTieredCompaction extends TestCompactionPolicy
{
     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 24, 23);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24
},
+      new long[] { Long.MIN_VALUE }, false, true);
   }
 
   /**
-  * Older than now(161) - maxAge(100)
-  * @throws IOException with error
-  */
- @Test
- public void olderThanMaxAge() throws IOException {
-   long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-   long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157
};
-   long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
-
-   compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30, 33, 42, 41,
40);
- }
+   * Older than now(161) - maxAge(100)
+   * @throws IOException with error
+   */
+  @Test
+  public void olderThanMaxAge() throws IOException {
+    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157
};
+    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
+
+    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41,
42, 33,
+        30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
+  }
 
   /**
    * Out-of-order data
    * @throws IOException with error
    */
   @Test
-  public void OutOfOrder() throws IOException {
+  public void outOfOrder() throws IOException {
     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
     long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
 
-    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 1, 24, 23, 28, 22, 34,
-      33, 32, 31);
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32,
33, 34,
+        22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
+  }
+
+  /**
+   * Negative epoch time
+   * @throws IOException with error
+   */
+  @Test
+  public void negativeEpochtime() throws IOException {
+    long[] minTimestamps =
+        new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000
};
+    long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
+    long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
+
+    compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
+      new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
+      new long[] { Long.MIN_VALUE, -24 }, false, true);
+  }
+
+  /**
+   * Major compaction
+   * @throws IOException with error
+   */
+  @Test
+  public void majorCompation() throws IOException {
+    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157
};
+    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
+
+    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50,
51, 40,41, 42,
+      33, 30, 31, 2, 1 }, new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156
}, true, true);
+  }
+
+  /**
+   * Major compaction with negative numbers
+   * @throws IOException with error
+   */
+  @Test
+  public void negativeForMajor() throws IOException {
+    long[] minTimestamps =
+        new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
+    long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
+    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
+
+    compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50,
51, 40,
+        41, 42, 33, 30, 31, 2, 1 },
+      new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
+  }
+
+  /**
+   * Major compaction with maximum values
+   * @throws IOException with error
+   */
+  @Test
+  public void maxValuesForMajor() throws IOException {
+    conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, Long.MAX_VALUE / 2);
+    conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 2);
+    store.storeEngine.getCompactionPolicy().setConf(conf);
+    long[] minTimestamps =
+        new long[] { Long.MIN_VALUE, -100 };
+    long[] maxTimestamps = new long[] { -8, Long.MAX_VALUE };
+    long[] sizes = new long[] { 0, 1 };
+
+    compactEquals(Long.MAX_VALUE, sfCreate(minTimestamps, maxTimestamps, sizes),
+      new long[] { 0, 1 },
+      new long[] { Long.MIN_VALUE, -4611686018427387903L, 0, 4611686018427387903L,
+      9223372036854775806L }, true, true);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
index 11a63b4..4448572 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -89,6 +91,10 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy {
     conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
     conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
     store.storeEngine.getCompactionPolicy().setConf(conf);
+    ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
+    EnvironmentEdgeManager.injectEdge(timeMachine);
+    // Has to be  > 0 and < now.
+    timeMachine.setValue(1);
     try {
       // trigger an aged major compaction
       compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);

http://git-wip-us.apache.org/repos/asf/hbase/blob/940bde52/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
index 412e5a7..9a4bb8e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
@@ -42,7 +42,7 @@ public class EverythingPolicy extends RatioBasedCompactionPolicy {
   }
 
   @Override
-  final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile>
candidates,
+  protected final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile>
candidates,
     final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException {
 
     if (candidates.size() < comConf.getMinFilesToCompact()) {


Mime
View raw message