Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3DC5F200CD2 for ; Wed, 12 Jul 2017 17:02:32 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3C4541690C0; Wed, 12 Jul 2017 15:02:32 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6BC51169092 for ; Wed, 12 Jul 2017 17:02:30 +0200 (CEST) Received: (qmail 33774 invoked by uid 500); 12 Jul 2017 15:02:23 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 33660 invoked by uid 99); 12 Jul 2017 15:02:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 12 Jul 2017 15:02:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 627AEF552D; Wed, 12 Jul 2017 15:02:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Wed, 12 Jul 2017 15:02:31 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [10/51] [partial] hbase-site git commit: Published site at 82d554e3783372cc6b05489452c815b57c06f6cd. archived-at: Wed, 12 Jul 2017 15:02:32 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/2777c693/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestStore.MyCompactingMemStore.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestStore.MyCompactingMemStore.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestStore.MyCompactingMemStore.html index 4196a6c..6c65fd1 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestStore.MyCompactingMemStore.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/TestStore.MyCompactingMemStore.html @@ -114,1476 +114,1636 @@ 106import org.mockito.Mockito; 107 108import com.google.common.collect.Lists; -109 -110/** -111 * Test class for the Store -112 */ -113@Category({RegionServerTests.class, MediumTests.class}) -114public class TestStore { -115 private static final Log LOG = LogFactory.getLog(TestStore.class); -116 @Rule public TestName name = new TestName(); -117 -118 HStore store; -119 byte [] table = Bytes.toBytes("table"); -120 byte [] family = Bytes.toBytes("family"); +109import org.apache.hadoop.hbase.filter.Filter; +110import org.apache.hadoop.hbase.filter.FilterBase; +111import static org.junit.Assert.assertEquals; +112import static org.junit.Assert.assertTrue; +113 +114/** +115 * Test class for the Store +116 */ +117@Category({RegionServerTests.class, MediumTests.class}) +118public class TestStore { +119 private static final Log LOG = LogFactory.getLog(TestStore.class); +120 @Rule public TestName name = new TestName(); 121 -122 byte [] row = Bytes.toBytes("row"); -123 byte [] row2 = Bytes.toBytes("row2"); -124 byte [] qf1 = Bytes.toBytes("qf1"); -125 byte [] qf2 = Bytes.toBytes("qf2"); -126 byte [] qf3 = Bytes.toBytes("qf3"); -127 byte [] qf4 = Bytes.toBytes("qf4"); -128 byte [] qf5 = Bytes.toBytes("qf5"); -129 byte [] qf6 = Bytes.toBytes("qf6"); -130 -131 NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR); -132 -133 List<Cell> expected = new ArrayList<>(); -134 List<Cell> result = new ArrayList<>(); -135 -136 long id = System.currentTimeMillis(); -137 Get get = new Get(row); -138 -139 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); -140 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString(); -141 +122 HStore store; +123 byte [] table = Bytes.toBytes("table"); +124 byte [] family = Bytes.toBytes("family"); +125 +126 byte [] row = Bytes.toBytes("row"); +127 byte [] row2 = Bytes.toBytes("row2"); +128 byte [] qf1 = Bytes.toBytes("qf1"); +129 byte [] qf2 = Bytes.toBytes("qf2"); +130 byte [] qf3 = Bytes.toBytes("qf3"); +131 byte [] qf4 = Bytes.toBytes("qf4"); +132 byte [] qf5 = Bytes.toBytes("qf5"); +133 byte [] qf6 = Bytes.toBytes("qf6"); +134 +135 NavigableSet<byte[]> qualifiers = new ConcurrentSkipListSet<>(Bytes.BYTES_COMPARATOR); +136 +137 List<Cell> expected = new ArrayList<>(); +138 List<Cell> result = new ArrayList<>(); +139 +140 long id = System.currentTimeMillis(); +141 Get get = new Get(row); 142 -143 /** -144 * Setup -145 * @throws IOException -146 */ -147 @Before -148 public void setUp() throws IOException { -149 qualifiers.add(qf1); -150 qualifiers.add(qf3); -151 qualifiers.add(qf5); -152 -153 Iterator<byte[]> iter = qualifiers.iterator(); -154 while(iter.hasNext()){ -155 byte [] next = iter.next(); -156 expected.add(new KeyValue(row, family, next, 1, (byte[])null)); -157 get.addColumn(family, next); -158 } -159 } -160 -161 private void init(String methodName) throws IOException { -162 init(methodName, TEST_UTIL.getConfiguration()); +143 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); +144 private final String DIR = TEST_UTIL.getDataTestDir("TestStore").toString(); +145 +146 +147 /** +148 * Setup +149 * @throws IOException +150 */ +151 @Before +152 public void setUp() throws IOException { +153 qualifiers.add(qf1); +154 qualifiers.add(qf3); +155 qualifiers.add(qf5); +156 +157 Iterator<byte[]> iter = qualifiers.iterator(); +158 while(iter.hasNext()){ +159 byte [] next = iter.next(); +160 expected.add(new KeyValue(row, family, next, 1, (byte[])null)); +161 get.addColumn(family, next); +162 } 163 } 164 -165 private Store init(String methodName, Configuration conf) -166 throws IOException { -167 HColumnDescriptor hcd = new HColumnDescriptor(family); -168 // some of the tests write 4 versions and then flush -169 // (with HBASE-4241, lower versions are collected on flush) -170 hcd.setMaxVersions(4); -171 return init(methodName, conf, hcd); -172 } -173 -174 private Store init(String methodName, Configuration conf, -175 HColumnDescriptor hcd) throws IOException { -176 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); -177 return init(methodName, conf, htd, hcd); -178 } -179 -180 private Store init(String methodName, Configuration conf, HTableDescriptor htd, -181 HColumnDescriptor hcd) throws IOException { -182 return init(methodName, conf, htd, hcd, null); -183 } -184 -185 @SuppressWarnings("deprecation") -186 private Store init(String methodName, Configuration conf, HTableDescriptor htd, -187 HColumnDescriptor hcd, MyScannerHook hook) throws IOException { -188 return init(methodName, conf, htd, hcd, hook, false); -189 } -190 @SuppressWarnings("deprecation") -191 private Store init(String methodName, Configuration conf, HTableDescriptor htd, -192 HColumnDescriptor hcd, MyScannerHook hook, boolean switchToPread) throws IOException { -193 //Setting up a Store -194 Path basedir = new Path(DIR+methodName); -195 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); -196 final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName)); -197 -198 FileSystem fs = FileSystem.get(conf); -199 -200 fs.delete(logdir, true); +165 private void init(String methodName) throws IOException { +166 init(methodName, TEST_UTIL.getConfiguration()); +167 } +168 +169 private Store init(String methodName, Configuration conf) +170 throws IOException { +171 HColumnDescriptor hcd = new HColumnDescriptor(family); +172 // some of the tests write 4 versions and then flush +173 // (with HBASE-4241, lower versions are collected on flush) +174 hcd.setMaxVersions(4); +175 return init(methodName, conf, hcd); +176 } +177 +178 private Store init(String methodName, Configuration conf, +179 HColumnDescriptor hcd) throws IOException { +180 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table)); +181 return init(methodName, conf, htd, hcd); +182 } +183 +184 private Store init(String methodName, Configuration conf, HTableDescriptor htd, +185 HColumnDescriptor hcd) throws IOException { +186 return init(methodName, conf, htd, hcd, null); +187 } +188 +189 @SuppressWarnings("deprecation") +190 private Store init(String methodName, Configuration conf, HTableDescriptor htd, +191 HColumnDescriptor hcd, MyStoreHook hook) throws IOException { +192 return init(methodName, conf, htd, hcd, hook, false); +193 } +194 @SuppressWarnings("deprecation") +195 private Store init(String methodName, Configuration conf, HTableDescriptor htd, +196 HColumnDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException { +197 //Setting up a Store +198 Path basedir = new Path(DIR+methodName); +199 Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); +200 final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName)); 201 -202 if (htd.hasFamily(hcd.getName())) { -203 htd.modifyFamily(hcd); -204 } else { -205 htd.addFamily(hcd); -206 } -207 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, -208 MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null); -209 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); -210 final Configuration walConf = new Configuration(conf); -211 FSUtils.setRootDir(walConf, basedir); -212 final WALFactory wals = new WALFactory(walConf, null, methodName); -213 HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(), -214 info.getTable().getNamespace()), fs, conf, info, htd, null); -215 if (hook == null) { -216 store = new HStore(region, hcd, conf); -217 } else { -218 store = new MyStore(region, hcd, conf, hook, switchToPread); -219 } -220 return store; -221 } -222 -223 /** -224 * Test we do not lose data if we fail a flush and then close. -225 * Part of HBase-10466 -226 * @throws Exception -227 */ -228 @Test -229 public void testFlushSizeAccounting() throws Exception { -230 LOG.info("Setting up a faulty file system that cannot write in " + -231 this.name.getMethodName()); -232 final Configuration conf = HBaseConfiguration.create(); -233 // Only retry once. -234 conf.setInt("hbase.hstore.flush.retries.number", 1); -235 User user = User.createUserForTesting(conf, this.name.getMethodName(), -236 new String[]{"foo"}); -237 // Inject our faulty LocalFileSystem -238 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); -239 user.runAs(new PrivilegedExceptionAction<Object>() { -240 @Override -241 public Object run() throws Exception { -242 // Make sure it worked (above is sensitive to caching details in hadoop core) -243 FileSystem fs = FileSystem.get(conf); -244 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); -245 FaultyFileSystem ffs = (FaultyFileSystem)fs; -246 -247 // Initialize region -248 init(name.getMethodName(), conf); -249 -250 MemstoreSize size = store.memstore.getFlushableSize(); -251 Assert.assertEquals(0, size.getDataSize()); -252 LOG.info("Adding some data"); -253 MemstoreSize kvSize = new MemstoreSize(); -254 store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); -255 // add the heap size of active (mutable) segment -256 kvSize.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD); -257 size = store.memstore.getFlushableSize(); -258 Assert.assertEquals(kvSize, size); -259 // Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right. -260 try { -261 LOG.info("Flushing"); -262 flushStore(store, id++); -263 Assert.fail("Didn't bubble up IOE!"); -264 } catch (IOException ioe) { -265 Assert.assertTrue(ioe.getMessage().contains("Fault injected")); -266 } -267 // due to snapshot, change mutable to immutable segment -268 kvSize.incMemstoreSize(0, -269 CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD); -270 size = store.memstore.getFlushableSize(); -271 Assert.assertEquals(kvSize, size); -272 MemstoreSize kvSize2 = new MemstoreSize(); -273 store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2); -274 kvSize2.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD); -275 // Even though we add a new kv, we expect the flushable size to be 'same' since we have -276 // not yet cleared the snapshot -- the above flush failed. -277 Assert.assertEquals(kvSize, size); -278 ffs.fault.set(false); -279 flushStore(store, id++); -280 size = store.memstore.getFlushableSize(); -281 // Size should be the foreground kv size. -282 Assert.assertEquals(kvSize2, size); +202 FileSystem fs = FileSystem.get(conf); +203 +204 fs.delete(logdir, true); +205 +206 if (htd.hasFamily(hcd.getName())) { +207 htd.modifyFamily(hcd); +208 } else { +209 htd.addFamily(hcd); +210 } +211 ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, +212 MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null); +213 HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); +214 final Configuration walConf = new Configuration(conf); +215 FSUtils.setRootDir(walConf, basedir); +216 final WALFactory wals = new WALFactory(walConf, null, methodName); +217 HRegion region = new HRegion(tableDir, wals.getWAL(info.getEncodedNameAsBytes(), +218 info.getTable().getNamespace()), fs, conf, info, htd, null); +219 if (hook == null) { +220 store = new HStore(region, hcd, conf); +221 } else { +222 store = new MyStore(region, hcd, conf, hook, switchToPread); +223 } +224 return store; +225 } +226 +227 /** +228 * Test we do not lose data if we fail a flush and then close. +229 * Part of HBase-10466 +230 * @throws Exception +231 */ +232 @Test +233 public void testFlushSizeAccounting() throws Exception { +234 LOG.info("Setting up a faulty file system that cannot write in " + +235 this.name.getMethodName()); +236 final Configuration conf = HBaseConfiguration.create(); +237 // Only retry once. +238 conf.setInt("hbase.hstore.flush.retries.number", 1); +239 User user = User.createUserForTesting(conf, this.name.getMethodName(), +240 new String[]{"foo"}); +241 // Inject our faulty LocalFileSystem +242 conf.setClass("fs.file.impl", FaultyFileSystem.class, FileSystem.class); +243 user.runAs(new PrivilegedExceptionAction<Object>() { +244 @Override +245 public Object run() throws Exception { +246 // Make sure it worked (above is sensitive to caching details in hadoop core) +247 FileSystem fs = FileSystem.get(conf); +248 Assert.assertEquals(FaultyFileSystem.class, fs.getClass()); +249 FaultyFileSystem ffs = (FaultyFileSystem)fs; +250 +251 // Initialize region +252 init(name.getMethodName(), conf); +253 +254 MemstoreSize size = store.memstore.getFlushableSize(); +255 Assert.assertEquals(0, size.getDataSize()); +256 LOG.info("Adding some data"); +257 MemstoreSize kvSize = new MemstoreSize(); +258 store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); +259 // add the heap size of active (mutable) segment +260 kvSize.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD); +261 size = store.memstore.getFlushableSize(); +262 Assert.assertEquals(kvSize, size); +263 // Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right. +264 try { +265 LOG.info("Flushing"); +266 flushStore(store, id++); +267 Assert.fail("Didn't bubble up IOE!"); +268 } catch (IOException ioe) { +269 Assert.assertTrue(ioe.getMessage().contains("Fault injected")); +270 } +271 // due to snapshot, change mutable to immutable segment +272 kvSize.incMemstoreSize(0, +273 CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD); +274 size = store.memstore.getFlushableSize(); +275 Assert.assertEquals(kvSize, size); +276 MemstoreSize kvSize2 = new MemstoreSize(); +277 store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2); +278 kvSize2.incMemstoreSize(0, MutableSegment.DEEP_OVERHEAD); +279 // Even though we add a new kv, we expect the flushable size to be 'same' since we have +280 // not yet cleared the snapshot -- the above flush failed. +281 Assert.assertEquals(kvSize, size); +282 ffs.fault.set(false); 283 flushStore(store, id++); 284 size = store.memstore.getFlushableSize(); -285 assertEquals(0, size.getDataSize()); -286 assertEquals(MutableSegment.DEEP_OVERHEAD, size.getHeapSize()); -287 return null; -288 } -289 }); -290 } -291 -292 /** -293 * Verify that compression and data block encoding are respected by the -294 * Store.createWriterInTmp() method, used on store flush. -295 */ -296 @Test -297 public void testCreateWriter() throws Exception { -298 Configuration conf = HBaseConfiguration.create(); -299 FileSystem fs = FileSystem.get(conf); -300 -301 HColumnDescriptor hcd = new HColumnDescriptor(family); -302 hcd.setCompressionType(Compression.Algorithm.GZ); -303 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF); -304 init(name.getMethodName(), conf, hcd); -305 -306 // Test createWriterInTmp() -307 StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false); -308 Path path = writer.getPath(); -309 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); -310 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); -311 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3))); -312 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4))); -313 writer.close(); -314 -315 // Verify that compression and encoding settings are respected -316 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf); -317 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm()); -318 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding()); -319 reader.close(); -320 } -321 -322 @Test -323 public void testDeleteExpiredStoreFiles() throws Exception { -324 testDeleteExpiredStoreFiles(0); -325 testDeleteExpiredStoreFiles(1); -326 } -327 -328 /* -329 * @param minVersions the MIN_VERSIONS for the column family -330 */ -331 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception { -332 int storeFileNum = 4; -333 int ttl = 4; -334 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); -335 EnvironmentEdgeManagerTestHelper.injectEdge(edge); -336 -337 Configuration conf = HBaseConfiguration.create(); -338 // Enable the expired store file deletion -339 conf.setBoolean("hbase.store.delete.expired.storefile", true); -340 // Set the compaction threshold higher to avoid normal compactions. -341 conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5); -342 -343 HColumnDescriptor hcd = new HColumnDescriptor(family); -344 hcd.setMinVersions(minVersions); -345 hcd.setTimeToLive(ttl); -346 init(name.getMethodName() + "-" + minVersions, conf, hcd); -347 -348 long storeTtl = this.store.getScanInfo().getTtl(); -349 long sleepTime = storeTtl / storeFileNum; -350 long timeStamp; -351 // There are 4 store files and the max time stamp difference among these -352 // store files will be (this.store.ttl / storeFileNum) -353 for (int i = 1; i <= storeFileNum; i++) { -354 LOG.info("Adding some data for the store file #" + i); -355 timeStamp = EnvironmentEdgeManager.currentTime(); -356 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null); -357 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null); -358 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null); -359 flush(i); -360 edge.incrementTime(sleepTime); -361 } -362 -363 // Verify the total number of store files -364 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size()); -365 -366 // Each call will find one expired store file and delete it before compaction happens. -367 // There will be no compaction due to threshold above. Last file will not be replaced. -368 for (int i = 1; i <= storeFileNum - 1; i++) { -369 // verify the expired store file. -370 assertNull(this.store.requestCompaction()); -371 Collection<StoreFile> sfs = this.store.getStorefiles(); -372 // Ensure i files are gone. -373 if (minVersions == 0) { -374 assertEquals(storeFileNum - i, sfs.size()); -375 // Ensure only non-expired files remain. -376 for (StoreFile sf : sfs) { -377 assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl)); -378 } -379 } else { -380 assertEquals(storeFileNum, sfs.size()); -381 } -382 // Let the next store file expired. -383 edge.incrementTime(sleepTime); -384 } -385 assertNull(this.store.requestCompaction()); -386 -387 Collection<StoreFile> sfs = this.store.getStorefiles(); -388 // Assert the last expired file is not removed. -389 if (minVersions == 0) { -390 assertEquals(1, sfs.size()); -391 } -392 long ts = sfs.iterator().next().getReader().getMaxTimestamp(); -393 assertTrue(ts < (edge.currentTime() - storeTtl)); -394 -395 for (StoreFile sf : sfs) { -396 sf.closeReader(true); -397 } -398 } -399 -400 @Test -401 public void testLowestModificationTime() throws Exception { -402 Configuration conf = HBaseConfiguration.create(); -403 FileSystem fs = FileSystem.get(conf); -404 // Initialize region -405 init(name.getMethodName(), conf); -406 -407 int storeFileNum = 4; -408 for (int i = 1; i <= storeFileNum; i++) { -409 LOG.info("Adding some data for the store file #"+i); -410 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); -411 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); -412 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); -413 flush(i); -414 } -415 // after flush; check the lowest time stamp -416 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); -417 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); -418 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); -419 -420 // after compact; check the lowest time stamp -421 store.compact(store.requestCompaction(), NoLimitThroughputController.INSTANCE, null); -422 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); -423 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); -424 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); -425 } -426 -427 private static long getLowestTimeStampFromFS(FileSystem fs, -428 final Collection<StoreFile> candidates) throws IOException { -429 long minTs = Long.MAX_VALUE; -430 if (candidates.isEmpty()) { -431 return minTs; -432 } -433 Path[] p = new Path[candidates.size()]; -434 int i = 0; -435 for (StoreFile sf : candidates) { -436 p[i] = sf.getPath(); -437 ++i; -438 } -439 -440 FileStatus[] stats = fs.listStatus(p); -441 if (stats == null || stats.length == 0) { -442 return minTs; -443 } -444 for (FileStatus s : stats) { -445 minTs = Math.min(minTs, s.getModificationTime()); -446 } -447 return minTs; -448 } -449 -450 ////////////////////////////////////////////////////////////////////////////// -451 // Get tests -452 ////////////////////////////////////////////////////////////////////////////// +285 // Size should be the foreground kv size. +286 Assert.assertEquals(kvSize2, size); +287 flushStore(store, id++); +288 size = store.memstore.getFlushableSize(); +289 assertEquals(0, size.getDataSize()); +290 assertEquals(MutableSegment.DEEP_OVERHEAD, size.getHeapSize()); +291 return null; +292 } +293 }); +294 } +295 +296 /** +297 * Verify that compression and data block encoding are respected by the +298 * Store.createWriterInTmp() method, used on store flush. +299 */ +300 @Test +301 public void testCreateWriter() throws Exception { +302 Configuration conf = HBaseConfiguration.create(); +303 FileSystem fs = FileSystem.get(conf); +304 +305 HColumnDescriptor hcd = new HColumnDescriptor(family); +306 hcd.setCompressionType(Compression.Algorithm.GZ); +307 hcd.setDataBlockEncoding(DataBlockEncoding.DIFF); +308 init(name.getMethodName(), conf, hcd); +309 +310 // Test createWriterInTmp() +311 StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false); +312 Path path = writer.getPath(); +313 writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); +314 writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); +315 writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3))); +316 writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4))); +317 writer.close(); +318 +319 // Verify that compression and encoding settings are respected +320 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf); +321 Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm()); +322 Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding()); +323 reader.close(); +324 } +325 +326 @Test +327 public void testDeleteExpiredStoreFiles() throws Exception { +328 testDeleteExpiredStoreFiles(0); +329 testDeleteExpiredStoreFiles(1); +330 } +331 +332 /* +333 * @param minVersions the MIN_VERSIONS for the column family +334 */ +335 public void testDeleteExpiredStoreFiles(int minVersions) throws Exception { +336 int storeFileNum = 4; +337 int ttl = 4; +338 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(); +339 EnvironmentEdgeManagerTestHelper.injectEdge(edge); +340 +341 Configuration conf = HBaseConfiguration.create(); +342 // Enable the expired store file deletion +343 conf.setBoolean("hbase.store.delete.expired.storefile", true); +344 // Set the compaction threshold higher to avoid normal compactions. +345 conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 5); +346 +347 HColumnDescriptor hcd = new HColumnDescriptor(family); +348 hcd.setMinVersions(minVersions); +349 hcd.setTimeToLive(ttl); +350 init(name.getMethodName() + "-" + minVersions, conf, hcd); +351 +352 long storeTtl = this.store.getScanInfo().getTtl(); +353 long sleepTime = storeTtl / storeFileNum; +354 long timeStamp; +355 // There are 4 store files and the max time stamp difference among these +356 // store files will be (this.store.ttl / storeFileNum) +357 for (int i = 1; i <= storeFileNum; i++) { +358 LOG.info("Adding some data for the store file #" + i); +359 timeStamp = EnvironmentEdgeManager.currentTime(); +360 this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null); +361 this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null); +362 this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null); +363 flush(i); +364 edge.incrementTime(sleepTime); +365 } +366 +367 // Verify the total number of store files +368 Assert.assertEquals(storeFileNum, this.store.getStorefiles().size()); +369 +370 // Each call will find one expired store file and delete it before compaction happens. +371 // There will be no compaction due to threshold above. Last file will not be replaced. +372 for (int i = 1; i <= storeFileNum - 1; i++) { +373 // verify the expired store file. +374 assertNull(this.store.requestCompaction()); +375 Collection<StoreFile> sfs = this.store.getStorefiles(); +376 // Ensure i files are gone. +377 if (minVersions == 0) { +378 assertEquals(storeFileNum - i, sfs.size()); +379 // Ensure only non-expired files remain. +380 for (StoreFile sf : sfs) { +381 assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl)); +382 } +383 } else { +384 assertEquals(storeFileNum, sfs.size()); +385 } +386 // Let the next store file expired. +387 edge.incrementTime(sleepTime); +388 } +389 assertNull(this.store.requestCompaction()); +390 +391 Collection<StoreFile> sfs = this.store.getStorefiles(); +392 // Assert the last expired file is not removed. +393 if (minVersions == 0) { +394 assertEquals(1, sfs.size()); +395 } +396 long ts = sfs.iterator().next().getReader().getMaxTimestamp(); +397 assertTrue(ts < (edge.currentTime() - storeTtl)); +398 +399 for (StoreFile sf : sfs) { +400 sf.closeReader(true); +401 } +402 } +403 +404 @Test +405 public void testLowestModificationTime() throws Exception { +406 Configuration conf = HBaseConfiguration.create(); +407 FileSystem fs = FileSystem.get(conf); +408 // Initialize region +409 init(name.getMethodName(), conf); +410 +411 int storeFileNum = 4; +412 for (int i = 1; i <= storeFileNum; i++) { +413 LOG.info("Adding some data for the store file #"+i); +414 this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); +415 this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); +416 this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); +417 flush(i); +418 } +419 // after flush; check the lowest time stamp +420 long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); +421 long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); +422 Assert.assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); +423 +424 // after compact; check the lowest time stamp +425 store.compact(store.requestCompaction(), NoLimitThroughputController.INSTANCE, null); +426 lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); +427 lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); +428 Assert.assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); +429 } +430 +431 private static long getLowestTimeStampFromFS(FileSystem fs, +432 final Collection<StoreFile> candidates) throws IOException { +433 long minTs = Long.MAX_VALUE; +434 if (candidates.isEmpty()) { +435 return minTs; +436 } +437 Path[] p = new Path[candidates.size()]; +438 int i = 0; +439 for (StoreFile sf : candidates) { +440 p[i] = sf.getPath(); +441 ++i; +442 } +443 +444 FileStatus[] stats = fs.listStatus(p); +445 if (stats == null || stats.length == 0) { +446 return minTs; +447 } +448 for (FileStatus s : stats) { +449 minTs = Math.min(minTs, s.getModificationTime()); +450 } +451 return minTs; +452 } 453 -454 private static final int BLOCKSIZE_SMALL = 8192; -455 /** -456 * Test for hbase-1686. -457 * @throws IOException -458 */ -459 @Test -460 public void testEmptyStoreFile() throws IOException { -461 init(this.name.getMethodName()); -462 // Write a store file. -463 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); -464 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); -465 flush(1); -466 // Now put in place an empty store file. Its a little tricky. Have to -467 // do manually with hacked in sequence id. -468 StoreFile f = this.store.getStorefiles().iterator().next(); -469 Path storedir = f.getPath().getParent(); -470 long seqid = f.getMaxSequenceId(); -471 Configuration c = HBaseConfiguration.create(); -472 FileSystem fs = FileSystem.get(c); -473 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build(); -474 StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), -475 fs) -476 .withOutputDir(storedir) -477 .withFileContext(meta) -478 .build(); -479 w.appendMetadata(seqid + 1, false); -480 w.close(); -481 this.store.close(); -482 // Reopen it... should pick up two files -483 this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c); -484 Assert.assertEquals(2, this.store.getStorefilesCount()); -485 -486 result = HBaseTestingUtility.getFromStoreFile(store, -487 get.getRow(), -488 qualifiers); -489 Assert.assertEquals(1, result.size()); -490 } -491 -492 /** -493 * Getting data from memstore only -494 * @throws IOException -495 */ -496 @Test -497 public void testGet_FromMemStoreOnly() throws IOException { -498 init(this.name.getMethodName()); -499 -500 //Put data in memstore -501 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); -502 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); -503 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); -504 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); -505 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); -506 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); -507 -508 //Get -509 result = HBaseTestingUtility.getFromStoreFile(store, -510 get.getRow(), qualifiers); +454 ////////////////////////////////////////////////////////////////////////////// +455 // Get tests +456 ////////////////////////////////////////////////////////////////////////////// +457 +458 private static final int BLOCKSIZE_SMALL = 8192; +459 /** +460 * Test for hbase-1686. +461 * @throws IOException +462 */ +463 @Test +464 public void testEmptyStoreFile() throws IOException { +465 init(this.name.getMethodName()); +466 // Write a store file. +467 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); +468 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); +469 flush(1); +470 // Now put in place an empty store file. Its a little tricky. Have to +471 // do manually with hacked in sequence id. +472 StoreFile f = this.store.getStorefiles().iterator().next(); +473 Path storedir = f.getPath().getParent(); +474 long seqid = f.getMaxSequenceId(); +475 Configuration c = HBaseConfiguration.create(); +476 FileSystem fs = FileSystem.get(c); +477 HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build(); +478 StoreFileWriter w = new StoreFileWriter.Builder(c, new CacheConfig(c), +479 fs) +480 .withOutputDir(storedir) +481 .withFileContext(meta) +482 .build(); +483 w.appendMetadata(seqid + 1, false); +484 w.close(); +485 this.store.close(); +486 // Reopen it... should pick up two files +487 this.store = new HStore(this.store.getHRegion(), this.store.getColumnFamilyDescriptor(), c); +488 Assert.assertEquals(2, this.store.getStorefilesCount()); +489 +490 result = HBaseTestingUtility.getFromStoreFile(store, +491 get.getRow(), +492 qualifiers); +493 Assert.assertEquals(1, result.size()); +494 } +495 +496 /** +497 * Getting data from memstore only +498 * @throws IOException +499 */ +500 @Test +501 public void testGet_FromMemStoreOnly() throws IOException { +502 init(this.name.getMethodName()); +503 +504 //Put data in memstore +505 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); +506 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); +507 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); +508 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); +509 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); +510 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); 511 -512 //Compare -513 assertCheck(); -514 } +512 //Get +513 result = HBaseTestingUtility.getFromStoreFile(store, +514 get.getRow(), qualifiers); 515 -516 /** -517 * Getting data from files only -518 * @throws IOException -519 */ -520 @Test -521 public void testGet_FromFilesOnly() throws IOException { -522 init(this.name.getMethodName()); -523 -524 //Put data in memstore -525 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); -526 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); -527 //flush -528 flush(1); -529 -530 //Add more data -531 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); -532 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); -533 //flush -534 flush(2); -535 -536 //Add more data -537 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); -538 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); -539 //flush -540 flush(3); -541 -542 //Get -543 result = HBaseTestingUtility.getFromStoreFile(store, -544 get.getRow(), -545 qualifiers); -546 //this.store.get(get, qualifiers, result); -547 -548 //Need to sort the result since multiple files -549 Collections.sort(result, CellComparator.COMPARATOR); -550 -551 //Compare -552 assertCheck(); -553 } +516 //Compare +517 assertCheck(); +518 } +519 +520 /** +521 * Getting data from files only +522 * @throws IOException +523 */ +524 @Test +525 public void testGet_FromFilesOnly() throws IOException { +526 init(this.name.getMethodName()); +527 +528 //Put data in memstore +529 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); +530 this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); +531 //flush +532 flush(1); +533 +534 //Add more data +535 this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); +536 this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); +537 //flush +538 flush(2); +539 +540 //Add more data +541 this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); +542 this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); +543 //flush +544 flush(3); +545 +546 //Get +547 result = HBaseTestingUtility.getFromStoreFile(store, +548 get.getRow(), +549 qualifiers); +550 //this.store.get(get, qualifiers, result); +551 +552 //Need to sort the result since multiple files +553 Collections.sort(result, CellComparator.COMPARATOR); 554 -555 /** -556 * Getting data from memstore and files -557 * @throws IOException -558 */ -559 @Test -560 public void testGet_FromMemStoreAndFiles() throws IOException { -561 init(this.name.getMethodName()); -562 -563 //Put data in memstore -564 this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null);