Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 47080200D6C for ; Wed, 20 Dec 2017 07:11:10 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 45286160C2A; Wed, 20 Dec 2017 06:11:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8AC71160C1B for ; Wed, 20 Dec 2017 07:11:09 +0100 (CET) Received: (qmail 23629 invoked by uid 500); 20 Dec 2017 06:11:08 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 23620 invoked by uid 99); 20 Dec 2017 06:11:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Dec 2017 06:11:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 11A63DFFD9; Wed, 20 Dec 2017 06:11:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ramkrishna@apache.org To: commits@hbase.apache.org Message-Id: <0d2a50aba082463c986edc44c1f15135@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-19468 FNFE during scans and flushes (Ram) Date: Wed, 20 Dec 2017 06:11:08 +0000 (UTC) archived-at: Wed, 20 Dec 2017 06:11:10 -0000 Repository: hbase Updated Branches: refs/heads/branch-1 d53e960c5 -> d28732fd9 HBASE-19468 FNFE during scans and flushes (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d28732fd Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d28732fd Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d28732fd Branch: refs/heads/branch-1 Commit: d28732fd98a7f7f932970e0183ceeb4c152ea01c Parents: d53e960 Author: ramkrish86 Authored: Wed Dec 20 17:06:27 2017 +0530 Committer: ramkrish86 Committed: Wed Dec 20 17:06:27 2017 +0530 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/StoreScanner.java | 29 +++++++++--- .../TestCompactedHFilesDischarger.java | 46 +++++++++++++++++++- 2 files changed, 68 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d28732fd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index fbdf7b2..632f146 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.NavigableSet; import java.util.concurrent.CountDownLatch; @@ -132,8 +133,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner private boolean scanUsePread = false; // Indicates whether there was flush during the course of the scan private volatile boolean flushed = false; + // generally we get one file from a flush - private final List flushedStoreFiles = new ArrayList(1); + private final List flushedstoreFileScanners = + new ArrayList(1); // generally we get one memstroe scanner from a flush private final List memStoreScannersAfterFlush = new ArrayList<>(1); // The current list of scanners @@ -463,6 +466,10 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner this.closing = true; clearAndClose(scannersForDelayedClose); clearAndClose(memStoreScannersAfterFlush); + // clear them at any case. In case scanner.next() was never called + // and there were some lease expiry we need to close all the scanners + // on the flushed files which are open + clearAndClose(flushedstoreFileScanners); // Under test, we dont have a this.store if (this.store != null) this.store.deleteChangedReaderObserver(this); @@ -833,7 +840,17 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner flushLock.lock(); try { flushed = true; - flushedStoreFiles.addAll(sfs); + final boolean isCompaction = false; + boolean usePread = get || scanUsePread; + // SEE HBASE-19468 where the flushed files are getting compacted even before a scanner + // calls next(). So its better we create scanners here rather than next() call. Ensure + // these scanners are properly closed() whether or not the scan is completed successfully + // Eagerly creating scanners so that we have the ref counting ticking on the newly created + // store files. In case of stream scanners this eager creation does not induce performance + // penalty because in scans (that uses stream scanners) the next() call is bound to happen. + List scanners = store.getScanners(sfs, cacheBlocks, get, usePread, + isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, false); + flushedstoreFileScanners.addAll(scanners); if (!CollectionUtils.isEmpty(memStoreScanners)) { clearAndClose(memStoreScannersAfterFlush); memStoreScannersAfterFlush.addAll(memStoreScanners); @@ -901,13 +918,13 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner List scanners = null; flushLock.lock(); try { - List allScanners = new ArrayList<>(flushedStoreFiles.size() + memStoreScannersAfterFlush.size()); - allScanners.addAll(store.getScanners(flushedStoreFiles, cacheBlocks, get, usePread, - isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, false)); + List allScanners = + new ArrayList<>(flushedstoreFileScanners.size() + memStoreScannersAfterFlush.size()); + allScanners.addAll(flushedstoreFileScanners); allScanners.addAll(memStoreScannersAfterFlush); scanners = selectScannersFrom(allScanners); // Clear the current set of flushed store files so that they don't get added again - flushedStoreFiles.clear(); + flushedstoreFileScanners.clear(); memStoreScannersAfterFlush.clear(); } finally { flushLock.unlock(); http://git-wip-us.apache.org/repos/asf/hbase/blob/d28732fd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java index f3d6575..2770db6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactedHFilesDischarger.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; @@ -336,6 +337,49 @@ public class TestCompactedHFilesDischarger { assertTrue(compactedfiles.size() == 0); } + @Test + public void testStoreFileMissing() throws Exception { + // Write 3 records and create 3 store files. + write("row1"); + region.flush(true); + write("row2"); + region.flush(true); + write("row3"); + region.flush(true); + + Scan scan = new Scan(); + scan.setCaching(1); + RegionScanner scanner = region.getScanner(scan); + List res = new ArrayList(); + // Read first item + scanner.next(res); + assertEquals("row1", Bytes.toString(CellUtil.cloneRow(res.get(0)))); + res.clear(); + // Create a new file in between scan nexts + write("row4"); + region.flush(true); + + // Compact the table + region.compact(true); + + // Create the cleaner object + CompactedHFilesDischarger cleaner = + new CompactedHFilesDischarger(1000, (Stoppable) null, rss, false); + cleaner.chore(); + // This issues scan next + scanner.next(res); + assertEquals("row2", Bytes.toString(CellUtil.cloneRow(res.get(0)))); + + scanner.close(); + } + + private void write(String row1) throws IOException { + byte[] row = Bytes.toBytes(row1); + Put put = new Put(row); + put.addColumn(fam, qual1, row); + region.put(put); + } + protected void countDown() { // count down 3 times latch.countDown(); @@ -369,7 +413,7 @@ public class TestCompactedHFilesDischarger { try { initiateScan(region); } catch (IOException e) { - // do nothing + e.printStackTrace(); } }