hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1489990 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/client/
Date Wed, 05 Jun 2013 18:18:20 GMT
Author: liyin
Date: Wed Jun  5 18:18:20 2013
New Revision: 1489990

URL: http://svn.apache.org/r1489990
Log:
[HBASE-8185] Adding functionality to create ClientLocalScanner without creating hardlinks.

Author: manukranthk

Summary: This feature targets the scenarios where the user will be responsible for creating
a full table snapshot and reads from the snapshot. This helps in scenarios where we need to
scan from a full snapshot to keep the view of the data consistent over scanner(mapper) failures
on the scan job.

Test Plan: Unit tests.

Reviewers: aaiyer, liyintang, rshroff

Reviewed By: liyintang

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D830097

Task ID: 2430669

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
Wed Jun  5 18:18:20 2013
@@ -72,6 +72,7 @@ public class ClientLocalScanner extends 
    */
   public static ThreadPoolExecutor scanPrefetchThreadPool;
   private static int numHandlers = 20;
+  private final boolean areHardlinksCreated;
   // Initializing the numHandlers statically since the thread pool can be
   // reused across different scan operations on the same client.
   static {
@@ -80,11 +81,13 @@ public class ClientLocalScanner extends 
             new DaemonThreadFactory("scan-prefetch-"));
   }
 
-  protected ClientLocalScanner(final Scan scan, HTable htable) {
+  protected ClientLocalScanner(final Scan scan, final HTable htable,
+      final boolean areHardlinksCreated) {
     super(scan, htable);
     // The seek + read functionality will be used in this case since
     // scanning large files is faster using seek + read.
     Store.isPread = false;
+    this.areHardlinksCreated = areHardlinksCreated;
   }
 
   /**
@@ -107,7 +110,7 @@ public class ClientLocalScanner extends 
     families = filterFamilies(families, scan);
 
     HRegionUtilities.parallelStoreOpener(info, conf, families, tableDir,
-        fs, this.stores);
+        fs, this.stores, this.areHardlinksCreated);
   }
 
   private Collection<HColumnDescriptor> filterFamilies(

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Wed Jun
 5 18:18:20 2013
@@ -590,7 +590,24 @@ public class HTable implements HTableInt
   }
 
   public ResultScanner getLocalScanner(final Scan scan) throws IOException {
-    ClientLocalScanner s = new ClientLocalScanner(scan, this);
+    return getLocalScanner(scan, true);
+  }
+
+  /**
+   * Create a ClientLocalScanner to scan the HTable using the scan provided.
+   *
+   * @param scan The scan object that determines the way scanning is done.
+   * @param createNewHardlinks If true, tells to create a snapshot of the store
+   * by creating new hard links.
+   * Otherwise, we assume that the table hierarchy under the root directory
+   * is not going to change and hence we read directly from it; use with caution
+   * @return
+   * @throws IOException
+   */
+  public ResultScanner getLocalScanner(final Scan scan,
+      boolean createNewHardlinks) throws IOException {
+    ClientLocalScanner s =
+        new ClientLocalScanner(scan, this, createNewHardlinks);
     s.initialize();
     return s;
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
Wed Jun  5 18:18:20 2013
@@ -75,9 +75,9 @@ public class HRegionUtilities {
   // TODO(manukranthk) : Refactor HRegion code to use this common code path.
   public static void parallelStoreOpener(final HRegionInfo info,
       final Configuration conf, Collection<HColumnDescriptor> families,
-      final Path tableDir, final FileSystem fs, Map<byte[], Store> stores)
-      throws IOException{
- // initialize the thread pool for opening stores in parallel.
+      final Path tableDir, final FileSystem fs, Map<byte[], Store> stores,
+      final boolean createNewHardlinks) throws IOException{
+    // initialize the thread pool for opening stores in parallel.
     ThreadPoolExecutor storeOpenerThreadPool =
       StoreThreadUtils.getStoreOpenAndCloseThreadPool("StoreOpenerThread-"
     + info.getRegionNameAsString(), info, conf);
@@ -88,7 +88,8 @@ public class HRegionUtilities {
     for (final HColumnDescriptor family : families) {
       completionService.submit(new Callable<Store>() {
         public Store call() throws IOException {
-          return new ReadOnlyStore(tableDir, info, family, fs, conf);
+          return new ReadOnlyStore(tableDir, info, family,
+              fs, conf, createNewHardlinks);
         }
       });
     }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
Wed Jun  5 18:18:20 2013
@@ -55,10 +55,14 @@ public class ReadOnlyStore extends Store
    * @throws IOException
    */
   public ReadOnlyStore(Path basedir, HRegionInfo regionInfo, HColumnDescriptor family,
-      FileSystem fs, Configuration confParam)
+      FileSystem fs, Configuration confParam, final boolean createNewHardlinks)
   throws IOException {
     super(basedir, family, fs, confParam, null, regionInfo);
-    this.storefiles = sortAndClone(loadStoreFilesWithHardLinks());
+    if (createNewHardlinks) {
+      this.storefiles = sortAndClone(loadStoreFilesWithHardLinks());
+    } else {
+      this.storefiles = sortAndClone(loadStoreFiles(this.fs.listStatus(this.homedir)));
+    }
   }
 
   private List<StoreFile> loadStoreFilesWithHardLinks() throws IOException{

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
Wed Jun  5 18:18:20 2013
@@ -36,16 +36,13 @@ import org.apache.hadoop.hbase.HBaseTest
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.InjectionEvent;
 import org.apache.hadoop.hbase.util.InjectionHandler;
 import org.junit.AfterClass;
@@ -58,8 +55,6 @@ public class TestClientLocalScanner {
       new HBaseTestingUtility();
   private static byte [] FAMILY = Bytes.toBytes("testFamily");
   private static byte [] FAMILY2 = Bytes.toBytes("testFamily2");
-  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
-  private static byte [] VALUE = Bytes.toBytes("testValue");
   private static int SLAVES = 3;
 
   /**
@@ -79,29 +74,6 @@ public class TestClientLocalScanner {
   }
 
   @Test
-  public void testInconsistentRegionDirectories() throws IOException {
-    byte [] tableName = Bytes.toBytes("testInconsistentRegionDirectories");
-    String rootDir = TEST_UTIL.getConfiguration().get("hbase.rootdir");
-    String tmpPath = "/tmp/testInconsistentRegionDirectories/";
-    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
-    Path p = new Path(tmpPath);
-    fs.mkdirs(p);
-    assertTrue(fs.listStatus(p).length == 0);
-    TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath);
-    HTable t = TEST_UTIL.createTable(tableName, FAMILY);
-    TEST_UTIL.loadTable(t, FAMILY);
-    try {
-      t.getLocalScanner(new Scan());
-    } catch (IOException e) {
-      assertTrue(fs.listStatus(p).length == 0);
-      return;
-    } finally {
-      TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir);
-    }
-    assertTrue(false);
-  }
-
-  @Test
   public void testCompareLocalScanToRemoteScan() throws IOException {
     byte [] name = Bytes.toBytes("testCompareLocalScanToRemoteScan");
     HTable t = TEST_UTIL.createTable(name, new byte[][] {FAMILY, FAMILY2});
@@ -114,11 +86,7 @@ public class TestClientLocalScanner {
     TEST_UTIL.flush(name);
     assertRowCount(t, rowCount);
 
-    Scan scan = new Scan();
-    scan.addFamily(FAMILY);
-    scan.setServerPrefetching(true);
-    scan.setCaching(100);
-    scan.setBatch(100);
+    Scan scan = getScan(100, 100, true, FAMILY);
     assertTrue(compareScanners(tmpTable.getScanner(scan),
         t.getLocalScanner(scan)));
     FileSystem fs = FileSystem.get(t.getConfiguration());
@@ -256,6 +224,69 @@ public class TestClientLocalScanner {
     assertEquals(rowCount - endKeyCount, countGreater);
   }
 
+  @Test
+  public void testInconsistentRegionDirectories() throws IOException {
+    byte [] tableName = Bytes.toBytes("testInconsistentRegionDirectories");
+    String rootDir = TEST_UTIL.getConfiguration().get("hbase.rootdir");
+    String tmpPath = "/tmp/testInconsistentRegionDirectories/";
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    Path p = new Path(tmpPath);
+    fs.mkdirs(p);
+    assertTrue(fs.listStatus(p).length == 0);
+    TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath);
+    HTable t = TEST_UTIL.createTable(tableName, FAMILY);
+    TEST_UTIL.loadTable(t, FAMILY);
+    try {
+      t.getLocalScanner(new Scan());
+    } catch (IOException e) {
+      assertTrue(fs.listStatus(p).length == 0);
+      return;
+    } finally {
+      TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir);
+    }
+    assertTrue(false);
+  }
+
+  @Test
+  public void testLocalScannerWithoutHardlinks() throws IOException {
+    byte [] tableName = Bytes.toBytes("testLocalScannerWithoutHardlinks");
+    HTable t = TEST_UTIL.createTable(tableName, FAMILY);
+    HTable tmpTable = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    int rowCount = TEST_UTIL.loadTable(t, FAMILY);
+    t.flushCommits();
+    TEST_UTIL.flush(tableName);
+    assertRowCount(t, rowCount);
+
+    Scan scan = getScan(100, 100, true, FAMILY);
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    String hardLinkFolder = TEST_UTIL.getConfiguration().get(
+        HConstants.READ_ONLY_HARDLINKS_FOLDER,
+        HConstants.READ_ONLY_HARDLINKS_FOLDER_DEFAULT);
+    Path folder = new Path(hardLinkFolder);
+
+    ResultScanner scanner = t.getLocalScanner(scan);
+    assertTrue(fs.exists(folder));
+    assertTrue(fs.listStatus(folder).length > 0);
+    fs.delete(folder, true);
+
+    scanner = t.getLocalScanner(scan, false);
+    assertTrue(compareScanners(tmpTable.getScanner(scan),
+        scanner));
+    assertTrue(!fs.exists(folder));
+    scanner.close();
+    scanner.close();
+  }
+
+  public Scan getScan(int caching, int batching,
+      boolean serverPrefetching, byte[] family) {
+    Scan scan = new Scan();
+    scan.addFamily(family);
+    scan.setServerPrefetching(serverPrefetching);
+    scan.setCaching(caching);
+    scan.setBatch(batching);
+    return scan;
+  }
+
   /**
    * Wait on table split.  May return because we waited long enough on the split
    * and it didn't happen.  Caller should check.



Mime
View raw message