gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alfonsonishik...@apache.org
Subject svn commit: r1556044 - in /gora/trunk/gora-hbase/src: main/java/org/apache/gora/hbase/store/HBaseStore.java test/conf/gora.properties test/java/org/apache/gora/hbase/store/TestHBaseStore.java
Date Mon, 06 Jan 2014 21:53:13 GMT
Author: alfonsonishikawa
Date: Mon Jan  6 21:53:13 2014
New Revision: 1556044

URL: http://svn.apache.org/r1556044
Log:
GORA-117 gora hbase does not have a mechanism to set the caching on a scanner, which makes
for poor performance on map/reduce jobs

Added:
    gora/trunk/gora-hbase/src/test/conf/gora.properties
Modified:
    gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
    gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java

Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1556044&r1=1556043&r2=1556044&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Mon Jan
 6 21:53:13 2014
@@ -54,6 +54,7 @@ import org.apache.gora.persistency.impl.
 import org.apache.gora.query.PartitionQuery;
 import org.apache.gora.query.Query;
 import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.store.DataStoreFactory;
 import org.apache.gora.store.impl.DataStoreBase;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
@@ -88,6 +89,9 @@ implements Configurable {
   private static final String DEPRECATED_MAPPING_FILE = "hbase-mapping.xml";
   public static final String DEFAULT_MAPPING_FILE = "gora-hbase-mapping.xml";
 
+  private static final String SCANNER_CACHING_PROPERTIES_KEY = "scanner.caching" ;
+  private static final int SCANNER_CACHING_PROPERTIES_DEFAULT = 0 ;
+  
   private volatile HBaseAdmin admin;
 
   private volatile HBaseTableConnection table;
@@ -96,6 +100,8 @@ implements Configurable {
 
   private volatile HBaseMapping mapping;
 
+  private int scannerCaching = SCANNER_CACHING_PROPERTIES_DEFAULT ;
+  
   public HBaseStore()  {
   }
 
@@ -127,6 +133,17 @@ implements Configurable {
       throw new RuntimeException(e);
     }
     
+    // Set scanner caching option
+    try {
+      this.setScannerCaching(
+          Integer.valueOf(DataStoreFactory.findProperty(this.properties, this,
+              SCANNER_CACHING_PROPERTIES_KEY,
+              String.valueOf(SCANNER_CACHING_PROPERTIES_DEFAULT)))) ;
+    }catch(Exception e){
+      LOG.error("Can not load " + SCANNER_CACHING_PROPERTIES_KEY + " from gora.properties.
Setting to default value: " + SCANNER_CACHING_PROPERTIES_DEFAULT, e) ;
+      this.setScannerCaching(SCANNER_CACHING_PROPERTIES_DEFAULT) ; // Default value if something
is wrong
+    }
+    
     if(autoCreateSchema) {
       createSchema();
     }
@@ -465,6 +482,9 @@ implements Configurable {
 
   public ResultScanner createScanner(Query<K, T> query) throws IOException {
     final Scan scan = new Scan();
+    
+    scan.setCaching(this.getScannerCaching()) ; 
+    
     if (query.getStartKey() != null) {
       scan.setStartRow(toBytes(query.getStartKey()));
     }
@@ -729,4 +749,29 @@ implements Configurable {
     this.conf = conf;
   }
 
+  /**
+   * Gets the Scanner Caching optimization value
+   * @return The value used internally in {@link Scan#setCaching(int)}
+   */
+  public int getScannerCaching() {
+    return this.scannerCaching ;
+  }
+  
+  /**
+   * Sets the value for Scanner Caching optimization
+   * 
+   * @see Scan#setCaching(int)
+   * 
+   * @param numRows the number of rows for caching >= 0
+   * @return &lt;&lt;Fluent interface&gt;&gt;
+   */
+  public HBaseStore<K, T> setScannerCaching(int numRows) {
+    if (numRows < 0) {
+      LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: " + numRows +
".") ;
+      return this ;
+    }
+    this.scannerCaching = numRows ;
+    return this ;
+  }
+  
 }

Added: gora/trunk/gora-hbase/src/test/conf/gora.properties
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/conf/gora.properties?rev=1556044&view=auto
==============================================================================
--- gora/trunk/gora-hbase/src/test/conf/gora.properties (added)
+++ gora/trunk/gora-hbase/src/test/conf/gora.properties Mon Jan  6 21:53:13 2014
@@ -0,0 +1,6 @@
+
+#Datastore is configured programatically in the tests
+#gora.datastore.default=
+
+#Caching option for HBaseStore
+gora.hbasestore.scanner.caching=1000
\ No newline at end of file

Modified: gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java?rev=1556044&r1=1556043&r2=1556044&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java (original)
+++ gora/trunk/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java Mon
Jan  6 21:53:13 2014
@@ -224,6 +224,11 @@ public class TestHBaseStore extends Data
     table.close();
   }
 
+  @Test
+  public void assertScannerCachingValue() {
+    assertEquals(1000, ((HBaseStore<String,WebPage>)this.webPageStore).getScannerCaching())
;
+    assertEquals(1000, ((HBaseStore<String,Employee>)this.employeeStore).getScannerCaching())
;
+  }
 
   @Ignore("We need to skip this test since gora considers endRow inclusive, while its exclusive
for HBase.")
   @Override



Mime
View raw message