gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lewi...@apache.org
Subject [4/5] gora git commit: Updates to GORA-443 issue
Date Fri, 03 Feb 2017 20:25:30 GMT
Updates to GORA-443 issue


Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/f5503ef3
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/f5503ef3
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/f5503ef3

Branch: refs/heads/master
Commit: f5503ef369f2193f6395e72cc1875a94c04b33a3
Parents: 163d91b
Author: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Authored: Wed Sep 14 18:28:50 2016 -0700
Committer: Lewis John McGibbney <lewis.mcgibbney@gmail.com>
Committed: Wed Sep 14 18:28:50 2016 -0700

----------------------------------------------------------------------
 README.md                                       |  7 +++
 gora-couchdb/pom.xml                            |  8 ---
 .../org/apache/gora/hbase/store/HBaseStore.java | 34 +++++++++----
 .../gora/hbase/store/HBaseTableConnection.java  | 52 ++++++--------------
 .../apache/gora/hbase/store/TestHBaseStore.java |  2 +-
 pom.xml                                         |  4 +-
 6 files changed, 49 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index e9021b3..4171456 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,13 @@
 #Apache Gora Project
 
 <img src="http://gora.apache.org/resources/img/powered-by-gora.png" align="right" width="300"
/>
+
+[![license](https://img.shields.io/github/license/apache/gora.svg?maxAge=2592000?style=plastic)](http://www.apache.org/licenses/LICENSE-2.0)
+[![Jenkins](https://img.shields.io/jenkins/s/https/builds.apache.org/gora-trunk.svg?maxAge=2592000?style=plastic)](https://builds.apache.org/job/gora-trunk/)
+[![Jenkins tests](https://img.shields.io/jenkins/t/https/builds.apache.org/gora-trunk.svg?maxAge=2592000?style=plastic)](https://builds.apache.org/job/gora-trunk)
+[![Maven Central](https://img.shields.io/maven-central/v/org.apache.gora/gora.svg?maxAge=2592000?style=plastic)](http://search.maven.org/#search|ga|1|g%3A%22org.apache.gora%22)
+[![SonarQube Tech Debt](https://img.shields.io/sonar/http/analysis.apache.org/org.apache.gora:gora/tech_debt.svg?maxAge=2592000?style=plastic)](https://analysis.apache.org/dashboard/index/76356)
+[![Twitter URL](https://img.shields.io/twitter/url/http/apachegora.svg?style=social&maxAge=2592000?style=plastic)](https://twitter.com/apachegora)
  
 The Apache Gora open source framework provides an in-memory data model 
 and persistence for big data. Gora supports persisting to column stores, 

http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-couchdb/pom.xml
----------------------------------------------------------------------
diff --git a/gora-couchdb/pom.xml b/gora-couchdb/pom.xml
index ab050cc..6f991e0 100644
--- a/gora-couchdb/pom.xml
+++ b/gora-couchdb/pom.xml
@@ -139,14 +139,6 @@
       <scope>test</scope>
     </dependency>
 
-
-    <dependency>
-      <groupId>org.apache.gora</groupId>
-      <artifactId>gora-core</artifactId>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
-
     <!-- END of Gora Internal Dependencies -->
 
     <dependency>

http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
----------------------------------------------------------------------
diff --git a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
index 65c07d9..7d9f527 100644
--- a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
+++ b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
@@ -57,7 +57,14 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.jdom.Document;
@@ -133,7 +140,7 @@ implements Configurable {
               SCANNER_CACHING_PROPERTIES_KEY,
               String.valueOf(SCANNER_CACHING_PROPERTIES_DEFAULT)))) ;
     }catch(Exception e){
-      LOG.error("Can not load " + SCANNER_CACHING_PROPERTIES_KEY + " from gora.properties.
Setting to default value: " + SCANNER_CACHING_PROPERTIES_DEFAULT, e) ;
+      LOG.error("Can not load {} from gora.properties. Setting to default value: {}.", SCANNER_CACHING_PROPERTIES_KEY,
SCANNER_CACHING_PROPERTIES_DEFAULT);
       this.setScannerCaching(SCANNER_CACHING_PROPERTIES_DEFAULT) ; // Default value if something
is wrong
     }
 
@@ -146,6 +153,7 @@ implements Configurable {
     } catch(IOException ex2){
       LOG.error(ex2.getMessage(), ex2);
     }
+    closeHBaseAdmin();
   }
 
   @Override
@@ -170,6 +178,7 @@ implements Configurable {
     } catch(IOException ex2){
       LOG.error(ex2.getMessage(), ex2);
     }
+    closeHBaseAdmin();
   }
 
   @Override
@@ -183,6 +192,7 @@ implements Configurable {
     } catch(IOException ex2){
       LOG.error(ex2.getMessage(), ex2);
     }
+    closeHBaseAdmin();
   }
 
   @Override
@@ -202,7 +212,7 @@ implements Configurable {
       Get get = new Get(toBytes(key));
       addFields(get, fields);
       Result result = table.get(get);
-      return newInstance(result, fields);      
+      return newInstance(result, fields);
     } catch(IOException ex2){
       LOG.error(ex2.getMessage(), ex2);
       return null;
@@ -572,10 +582,10 @@ implements Configurable {
       break;
     case MAP:
     case ARRAY:
-      delete.deleteFamily(col.family);
+      delete.addFamily(col.family);
       break;
     default:
-      delete.deleteColumn(col.family, col.qualifier);
+      delete.addColumn(col.family, col.qualifier);
       break;
     }
   }
@@ -756,9 +766,7 @@ implements Configurable {
           //tableNameFromMapping could be null here
           if (!tableName.equals(tableNameFromMapping)) {
           //TODO this might not be the desired behavior as the user might have actually made
a mistake.
-            LOG.warn("Mismatching schema's names. Mappingfile schema: '" + tableNameFromMapping

-                + "'. PersistentClass schema's name: '" + tableName + "'"
-                + "Assuming they are the same.");
+            LOG.warn("Mismatching schema's names. Mappingfile schema: '{}'. PersistentClass
schema's name: '{}'. Assuming they are the same.", tableNameFromMapping, tableName);
             if (tableNameFromMapping != null) {
               mappingBuilder.renameTable(tableNameFromMapping, tableName);
             }
@@ -836,10 +844,18 @@ implements Configurable {
    */
   public HBaseStore<K, T> setScannerCaching(int numRows) {
     if (numRows < 0) {
-      LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: " + numRows +
".") ;
+      LOG.warn("Invalid Scanner Caching optimization value. Cannot set to: {}.", numRows)
;
       return this ;
     }
     this.scannerCaching = numRows ;
     return this ;
   }
+  
+  private void closeHBaseAdmin(){
+    try {
+      admin.close();
+    } catch (IOException ioe) {
+      LOG.error("An error occured whilst closing HBase Admin", ioe);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java
----------------------------------------------------------------------
diff --git a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java
b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java
index 000a8b5..9f5b7c4 100644
--- a/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java
+++ b/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseTableConnection.java
@@ -18,19 +18,26 @@
 package org.apache.gora.hbase.store;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -44,7 +51,6 @@ public class HBaseTableConnection {
    * the connection when it is closed. HBase itself provides a utility called
    * HTablePool for maintaining a tPool of tables, but there are still some
    * drawbacks that are only solved in later releases.
-   *
    */
 
   private final Configuration conf;
@@ -56,26 +62,10 @@ public class HBaseTableConnection {
 
   private final BlockingQueue<Table> tPool = new LinkedBlockingQueue<>();
   private final BlockingQueue<ConcurrentLinkedQueue<Mutation>> bPool = new LinkedBlockingQueue<>();
+  @SuppressWarnings("unused")
   private final boolean autoFlush;
   private final TableName tableName;
 
-//  public class MutationPair {
-//    private Mutation mutation;
-//    private boolean type;
-//
-//    public void MutationPair(Mutation m, boolean t) {
-//      this.mutation = m;
-//      this.type = t;
-//    }
-//
-//    public boolean isType() {
-//      return type;
-//    }
-//
-//    public Mutation getMutation() {
-//      return mutation;
-//    }
-//  }
   /**
    * Instantiate new connection.
    *
@@ -101,7 +91,6 @@ public class HBaseTableConnection {
     Table table = tables.get();
     if (table == null) {
       table = connection.getTable(tableName);
-//      table.setAutoFlushTo(autoFlush);
       tPool.add(table); //keep track
       tables.set(table);
     }
@@ -111,8 +100,6 @@ public class HBaseTableConnection {
   private ConcurrentLinkedQueue<Mutation> getBuffer() throws IOException {
     ConcurrentLinkedQueue<Mutation> buffer = buffers.get();
     if (buffer == null) {
-//      BufferedMutatorParams params = new BufferedMutatorParams(this.tableName).listener(listener);
-//      buffer = connection.getBufferedMutator(this.tableName);
       buffer = new ConcurrentLinkedQueue<>();
       bPool.add(buffer);
       buffers.set(buffer);
@@ -125,9 +112,9 @@ public class HBaseTableConnection {
     for (ConcurrentLinkedQueue<Mutation> buffer : bPool) {
       for (Mutation m: buffer) {
         bufMutator.mutate(m);
-        bufMutator.flush();
       }
     }
+    bufMutator.flush();
     bufMutator.close();
   }
 
@@ -184,29 +171,18 @@ public class HBaseTableConnection {
 
   public void put(Put put) throws IOException {
     getBuffer().add(put);
-//    getBuffer().flush();
-//    getTable().put(put);
   }
 
-//  @Override
   public void put(List<Put> puts) throws IOException {
-//    getTable().put(puts);
     getBuffer().addAll(puts);
-//    getBuffer().flush();
   }
 
-//  @Override
   public void delete(Delete delete) throws IOException {
     getBuffer().add(delete);
-//    getBuffer().flush();
-//    getTable().delete(delete);
   }
 
-//  @Override
   public void delete(List<Delete> deletes) throws IOException {
-//    getTable().delete(deletes);
     getBuffer().addAll(deletes);
-//    getBuffer().flush();
   }
 
   public TableName getName() {

http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
----------------------------------------------------------------------
diff --git a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
index 87101ed..695dea1 100644
--- a/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
+++ b/gora-hbase/src/test/java/org/apache/gora/hbase/store/TestHBaseStore.java
@@ -82,7 +82,7 @@ public class TestHBaseStore extends DataStoreTestBase {
   @Override
   public void assertSchemaExists(String schemaName) throws Exception {
     HBaseAdmin admin = getTestDriver().getHbaseUtil().getHBaseAdmin();
-    assertTrue(admin.tableExists(schemaName));
+    assertTrue("Table should exist for...", admin.tableExists(schemaName));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/gora/blob/f5503ef3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 062c7d9..679afa0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -685,8 +685,8 @@
     <!-- Hadoop Dependencies -->
     <hadoop-2.version>2.5.2</hadoop-2.version>
     <hadoop-2.test.version>2.5.2</hadoop-2.test.version>
-    <hbase.version>1.2.2</hbase.version>
-    <hbase.test.version>1.2.2</hbase.test.version>
+    <hbase.version>1.2.3</hbase.version>
+    <hbase.test.version>1.2.3</hbase.test.version>
     <cxf-rt-frontend-jaxrs.version>2.5.2</cxf-rt-frontend-jaxrs.version>
     <!-- Amazon Dependencies -->
     <amazon.version>1.10.55</amazon.version>


Mime
View raw message