gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject gora git commit: GORA-414 Switch to Accumulo 1.6.4 and switched a test to MiniAccumulo
Date Tue, 17 Nov 2015 16:20:11 GMT
Repository: gora
Updated Branches:
  refs/heads/master a40f72f98 -> 356c1f8a8


GORA-414 Switch to Accumulo 1.6.4 and switched a test to MiniAccumulo


Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/356c1f8a
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/356c1f8a
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/356c1f8a

Branch: refs/heads/master
Commit: 356c1f8a825f883c8de43a3e25019ff230a26457
Parents: a40f72f
Author: Keith Turner <kturner@apache.org>
Authored: Mon Nov 16 18:43:46 2015 -0500
Committer: Keith Turner <keith@deenlo.com>
Committed: Mon Nov 16 21:14:08 2015 -0500

----------------------------------------------------------------------
 gora-accumulo/pom.xml                           | 10 ++++-
 .../gora/accumulo/store/AccumuloStore.java      | 39 ++++++++++++--------
 .../gora/accumulo/GoraAccumuloTestDriver.java   |  8 +++-
 .../src/test/resources/gora.properties          |  8 ++--
 4 files changed, 43 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml
index 8ab5143..bc131fa 100644
--- a/gora-accumulo/pom.xml
+++ b/gora-accumulo/pom.xml
@@ -50,7 +50,7 @@
   </ciManagement>
 
   <properties>
-    <accumulo.version>1.5.1</accumulo.version>
+    <accumulo.version>1.6.4</accumulo.version>
     <osgi.import>*</osgi.import>
     <osgi.export>org.apache.gora.accumulo*;version="${project.version}";-noimport:=true</osgi.export>
   </properties>
@@ -127,6 +127,14 @@
     </dependency>
 
     <dependency>
+      <!-- Accumulo mini was trying to load the Shell, which failed with the version of
jline in the parent pom -->
+      <groupId>jline</groupId>
+      <artifactId>jline</artifactId>
+      <version>2.11</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
       <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
index 31141c3..e6b6e65 100644
--- a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
+++ b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
@@ -35,7 +35,6 @@ import java.util.concurrent.TimeUnit;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 
-import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
@@ -67,11 +66,10 @@ import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.SortedKeyIterator;
 import org.apache.accumulo.core.iterators.user.TimestampFilter;
 import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.CredentialHelper;
-import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.security.Credentials;
 import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.TextUtil;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
@@ -121,7 +119,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
   private Connector conn;
   private BatchWriter batchWriter;
   private AccumuloMapping mapping;
-  private TCredentials credentials;
+  private Credentials credentials;
   private Encoder encoder;
 
   public static final Logger LOG = LoggerFactory.getLogger(AccumuloStore.class);
@@ -188,6 +186,15 @@ public class AccumuloStore<K,T extends PersistentBase> extends
DataStoreBase<K,T
 
   }
 
+  private static byte[] getBytes(Text text) {
+    byte[] bytes = text.getBytes();
+    if (bytes.length != text.getLength()) {
+      bytes = new byte[text.getLength()];
+      System.arraycopy(text.getBytes(), 0, bytes, 0, bytes.length);
+    }
+    return bytes;
+  }
+
   public K fromBytes(Class<K> clazz, byte[] val) {
     return fromBytes(encoder, clazz, val);
   }
@@ -363,7 +370,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
         } else {
           conn = new MockInstance().getConnector(user, token);
         }
-        credentials = CredentialHelper.create(user, token, conn.getInstance().getInstanceID());
+        credentials = new Credentials(user, token);
 
         if (autoCreateSchema && !schemaExists())
           createSchema();
@@ -452,8 +459,10 @@ public class AccumuloStore<K,T extends PersistentBase> extends
DataStoreBase<K,T
         conn.tableOperations().setProperty(mapping.tableName, entry.getKey(), entry.getValue());
       }
 
-    } catch (AccumuloException | AccumuloSecurityException | TableExistsException e) {
+    } catch (AccumuloException | AccumuloSecurityException e) {
       LOG.error(e.getMessage(), e);
+    } catch (TableExistsException e) {
+      LOG.debug(e.getMessage(), e);
     }
   }
 
@@ -614,7 +623,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
   public T get(K key, String[] fields) {
     try {
       // TODO make isolated scanner optional?
-      Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS));
+      Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
       Range rowRange = new Range(new Text(toBytes(key)));
 
       scanner.setRange(rowRange);
@@ -818,7 +827,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
 
   private Scanner createScanner(Query<K,T> query) throws TableNotFoundException {
     // TODO make isolated scanner optional?
-    Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS));
+    Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
     setFetchColumns(scanner, query.getFields());
 
     scanner.setRange(createRange(query));
@@ -871,12 +880,12 @@ public class AccumuloStore<K,T extends PersistentBase> extends
DataStoreBase<K,T
       if (conn instanceof MockConnector)
         tl = new MockTabletLocator();
       else
-        tl = TabletLocator.getInstance(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(),
mapping.tableName)));
+        tl = TabletLocator.getLocator(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(),
mapping.tableName)));
 
       Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
 
       tl.invalidateCache();
-      while (tl.binRanges(Collections.singletonList(createRange(query)), binnedRanges, credentials).size()
> 0) {
+      while (tl.binRanges(credentials, Collections.singletonList(createRange(query)), binnedRanges).size()
> 0) {
         // TODO log?
         if (!Tables.exists(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)))
           throw new TableDeletedException(Tables.getTableId(conn.getInstance(), mapping.tableName));
@@ -913,18 +922,18 @@ public class AccumuloStore<K,T extends PersistentBase> extends
DataStoreBase<K,T
           K startKey = null;
           if (startRow == null || !ke.contains(startRow)) {
             if (ke.getPrevEndRow() != null) {
-              startKey = followingKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getPrevEndRow()));
+              startKey = followingKey(encoder, getKeyClass(), getBytes(ke.getPrevEndRow()));
             }
           } else {
-            startKey = fromBytes(getKeyClass(), TextUtil.getBytes(startRow));
+            startKey = fromBytes(getKeyClass(), getBytes(startRow));
           }
 
           K endKey = null;
           if (endRow == null || !ke.contains(endRow)) {
             if (ke.getEndRow() != null)
-              endKey = lastPossibleKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getEndRow()));
+              endKey = lastPossibleKey(encoder, getKeyClass(), getBytes(ke.getEndRow()));
           } else {
-            endKey = fromBytes(getKeyClass(), TextUtil.getBytes(endRow));
+            endKey = fromBytes(getKeyClass(), getBytes(endRow));
           }
 
           PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query, startKey,
endKey, location);

http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
index 0d33444..bd3b502 100644
--- a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
+++ b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
@@ -18,6 +18,7 @@
 package org.apache.gora.accumulo;
 
 import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
 import org.apache.gora.GoraTestDriver;
 import org.apache.gora.accumulo.store.AccumuloStore;
 import org.junit.Rule;
@@ -33,7 +34,7 @@ public class GoraAccumuloTestDriver extends GoraTestDriver {
 
   private static final Logger LOG = LoggerFactory.getLogger(GoraAccumuloTestDriver.class);
   private static MiniAccumuloCluster cluster = null;
-  private static final String PASSWORD = "password";
+  private static final String PASSWORD = "drowssap";
 
   @Rule
   public TemporaryFolder tmpDir = new TemporaryFolder();
@@ -48,7 +49,10 @@ public class GoraAccumuloTestDriver extends GoraTestDriver {
     log.info("Starting Accumulo MiniAccumuloCluster...");
     try {
       tmpDir.create();
-      cluster = new MiniAccumuloCluster(tmpDir.getRoot(), PASSWORD);
+      MiniAccumuloConfig miniCfg = new MiniAccumuloConfig(tmpDir.getRoot(), PASSWORD);
+      miniCfg.setInstanceName("goraTest");
+      miniCfg.setZooKeeperPort(56321);
+      cluster = new MiniAccumuloCluster(miniCfg);
       cluster.start();
     } catch (Exception e) {
       LOG.error("Error starting Accumulo MiniAccumuloCluster: {}", e.getMessage());

http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/test/resources/gora.properties b/gora-accumulo/src/test/resources/gora.properties
index f89a360..8d1c4f5 100644
--- a/gora-accumulo/src/test/resources/gora.properties
+++ b/gora-accumulo/src/test/resources/gora.properties
@@ -14,8 +14,8 @@
 # limitations under the License.
 
 gora.datastore.default=org.apache.gora.accumulo.store.AccumuloStore
-gora.datastore.accumulo.mock=true
-gora.datastore.accumulo.instance=a14
-gora.datastore.accumulo.zookeepers=localhost
+gora.datastore.accumulo.mock=false
+gora.datastore.accumulo.instance=goraTest
+gora.datastore.accumulo.zookeepers=localhost:56321
 gora.datastore.accumulo.user=root
-gora.datastore.accumulo.password=
\ No newline at end of file
+gora.datastore.accumulo.password=drowssap
\ No newline at end of file


Mime
View raw message