accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch 1.8 updated: ACCUMULO-4798 optimized stat in ZooCache (#376)
Date Wed, 14 Feb 2018 21:45:31 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1.8
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.8 by this push:
     new eb15e45  ACCUMULO-4798 optimized stat in ZooCache (#376)
eb15e45 is described below

commit eb15e4562d29cde70e4a02189fcba182c1d2e6b8
Author: Keith Turner <kturner@apache.org>
AuthorDate: Wed Feb 14 16:44:36 2018 -0500

    ACCUMULO-4798 optimized stat in ZooCache (#376)
---
 .../apache/accumulo/fate/zookeeper/ZooCache.java   | 72 ++++++++++++----------
 .../apache/accumulo/fate/zookeeper/ZooLock.java    |  7 ++-
 .../accumulo/fate/zookeeper/ZooCacheTest.java      | 12 ++--
 .../accumulo/server/master/LiveTServerSet.java     |  4 +-
 .../org/apache/accumulo/server/util/AdminTest.java |  6 +-
 5 files changed, 56 insertions(+), 45 deletions(-)

diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
index 3daf20b..2ef938b 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
@@ -18,11 +18,6 @@ package org.apache.accumulo.fate.zookeeper;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.ConcurrentModificationException;
 import java.util.HashMap;
@@ -60,14 +55,39 @@ public class ZooCache {
   private final Lock cacheReadLock = cacheLock.readLock();
 
   private final HashMap<String,byte[]> cache;
-  private final HashMap<String,Stat> statCache;
+  private final HashMap<String,ZcStat> statCache;
   private final HashMap<String,List<String>> childrenCache;
 
   private final ZooReader zReader;
 
+  public static class ZcStat {
+    private long ephemeralOwner;
+
+    public ZcStat() {
+
+    }
+
+    private ZcStat(Stat stat) {
+      this.ephemeralOwner = stat.getEphemeralOwner();
+    }
+
+    public long getEphemeralOwner() {
+      return ephemeralOwner;
+    }
+
+    private void set(ZcStat cachedStat) {
+      this.ephemeralOwner = cachedStat.ephemeralOwner;
+    }
+
+    @VisibleForTesting
+    public void setEphemeralOwner(long ephemeralOwner) {
+      this.ephemeralOwner = ephemeralOwner;
+    }
+  }
+
   private static class ImmutableCacheCopies {
     final Map<String,byte[]> cache;
-    final Map<String,Stat> statCache;
+    final Map<String,ZcStat> statCache;
     final Map<String,List<String>> childrenCache;
     final long updateCount;
 
@@ -78,7 +98,7 @@ public class ZooCache {
       childrenCache = Collections.emptyMap();
     }
 
-    ImmutableCacheCopies(long updateCount, Map<String,byte[]> cache, Map<String,Stat>
statCache, Map<String,List<String>> childrenCache) {
+    ImmutableCacheCopies(long updateCount, Map<String,byte[]> cache, Map<String,ZcStat>
statCache, Map<String,List<String>> childrenCache) {
       this.updateCount = updateCount;
       this.cache = Collections.unmodifiableMap(new HashMap<>(cache));
       this.statCache = Collections.unmodifiableMap(new HashMap<>(statCache));
@@ -92,7 +112,7 @@ public class ZooCache {
       this.childrenCache = Collections.unmodifiableMap(new HashMap<>(childrenCache));
     }
 
-    ImmutableCacheCopies(long updateCount, Map<String,byte[]> cache, Map<String,Stat>
statCache, ImmutableCacheCopies prev) {
+    ImmutableCacheCopies(long updateCount, Map<String,byte[]> cache, Map<String,ZcStat>
statCache, ImmutableCacheCopies prev) {
       this.updateCount = updateCount;
       this.cache = Collections.unmodifiableMap(new HashMap<>(cache));
       this.statCache = Collections.unmodifiableMap(new HashMap<>(statCache));
@@ -328,20 +348,20 @@ public class ZooCache {
    *          status object to populate
    * @return path data, or null if non-existent
    */
-  public byte[] get(final String zPath, final Stat status) {
+  public byte[] get(final String zPath, final ZcStat status) {
     ZooRunnable<byte[]> zr = new ZooRunnable<byte[]>() {
 
       @Override
       public byte[] run() throws KeeperException, InterruptedException {
-        Stat stat = null;
+        ZcStat zstat = null;
 
         // only read volatile once so following code works with a consistent snapshot
         ImmutableCacheCopies lic = immutableCache;
         byte[] val = lic.cache.get(zPath);
         if (val != null || lic.cache.containsKey(zPath)) {
           if (status != null) {
-            stat = lic.statCache.get(zPath);
-            copyStats(status, stat);
+            zstat = lic.statCache.get(zPath);
+            copyStats(status, zstat);
           }
           return val;
         }
@@ -354,7 +374,7 @@ public class ZooCache {
         cacheWriteLock.lock();
         try {
           final ZooKeeper zooKeeper = getZooKeeper();
-          stat = zooKeeper.exists(zPath, watcher);
+          Stat stat = zooKeeper.exists(zPath, watcher);
           byte[] data = null;
           if (stat == null) {
             if (log.isTraceEnabled()) {
@@ -363,6 +383,7 @@ public class ZooCache {
           } else {
             try {
               data = zooKeeper.getData(zPath, watcher, stat);
+              zstat = new ZcStat(stat);
             } catch (KeeperException.BadVersionException e1) {
               throw new ConcurrentModificationException();
             } catch (KeeperException.NoNodeException e2) {
@@ -372,8 +393,8 @@ public class ZooCache {
               log.trace("zookeeper contained " + zPath + " " + (data == null ? null : new
String(data, UTF_8)));
             }
           }
-          put(zPath, data, stat);
-          copyStats(status, stat);
+          put(zPath, data, zstat);
+          copyStats(status, zstat);
           return data;
         } finally {
           cacheWriteLock.unlock();
@@ -392,26 +413,13 @@ public class ZooCache {
    * @param cachedStat
    *          cached statistic, that is or will be cached
    */
-  protected void copyStats(Stat userStat, Stat cachedStat) {
+  protected void copyStats(ZcStat userStat, ZcStat cachedStat) {
     if (userStat != null && cachedStat != null) {
-      try {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        DataOutputStream dos = new DataOutputStream(baos);
-        cachedStat.write(dos);
-        dos.close();
-
-        ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
-        DataInputStream dis = new DataInputStream(bais);
-        userStat.readFields(dis);
-
-        dis.close();
-      } catch (IOException e) {
-        throw new RuntimeException(e);
-      }
+      userStat.set(cachedStat);
     }
   }
 
-  private void put(String zPath, byte[] data, Stat stat) {
+  private void put(String zPath, byte[] data, ZcStat stat) {
     cacheWriteLock.lock();
     try {
       cache.put(zPath, data);
diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
index 90fb4aa..9cf5fd4 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat;
 import org.apache.accumulo.fate.zookeeper.ZooUtil.LockID;
 import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
 import org.apache.zookeeper.KeeperException;
@@ -411,7 +412,7 @@ public class ZooLock implements Watcher {
     if (!lid.node.equals(lockNode))
       return false;
 
-    Stat stat = new Stat();
+    ZcStat stat = new ZcStat();
     return zc.get(lid.path + "/" + lid.node, stat) != null && stat.getEphemeralOwner()
== lid.eid;
   }
 
@@ -429,7 +430,7 @@ public class ZooLock implements Watcher {
     return zk.getData(path + "/" + lockNode, false, null);
   }
 
-  public static byte[] getLockData(org.apache.accumulo.fate.zookeeper.ZooCache zc, String
path, Stat stat) {
+  public static byte[] getLockData(org.apache.accumulo.fate.zookeeper.ZooCache zc, String
path, ZcStat stat) {
 
     List<String> children = zc.getChildren(path);
 
@@ -461,7 +462,7 @@ public class ZooLock implements Watcher {
 
     String lockNode = children.get(0);
 
-    Stat stat = new Stat();
+    ZcStat stat = new ZcStat();
     if (zc.get(path + "/" + lockNode, stat) != null)
       return stat.getEphemeralOwner();
     return 0;
diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java
index 6c35ed1..6d323b7 100644
--- a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java
+++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java
@@ -33,7 +33,9 @@ import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 import java.util.List;
+import java.util.Random;
 
+import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
@@ -75,13 +77,13 @@ public class ZooCacheTest {
   }
 
   private void testGet(boolean fillStat) throws Exception {
-    Stat myStat = null;
+    ZcStat myStat = null;
     if (fillStat) {
-      myStat = new Stat();
+      myStat = new ZcStat();
     }
-    long now = System.currentTimeMillis();
+    final long ephemeralOwner = new Random().nextLong();
     Stat existsStat = new Stat();
-    existsStat.setMtime(now);
+    existsStat.setEphemeralOwner(ephemeralOwner);
     expect(zk.exists(eq(ZPATH), anyObject(Watcher.class))).andReturn(existsStat);
     expect(zk.getData(eq(ZPATH), anyObject(Watcher.class), eq(existsStat))).andReturn(DATA);
     replay(zk);
@@ -90,7 +92,7 @@ public class ZooCacheTest {
     assertArrayEquals(DATA, (fillStat ? zc.get(ZPATH, myStat) : zc.get(ZPATH)));
     verify(zk);
     if (fillStat) {
-      assertEquals(now, myStat.getMtime());
+      assertEquals(ephemeralOwner, myStat.getEphemeralOwner());
     }
 
     assertTrue(zc.dataCached(ZPATH));
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
index 782981c..f179acd 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
@@ -40,6 +40,7 @@ import org.apache.accumulo.core.util.AddressUtil;
 import org.apache.accumulo.core.util.HostAndPort;
 import org.apache.accumulo.core.util.ServerServices;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.accumulo.server.util.Halt;
 import org.apache.accumulo.server.util.time.SimpleTimer;
@@ -54,7 +55,6 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.KeeperException.NotEmptyException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -282,7 +282,7 @@ public class LiveTServerSet implements Watcher {
     TServerInfo info = current.get(zPath);
 
     final String lockPath = path + "/" + zPath;
-    Stat stat = new Stat();
+    ZcStat stat = new ZcStat();
     byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat);
 
     if (lockData == null) {
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java
index cb7dc28..8d0e7f3 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java
@@ -24,7 +24,7 @@ import java.util.UUID;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.fate.zookeeper.ZooCache;
-import org.apache.zookeeper.data.Stat;
+import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat;
 import org.easymock.EasyMock;
 import org.easymock.IAnswer;
 import org.junit.Test;
@@ -55,11 +55,11 @@ public class AdminTest {
 
     String serverPath = root + "/" + server;
     EasyMock.expect(zc.getChildren(serverPath)).andReturn(Collections.singletonList("child"));
-    EasyMock.expect(zc.get(EasyMock.eq(serverPath + "/child"), EasyMock.anyObject(Stat.class))).andAnswer(new
IAnswer<byte[]>() {
+    EasyMock.expect(zc.get(EasyMock.eq(serverPath + "/child"), EasyMock.anyObject(ZcStat.class))).andAnswer(new
IAnswer<byte[]>() {
 
       @Override
       public byte[] answer() throws Throwable {
-        Stat stat = (Stat) EasyMock.getCurrentArguments()[1];
+        ZcStat stat = (ZcStat) EasyMock.getCurrentArguments()[1];
         stat.setEphemeralOwner(session);
         return new byte[0];
       }

-- 
To stop receiving notification emails like this one, please contact
kturner@apache.org.

Mime
View raw message