From commits-return-21479-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Wed Feb 14 22:45:33 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 1BD51180621 for ; Wed, 14 Feb 2018 22:45:32 +0100 (CET) Received: (qmail 25346 invoked by uid 500); 14 Feb 2018 21:45:32 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 25337 invoked by uid 99); 14 Feb 2018 21:45:32 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Feb 2018 21:45:32 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 972E882494; Wed, 14 Feb 2018 21:45:31 +0000 (UTC) Date: Wed, 14 Feb 2018 21:45:31 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch 1.8 updated: ACCUMULO-4798 optimized stat in ZooCache (#376) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <151864473152.22349.16407555143016859864@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/1.8 X-Git-Reftype: branch X-Git-Oldrev: 9771aac269640ad25f5c7dc7f64d55e8316c1825 X-Git-Newrev: eb15e4562d29cde70e4a02189fcba182c1d2e6b8 X-Git-Rev: eb15e4562d29cde70e4a02189fcba182c1d2e6b8 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 1.8 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/1.8 by this push: new eb15e45 ACCUMULO-4798 optimized stat in ZooCache (#376) eb15e45 is described below commit eb15e4562d29cde70e4a02189fcba182c1d2e6b8 Author: Keith Turner AuthorDate: Wed Feb 14 16:44:36 2018 -0500 ACCUMULO-4798 optimized stat in ZooCache (#376) --- .../apache/accumulo/fate/zookeeper/ZooCache.java | 72 ++++++++++++---------- .../apache/accumulo/fate/zookeeper/ZooLock.java | 7 ++- .../accumulo/fate/zookeeper/ZooCacheTest.java | 12 ++-- .../accumulo/server/master/LiveTServerSet.java | 4 +- .../org/apache/accumulo/server/util/AdminTest.java | 6 +- 5 files changed, 56 insertions(+), 45 deletions(-) diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java index 3daf20b..2ef938b 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java @@ -18,11 +18,6 @@ package org.apache.accumulo.fate.zookeeper; import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.HashMap; @@ -60,14 +55,39 @@ public class ZooCache { private final Lock cacheReadLock = cacheLock.readLock(); private final HashMap cache; - private final HashMap statCache; + private final HashMap statCache; private final HashMap> childrenCache; private final ZooReader zReader; + public static class ZcStat { + private long ephemeralOwner; + + public ZcStat() { + + } + + private ZcStat(Stat stat) { + this.ephemeralOwner = stat.getEphemeralOwner(); + } + + public long getEphemeralOwner() { + return ephemeralOwner; + } + + private void set(ZcStat cachedStat) { + this.ephemeralOwner = cachedStat.ephemeralOwner; + } + + @VisibleForTesting + public void setEphemeralOwner(long ephemeralOwner) { + this.ephemeralOwner = ephemeralOwner; + } + } + private static class ImmutableCacheCopies { final Map cache; - final Map statCache; + final Map statCache; final Map> childrenCache; final long updateCount; @@ -78,7 +98,7 @@ public class ZooCache { childrenCache = Collections.emptyMap(); } - ImmutableCacheCopies(long updateCount, Map cache, Map statCache, Map> childrenCache) { + ImmutableCacheCopies(long updateCount, Map cache, Map statCache, Map> childrenCache) { this.updateCount = updateCount; this.cache = Collections.unmodifiableMap(new HashMap<>(cache)); this.statCache = Collections.unmodifiableMap(new HashMap<>(statCache)); @@ -92,7 +112,7 @@ public class ZooCache { this.childrenCache = Collections.unmodifiableMap(new HashMap<>(childrenCache)); } - ImmutableCacheCopies(long updateCount, Map cache, Map statCache, ImmutableCacheCopies prev) { + ImmutableCacheCopies(long updateCount, Map cache, Map statCache, ImmutableCacheCopies prev) { this.updateCount = updateCount; this.cache = Collections.unmodifiableMap(new HashMap<>(cache)); this.statCache = Collections.unmodifiableMap(new HashMap<>(statCache)); @@ -328,20 +348,20 @@ public class ZooCache { * status object to populate * @return path data, or null if non-existent */ - public byte[] get(final String zPath, final Stat status) { + public byte[] get(final String zPath, final ZcStat status) { ZooRunnable zr = new ZooRunnable() { @Override public byte[] run() throws KeeperException, InterruptedException { - Stat stat = null; + ZcStat zstat = null; // only read volatile once so following code works with a consistent snapshot ImmutableCacheCopies lic = immutableCache; byte[] val = lic.cache.get(zPath); if (val != null || lic.cache.containsKey(zPath)) { if (status != null) { - stat = lic.statCache.get(zPath); - copyStats(status, stat); + zstat = lic.statCache.get(zPath); + copyStats(status, zstat); } return val; } @@ -354,7 +374,7 @@ public class ZooCache { cacheWriteLock.lock(); try { final ZooKeeper zooKeeper = getZooKeeper(); - stat = zooKeeper.exists(zPath, watcher); + Stat stat = zooKeeper.exists(zPath, watcher); byte[] data = null; if (stat == null) { if (log.isTraceEnabled()) { @@ -363,6 +383,7 @@ public class ZooCache { } else { try { data = zooKeeper.getData(zPath, watcher, stat); + zstat = new ZcStat(stat); } catch (KeeperException.BadVersionException e1) { throw new ConcurrentModificationException(); } catch (KeeperException.NoNodeException e2) { @@ -372,8 +393,8 @@ public class ZooCache { log.trace("zookeeper contained " + zPath + " " + (data == null ? null : new String(data, UTF_8))); } } - put(zPath, data, stat); - copyStats(status, stat); + put(zPath, data, zstat); + copyStats(status, zstat); return data; } finally { cacheWriteLock.unlock(); @@ -392,26 +413,13 @@ public class ZooCache { * @param cachedStat * cached statistic, that is or will be cached */ - protected void copyStats(Stat userStat, Stat cachedStat) { + protected void copyStats(ZcStat userStat, ZcStat cachedStat) { if (userStat != null && cachedStat != null) { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - cachedStat.write(dos); - dos.close(); - - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - DataInputStream dis = new DataInputStream(bais); - userStat.readFields(dis); - - dis.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } + userStat.set(cachedStat); } } - private void put(String zPath, byte[] data, Stat stat) { + private void put(String zPath, byte[] data, ZcStat stat) { cacheWriteLock.lock(); try { cache.put(zPath, data); diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java index 90fb4aa..9cf5fd4 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooLock.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; import org.apache.accumulo.fate.zookeeper.ZooUtil.LockID; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.zookeeper.KeeperException; @@ -411,7 +412,7 @@ public class ZooLock implements Watcher { if (!lid.node.equals(lockNode)) return false; - Stat stat = new Stat(); + ZcStat stat = new ZcStat(); return zc.get(lid.path + "/" + lid.node, stat) != null && stat.getEphemeralOwner() == lid.eid; } @@ -429,7 +430,7 @@ public class ZooLock implements Watcher { return zk.getData(path + "/" + lockNode, false, null); } - public static byte[] getLockData(org.apache.accumulo.fate.zookeeper.ZooCache zc, String path, Stat stat) { + public static byte[] getLockData(org.apache.accumulo.fate.zookeeper.ZooCache zc, String path, ZcStat stat) { List children = zc.getChildren(path); @@ -461,7 +462,7 @@ public class ZooLock implements Watcher { String lockNode = children.get(0); - Stat stat = new Stat(); + ZcStat stat = new ZcStat(); if (zc.get(path + "/" + lockNode, stat) != null) return stat.getEphemeralOwner(); return 0; diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java index 6c35ed1..6d323b7 100644 --- a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java +++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooCacheTest.java @@ -33,7 +33,9 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import java.util.List; +import java.util.Random; +import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; @@ -75,13 +77,13 @@ public class ZooCacheTest { } private void testGet(boolean fillStat) throws Exception { - Stat myStat = null; + ZcStat myStat = null; if (fillStat) { - myStat = new Stat(); + myStat = new ZcStat(); } - long now = System.currentTimeMillis(); + final long ephemeralOwner = new Random().nextLong(); Stat existsStat = new Stat(); - existsStat.setMtime(now); + existsStat.setEphemeralOwner(ephemeralOwner); expect(zk.exists(eq(ZPATH), anyObject(Watcher.class))).andReturn(existsStat); expect(zk.getData(eq(ZPATH), anyObject(Watcher.class), eq(existsStat))).andReturn(DATA); replay(zk); @@ -90,7 +92,7 @@ public class ZooCacheTest { assertArrayEquals(DATA, (fillStat ? zc.get(ZPATH, myStat) : zc.get(ZPATH))); verify(zk); if (fillStat) { - assertEquals(now, myStat.getMtime()); + assertEquals(ephemeralOwner, myStat.getEphemeralOwner()); } assertTrue(zc.dataCached(ZPATH)); diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java index 782981c..f179acd 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java @@ -40,6 +40,7 @@ import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.HostAndPort; import org.apache.accumulo.core.util.ServerServices; import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.util.Halt; import org.apache.accumulo.server.util.time.SimpleTimer; @@ -54,7 +55,6 @@ import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NotEmptyException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -282,7 +282,7 @@ public class LiveTServerSet implements Watcher { TServerInfo info = current.get(zPath); final String lockPath = path + "/" + zPath; - Stat stat = new Stat(); + ZcStat stat = new ZcStat(); byte[] lockData = ZooLock.getLockData(getZooCache(), lockPath, stat); if (lockData == null) { diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java index cb7dc28..8d0e7f3 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/AdminTest.java @@ -24,7 +24,7 @@ import java.util.UUID; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.fate.zookeeper.ZooCache; -import org.apache.zookeeper.data.Stat; +import org.apache.accumulo.fate.zookeeper.ZooCache.ZcStat; import org.easymock.EasyMock; import org.easymock.IAnswer; import org.junit.Test; @@ -55,11 +55,11 @@ public class AdminTest { String serverPath = root + "/" + server; EasyMock.expect(zc.getChildren(serverPath)).andReturn(Collections.singletonList("child")); - EasyMock.expect(zc.get(EasyMock.eq(serverPath + "/child"), EasyMock.anyObject(Stat.class))).andAnswer(new IAnswer() { + EasyMock.expect(zc.get(EasyMock.eq(serverPath + "/child"), EasyMock.anyObject(ZcStat.class))).andAnswer(new IAnswer() { @Override public byte[] answer() throws Throwable { - Stat stat = (Stat) EasyMock.getCurrentArguments()[1]; + ZcStat stat = (ZcStat) EasyMock.getCurrentArguments()[1]; stat.setEphemeralOwner(session); return new byte[0]; } -- To stop receiving notification emails like this one, please contact kturner@apache.org.