zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cami...@apache.org
Subject svn commit: r1736936 - in /zookeeper/branches/branch-3.4: ./ src/java/main/org/apache/zookeeper/server/ src/java/main/org/apache/zookeeper/server/upgrade/ src/java/test/org/apache/zookeeper/server/
Date Mon, 28 Mar 2016 23:14:15 GMT
Author: camille
Date: Mon Mar 28 23:14:14 2016
New Revision: 1736936

URL: http://svn.apache.org/viewvc?rev=1736936&view=rev
Log:
ZOOKEEPER-2141 ACL cache in DataTree never removes entries (Adam Milne-Smith via camille)

Added:
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ReferenceCountedACLCache.java
    zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ReferenceCountedACLCacheTest.java
Modified:
    zookeeper/branches/branch-3.4/CHANGES.txt
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/DataTree.java
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZKDatabase.java
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/DataNodeV1.java
    zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/UpgradeSnapShotV1.java

Modified: zookeeper/branches/branch-3.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/CHANGES.txt?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/CHANGES.txt (original)
+++ zookeeper/branches/branch-3.4/CHANGES.txt Mon Mar 28 23:14:14 2016
@@ -26,6 +26,8 @@ BUGFIXES:
 
   ZOOKEEPER-2195 fsync.warningthresholdms in zoo.cfg not working
   (Biju Nair via phunt)
+  
+  ZOOKEEPER-2141 ACL cache in DataTree never removes entries (Adam Milne-Smith via camille)
 
 IMPROVEMENTS:
 

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/DataTree.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/DataTree.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/DataTree.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/DataTree.java
Mon Mar 28 23:14:14 2016
@@ -22,16 +22,13 @@ import java.io.IOException;
 import java.io.PrintWriter;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
-import org.apache.jute.Index;
 import org.apache.jute.InputArchive;
 import org.apache.jute.OutputArchive;
 import org.apache.jute.Record;
@@ -45,12 +42,12 @@ import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event;
 import org.apache.zookeeper.Watcher.Event.EventType;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
-import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooDefs.OpCode;
 import org.apache.zookeeper.common.PathTrie;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.data.StatPersisted;
+import org.apache.zookeeper.server.upgrade.DataNodeV1;
 import org.apache.zookeeper.txn.CheckVersionTxn;
 import org.apache.zookeeper.txn.CreateTxn;
 import org.apache.zookeeper.txn.DeleteTxn;
@@ -116,23 +113,7 @@ public class DataTree {
     private final Map<Long, HashSet<String>> ephemerals =
         new ConcurrentHashMap<Long, HashSet<String>>();
 
-    /**
-     * this is map from longs to acl's. It saves acl's being stored for each
-     * datanode.
-     */
-    public final Map<Long, List<ACL>> longKeyMap =
-        new HashMap<Long, List<ACL>>();
-
-    /**
-     * this a map from acls to long.
-     */
-    public final Map<List<ACL>, Long> aclKeyMap =
-        new HashMap<List<ACL>, Long>();
-
-    /**
-     * these are the number of acls that we have in the datatree
-     */
-    protected long aclIndex = 0;
+    private final ReferenceCountedACLCache aclCache = new ReferenceCountedACLCache();
 
     @SuppressWarnings("unchecked")
     public HashSet<String> getEphemerals(long sessionId) {
@@ -151,74 +132,6 @@ public class DataTree {
         return ephemerals;
     }
 
-    private long incrementIndex() {
-        return ++aclIndex;
-    }
-
-    /**
-     * compare two list of acls. if there elements are in the same order and the
-     * same size then return true else return false
-     *
-     * @param lista
-     *            the list to be compared
-     * @param listb
-     *            the list to be compared
-     * @return true if and only if the lists are of the same size and the
-     *         elements are in the same order in lista and listb
-     */
-    private boolean listACLEquals(List<ACL> lista, List<ACL> listb) {
-        if (lista.size() != listb.size()) {
-            return false;
-        }
-        for (int i = 0; i < lista.size(); i++) {
-            ACL a = lista.get(i);
-            ACL b = listb.get(i);
-            if (!a.equals(b)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /**
-     * converts the list of acls to a list of longs.
-     *
-     * @param acls
-     * @return a list of longs that map to the acls
-     */
-    public synchronized Long convertAcls(List<ACL> acls) {
-        if (acls == null)
-            return -1L;
-        // get the value from the map
-        Long ret = aclKeyMap.get(acls);
-        // could not find the map
-        if (ret != null)
-            return ret;
-        long val = incrementIndex();
-        longKeyMap.put(val, acls);
-        aclKeyMap.put(acls, val);
-        return val;
-    }
-
-    /**
-     * converts a list of longs to a list of acls.
-     *
-     * @param longVal
-     *            the list of longs
-     * @return a list of ACLs that map to longs
-     */
-    public synchronized List<ACL> convertLong(Long longVal) {
-        if (longVal == null)
-            return null;
-        if (longVal == -1L)
-            return Ids.OPEN_ACL_UNSAFE;
-        List<ACL> acls = longKeyMap.get(longVal);
-        if (acls == null) {
-            LOG.error("ERROR: ACL not available for long " + longVal);
-            throw new RuntimeException("Failed to fetch acls for " + longVal);
-        }
-        return acls;
-    }
 
     public Collection<Long> getSessions() {
         return ephemerals.keySet();
@@ -484,7 +397,7 @@ public class DataTree {
             }    
             parent.stat.setCversion(parentCVersion);
             parent.stat.setPzxid(zxid);
-            Long longval = convertAcls(acl);
+            Long longval = aclCache.convertAcls(acl);
             DataNode child = new DataNode(parent, data, longval, stat);
             parent.addChild(childName);
             nodes.put(path, child);
@@ -544,6 +457,9 @@ public class DataTree {
             throw new KeeperException.NoNodeException();
         }
         nodes.remove(path);
+        synchronized (node) {
+            aclCache.removeUsage(node.acl);
+        }
         DataNode parent = nodes.get(parentName);
         if (parent == null) {
             throw new KeeperException.NoNodeException();
@@ -706,8 +622,9 @@ public class DataTree {
             throw new KeeperException.NoNodeException();
         }
         synchronized (n) {
+            aclCache.removeUsage(n.acl);
             n.stat.setAversion(version);
-            n.acl = convertAcls(acl);
+            n.acl = aclCache.convertAcls(acl);
             n.copyStat(stat);
             return stat;
         }
@@ -722,10 +639,26 @@ public class DataTree {
         }
         synchronized (n) {
             n.copyStat(stat);
-            return new ArrayList<ACL>(convertLong(n.acl));
+            return new ArrayList<ACL>(aclCache.convertLong(n.acl));
+        }
+    }
+
+    public List<ACL> getACL(DataNode node) {
+        synchronized (node) {
+            return aclCache.convertLong(node.acl);
+        }
+    }
+
+    public Long getACL(DataNodeV1 oldDataNode) {
+        synchronized (oldDataNode) {
+            return aclCache.convertAcls(oldDataNode.acl);
         }
     }
 
+    public int aclCacheSize() {
+        return aclCache.size();
+    }
+
     static public class ProcessTxnResult {
         public long clientId;
 
@@ -1142,46 +1075,9 @@ public class DataTree {
 
     public boolean initialized = false;
 
-    private void deserializeList(Map<Long, List<ACL>> longKeyMap,
-            InputArchive ia) throws IOException {
-        int i = ia.readInt("map");
-        while (i > 0) {
-            Long val = ia.readLong("long");
-            if (aclIndex < val) {
-                aclIndex = val;
-            }
-            List<ACL> aclList = new ArrayList<ACL>();
-            Index j = ia.startVector("acls");
-            while (!j.done()) {
-                ACL acl = new ACL();
-                acl.deserialize(ia, "acl");
-                aclList.add(acl);
-                j.incr();
-            }
-            longKeyMap.put(val, aclList);
-            aclKeyMap.put(aclList, val);
-            i--;
-        }
-    }
-
-    private synchronized void serializeList(Map<Long, List<ACL>> longKeyMap,
-            OutputArchive oa) throws IOException {
-        oa.writeInt(longKeyMap.size(), "map");
-        Set<Map.Entry<Long, List<ACL>>> set = longKeyMap.entrySet();
-        for (Map.Entry<Long, List<ACL>> val : set) {
-            oa.writeLong(val.getKey(), "long");
-            List<ACL> aclList = val.getValue();
-            oa.startVector(aclList, "acls");
-            for (ACL acl : aclList) {
-                acl.serialize(oa, "acl");
-            }
-            oa.endVector(aclList, "acls");
-        }
-    }
-
     public void serialize(OutputArchive oa, String tag) throws IOException {
         scount = 0;
-        serializeList(longKeyMap, oa);
+        aclCache.serialize(oa);
         serializeNode(oa, new StringBuilder(""));
         // / marks end of stream
         // we need to check if clear had been called in between the snapshot.
@@ -1191,7 +1087,7 @@ public class DataTree {
     }
 
     public void deserialize(InputArchive ia, String tag) throws IOException {
-        deserializeList(longKeyMap, ia);
+        aclCache.deserialize(ia);
         nodes.clear();
         pTrie.clear();
         String path = ia.readString("path");
@@ -1199,6 +1095,9 @@ public class DataTree {
             DataNode node = new DataNode();
             ia.readRecord(node, "node");
             nodes.put(path, node);
+            synchronized (node) {
+                aclCache.addUsage(node.acl);
+            }
             int lastSlash = path.lastIndexOf('/');
             if (lastSlash == -1) {
                 root = node;
@@ -1228,6 +1127,8 @@ public class DataTree {
         // update the quotas - create path trie
         // and also update the stat nodes
         setupQuota();
+
+        aclCache.purgeUnused();
     }
 
     /**

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/FinalRequestProcessor.java
Mon Mar 28 23:14:14 2016
@@ -281,11 +281,7 @@ public class FinalRequestProcessor imple
                 if (n == null) {
                     throw new KeeperException.NoNodeException();
                 }
-                Long aclL;
-                synchronized(n) {
-                    aclL = n.acl;
-                }
-                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().convertLong(aclL),
+                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().aclForNode(n),
                         ZooDefs.Perms.READ,
                         request.authInfo);
                 Stat stat = new Stat();
@@ -327,12 +323,7 @@ public class FinalRequestProcessor imple
                 if (n == null) {
                     throw new KeeperException.NoNodeException();
                 }
-                Long aclG;
-                synchronized(n) {
-                    aclG = n.acl;
-                    
-                }
-                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().convertLong(aclG),

+                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().aclForNode(n),
                         ZooDefs.Perms.READ,
                         request.authInfo);
                 List<String> children = zks.getZKDatabase().getChildren(
@@ -351,11 +342,7 @@ public class FinalRequestProcessor imple
                 if (n == null) {
                     throw new KeeperException.NoNodeException();
                 }
-                Long aclG;
-                synchronized(n) {
-                    aclG = n.acl;
-                }
-                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().convertLong(aclG),

+                PrepRequestProcessor.checkACL(zks, zks.getZKDatabase().aclForNode(n),
                         ZooDefs.Perms.READ,
                         request.authInfo);
                 List<String> children = zks.getZKDatabase().getChildren(

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/PrepRequestProcessor.java
Mon Mar 28 23:14:14 2016
@@ -150,15 +150,13 @@ public class PrepRequestProcessor extend
             if (lastChange == null) {
                 DataNode n = zks.getZKDatabase().getNode(path);
                 if (n != null) {
-                    Long acl;
                     Set<String> children;
                     synchronized(n) {
-                        acl = n.acl;
                         children = n.getChildren();
                     }
                     lastChange = new ChangeRecord(-1, path, n.stat,
                         children != null ? children.size() : 0,
-                            zks.getZKDatabase().convertLong(acl));
+                            zks.getZKDatabase().aclForNode(n));
                 }
             }
         }

Added: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ReferenceCountedACLCache.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ReferenceCountedACLCache.java?rev=1736936&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ReferenceCountedACLCache.java
(added)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ReferenceCountedACLCache.java
Mon Mar 28 23:14:14 2016
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import org.apache.jute.Index;
+import org.apache.jute.InputArchive;
+import org.apache.jute.OutputArchive;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class ReferenceCountedACLCache {
+    private static final Logger LOG = LoggerFactory.getLogger(ReferenceCountedACLCache.class);
+
+    /**
+     * this is map from longs to acl's. It saves acl's being stored for each
+     * datanode.
+     */
+    private final Map<Long, List<ACL>> longKeyMap =
+            new HashMap<Long, List<ACL>>();
+
+    /**
+     * this a map from acls to long.
+     */
+    private final Map<List<ACL>, Long> aclKeyMap =
+            new HashMap<List<ACL>, Long>();
+
+    private final Map<Long, AtomicLongWithEquals> referenceCounter =
+            new HashMap<Long, AtomicLongWithEquals>();
+    private final long OPEN_UNSAFE_ACL_ID = -1L;
+
+    /**
+     * these are the number of acls that we have in the datatree
+     */
+    private long aclIndex = 0;
+
+    /**
+     * converts the list of acls to a list of longs.
+     * Increments the reference counter for this ACL.
+     * @param acls
+     * @return a list of longs that map to the acls
+     */
+    public synchronized Long convertAcls(List<ACL> acls) {
+        if (acls == null)
+            return OPEN_UNSAFE_ACL_ID;
+
+        // get the value from the map
+        Long ret = aclKeyMap.get(acls);
+        if (ret == null) {
+            ret = incrementIndex();
+            longKeyMap.put(ret, acls);
+            aclKeyMap.put(acls, ret);
+        }
+
+        addUsage(ret);
+
+        return ret;
+    }
+
+    /**
+     * converts a list of longs to a list of acls.
+     *
+     * @param longVal
+     *            the list of longs
+     * @return a list of ACLs that map to longs
+     */
+    public synchronized List<ACL> convertLong(Long longVal) {
+        if (longVal == null)
+            return null;
+        if (longVal == OPEN_UNSAFE_ACL_ID)
+            return ZooDefs.Ids.OPEN_ACL_UNSAFE;
+        List<ACL> acls = longKeyMap.get(longVal);
+        if (acls == null) {
+            LOG.error("ERROR: ACL not available for long " + longVal);
+            throw new RuntimeException("Failed to fetch acls for " + longVal);
+        }
+        return acls;
+    }
+
+    private long incrementIndex() {
+        return ++aclIndex;
+    }
+
+    public synchronized void deserialize(InputArchive ia) throws IOException {
+        clear();
+        int i = ia.readInt("map");
+        while (i > 0) {
+            Long val = ia.readLong("long");
+            if (aclIndex < val) {
+                aclIndex = val;
+            }
+            List<ACL> aclList = new ArrayList<ACL>();
+            Index j = ia.startVector("acls");
+            while (!j.done()) {
+                ACL acl = new ACL();
+                acl.deserialize(ia, "acl");
+                aclList.add(acl);
+                j.incr();
+            }
+            longKeyMap.put(val, aclList);
+            aclKeyMap.put(aclList, val);
+            referenceCounter.put(val, new AtomicLongWithEquals(0));
+            i--;
+        }
+    }
+
+    public synchronized void serialize(OutputArchive oa) throws IOException {
+        oa.writeInt(longKeyMap.size(), "map");
+        Set<Map.Entry<Long, List<ACL>>> set = longKeyMap.entrySet();
+        for (Map.Entry<Long, List<ACL>> val : set) {
+            oa.writeLong(val.getKey(), "long");
+            List<ACL> aclList = val.getValue();
+            oa.startVector(aclList, "acls");
+            for (ACL acl : aclList) {
+                acl.serialize(oa, "acl");
+            }
+            oa.endVector(aclList, "acls");
+        }
+    }
+
+    public int size() {
+        return aclKeyMap.size();
+    }
+
+    private void clear() {
+        aclKeyMap.clear();
+        longKeyMap.clear();
+        referenceCounter.clear();
+    }
+
+    public synchronized void addUsage(Long acl) {
+        if (acl == OPEN_UNSAFE_ACL_ID) {
+            return;
+        }
+
+        if (!longKeyMap.containsKey(acl)) {
+            LOG.info("Ignoring acl " + acl + " as it does not exist in the cache");
+            return;
+        }
+
+        AtomicLong count = referenceCounter.get(acl);
+        if (count == null) {
+            referenceCounter.put(acl, new AtomicLongWithEquals(1));
+        } else {
+            count.incrementAndGet();
+        }
+    }
+
+    public synchronized void removeUsage(Long acl) {
+        if (acl == OPEN_UNSAFE_ACL_ID) {
+            return;
+        }
+
+        if (!longKeyMap.containsKey(acl)) {
+            LOG.info("Ignoring acl " + acl + " as it does not exist in the cache");
+            return;
+        }
+
+        long newCount = referenceCounter.get(acl).decrementAndGet();
+        if (newCount <= 0) {
+            referenceCounter.remove(acl);
+            aclKeyMap.remove(longKeyMap.get(acl));
+            longKeyMap.remove(acl);
+        }
+    }
+
+    public synchronized void purgeUnused() {
+        Iterator<Map.Entry<Long, AtomicLongWithEquals>> refCountIter = referenceCounter.entrySet().iterator();
+        while (refCountIter.hasNext()) {
+            Map.Entry<Long, AtomicLongWithEquals> entry = refCountIter.next();
+            if (entry.getValue().get() <= 0) {
+                Long acl = entry.getKey();
+                aclKeyMap.remove(longKeyMap.get(acl));
+                longKeyMap.remove(acl);
+                refCountIter.remove();
+            }
+        }
+    }
+
+    @Override
+    public synchronized boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ReferenceCountedACLCache that = (ReferenceCountedACLCache) o;
+        synchronized (that) {
+            if (aclIndex != that.aclIndex) return false;
+        }
+        if (aclKeyMap != null ? !aclKeyMap.equals(that.aclKeyMap) : that.aclKeyMap != null)
return false;
+        if (longKeyMap != null ? !longKeyMap.equals(that.longKeyMap) : that.longKeyMap !=
null) return false;
+        if (referenceCounter != null ? !referenceCounter.equals(that.referenceCounter) :
that.referenceCounter != null)
+            return false;
+        return true;
+    }
+
+    @Override
+    public synchronized int hashCode() {
+        int result = longKeyMap != null ? longKeyMap.hashCode() : 0;
+        result = 31 * result + (aclKeyMap != null ? aclKeyMap.hashCode() : 0);
+        result = 31 * result + (referenceCounter != null ? referenceCounter.hashCode() :
0);
+        result = 31 * result + (int) (aclIndex ^ (aclIndex >>> 32));
+        return result;
+    }
+
+    /*
+    For reasons we don't all agree with, AtomicLong does not have an equals.
+     */
+    private static class AtomicLongWithEquals extends AtomicLong {
+
+        private static final long serialVersionUID = 3355155896813725462L;
+
+        public AtomicLongWithEquals(long i) {
+            super(i);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            return equals((AtomicLongWithEquals) o);
+        }
+
+        public boolean equals(AtomicLongWithEquals that) {
+            return get() == that.get();
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * Long.valueOf(get()).hashCode();
+        }
+    }
+}

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZKDatabase.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZKDatabase.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZKDatabase.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZKDatabase.java
Mon Mar 28 23:14:14 2016
@@ -267,7 +267,10 @@ public class ZKDatabase {
         }
     }
 
-    
+
+    public List<ACL> aclForNode(DataNode n) {
+        return dataTree.getACL(n);
+    }
     /**
      * remove a cnxn from the datatree
      * @param cnxn the cnxn to remove from the datatree
@@ -350,15 +353,6 @@ public class ZKDatabase {
     }
 
     /**
-     * convert from long to the acl entry
-     * @param aclL the long for which to get the acl
-     * @return the acl corresponding to this long entry
-     */
-    public List<ACL> convertLong(Long aclL) {
-        return dataTree.convertLong(aclL);
-    }
-
-    /**
      * get data and stat for a path 
      * @param path the path being queried
      * @param stat the stat for this path
@@ -422,7 +416,7 @@ public class ZKDatabase {
      * @return the acl size of the datatree
      */
     public int getAclSize() {
-        return dataTree.longKeyMap.size();
+        return dataTree.aclCacheSize();
     }
 
     /**

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/DataNodeV1.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/DataNodeV1.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/DataNodeV1.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/DataNodeV1.java
Mon Mar 28 23:14:14 2016
@@ -73,7 +73,7 @@ public class DataNodeV1 implements Recor
 
     byte data[];
 
-    List<ACL> acl;
+    public List<ACL> acl;
 
     public StatPersistedV1 stat;
 

Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/UpgradeSnapShotV1.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/UpgradeSnapShotV1.java?rev=1736936&r1=1736935&r2=1736936&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/UpgradeSnapShotV1.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/upgrade/UpgradeSnapShotV1.java
Mon Mar 28 23:14:14 2016
@@ -268,7 +268,7 @@ public class UpgradeSnapShotV1 implement
             DataNodeV1 oldDataNode) {
         StatPersisted stat = convertStat(oldDataNode.stat);
         DataNode dataNode =  new DataNode(parent, oldDataNode.data,
-                dt.convertAcls(oldDataNode.acl), stat);
+                dt.getACL(oldDataNode), stat);
         dataNode.setChildren(oldDataNode.children);
         return dataNode;
     }

Added: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ReferenceCountedACLCacheTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ReferenceCountedACLCacheTest.java?rev=1736936&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ReferenceCountedACLCacheTest.java
(added)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ReferenceCountedACLCacheTest.java
Mon Mar 28 23:14:14 2016
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import org.apache.jute.BinaryInputArchive;
+import org.apache.jute.BinaryOutputArchive;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ReferenceCountedACLCacheTest {
+    @Test
+    public void testSameACLGivesSameID() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+
+        List<ACL> testACL2 = createACL("myid");
+
+        assertEquals(aclId, cache.convertAcls(testACL2));
+    }
+
+    @Test
+    public void testWhetherOrderingMatters() {
+        List<ACL> testACL = new ArrayList<ACL>();
+        testACL.add(new ACL(ZooDefs.Perms.READ, new Id("scheme", "ro")));
+        testACL.add(new ACL(ZooDefs.Perms.WRITE, new Id("scheme", "rw")));
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+
+        List<ACL> testACL2 = new ArrayList<ACL>();
+        testACL2.add(new ACL(ZooDefs.Perms.WRITE, new Id("scheme", "rw")));
+        testACL2.add(new ACL(ZooDefs.Perms.READ, new Id("scheme", "ro")));
+
+        assertFalse(aclId.equals(cache.convertAcls(testACL2)));
+    }
+
+    @Test
+    public void testBidirectionality() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+
+        assertEquals(testACL, cache.convertLong(aclId));
+    }
+
+    @Test
+    public void testCacheSize() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+        assertEquals(1, cache.size());
+
+        List<ACL> testACL2 = createACL("myid");
+
+        assertEquals(aclId, cache.convertAcls(testACL2));
+        assertEquals(1, cache.size());
+
+        List<ACL> testACL3 = createACL("differentId");
+
+        Long aclId3 = cache.convertAcls(testACL3);
+        assertFalse(aclId3.equals(aclId));
+        assertEquals(2, cache.size());
+    }
+
+    @Test
+    public void testAddThenRemove() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+        assertEquals(1, cache.size());
+
+        cache.removeUsage(aclId);
+        assertEquals(0, cache.size());
+    }
+
+    @Test
+    public void testMultipleAddsAndRemove() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+        assertEquals(1, cache.size());
+
+        cache.convertAcls(testACL);
+        assertEquals(1, cache.size());
+
+        List<ACL> testACL2 = createACL("anotherId");
+        cache.convertAcls(testACL2);
+
+        cache.removeUsage(aclId);
+        assertEquals(2, cache.size());
+        cache.removeUsage(aclId);
+        assertEquals(1, cache.size());
+
+        Long newId = cache.convertAcls(testACL);
+        assertFalse(aclId.equals(newId));
+    }
+
+    @Test
+    public void testAddUsage() {
+        List<ACL> testACL = createACL("myid");
+
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        Long aclId = cache.convertAcls(testACL);
+        assertEquals(1, cache.size());
+
+        cache.addUsage(aclId);
+        assertEquals(1, cache.size());
+
+        cache.removeUsage(aclId);
+        assertEquals(1, cache.size());
+        cache.removeUsage(aclId);
+        assertEquals(0, cache.size());
+    }
+
+    @Test
+    public void testAddNonExistentUsage() {
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+        cache.addUsage(1L);
+
+        assertEquals(0, cache.size());
+        /*
+        On startup, it's possible that we'll try calling addUsage of an ID not in the cache.
 This is safe to ignore
+        as it'll be added later when we traverse the tranlog.  See discussion here:
+        http://mail-archives.apache.org/mod_mbox/zookeeper-user/201507.mbox/%3CCAB5oV2_ujhvBA1sEkCG2WRakPjCy%2BNR10620WK2G1GGgmEO44g%40mail.gmail.com%3E
+
+        This test makes sure that we don't add the ID to the cache in this case as that would
result in dupes later
+        and consequently incorrect counts and entries that will never be cleaned out.
+         */
+    }
+
+    @Test
+    public void testSerializeDeserialize() throws IOException {
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+
+        List<ACL> acl1 = createACL("one");
+        List<ACL> acl2 = createACL("two");
+        List<ACL> acl3 = createACL("three");
+        List<ACL> acl4 = createACL("four");
+        List<ACL> acl5 = createACL("five");
+
+        Long aclId1 = convertACLsNTimes(cache, acl1, 1);
+        Long aclId2 = convertACLsNTimes(cache, acl2, 2);
+        Long aclId3 = convertACLsNTimes(cache, acl3, 3);
+        Long aclId4 = convertACLsNTimes(cache, acl4, 4);
+        Long aclId5 = convertACLsNTimes(cache, acl5, 5);
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        BinaryOutputArchive archive = BinaryOutputArchive.getArchive(baos);
+        cache.serialize(archive);
+
+        BinaryInputArchive inArchive = BinaryInputArchive.getArchive(new ByteArrayInputStream(baos.toByteArray()));
+        ReferenceCountedACLCache deserializedCache = new ReferenceCountedACLCache();
+        deserializedCache.deserialize(inArchive);
+        callAddUsageNTimes(deserializedCache, aclId1, 1);
+        callAddUsageNTimes(deserializedCache, aclId2, 2);
+        callAddUsageNTimes(deserializedCache, aclId3, 3);
+        callAddUsageNTimes(deserializedCache, aclId4, 4);
+        callAddUsageNTimes(deserializedCache, aclId5, 5);
+
+        assertEquals(cache, deserializedCache);
+    }
+
+    @Test
+    public void testPurgeUnused() throws IOException {
+        ReferenceCountedACLCache cache = new ReferenceCountedACLCache();
+
+        List<ACL> acl1 = createACL("one");
+        List<ACL> acl2 = createACL("two");
+        List<ACL> acl3 = createACL("three");
+        List<ACL> acl4 = createACL("four");
+        List<ACL> acl5 = createACL("five");
+
+        Long aclId1 = convertACLsNTimes(cache, acl1, 1);
+        Long aclId2 = convertACLsNTimes(cache, acl2, 2);
+        Long aclId3 = convertACLsNTimes(cache, acl3, 3);
+        Long aclId4 = convertACLsNTimes(cache, acl4, 4);
+        Long aclId5 = convertACLsNTimes(cache, acl5, 5);
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        BinaryOutputArchive archive = BinaryOutputArchive.getArchive(baos);
+        cache.serialize(archive);
+
+        BinaryInputArchive inArchive = BinaryInputArchive.getArchive(new ByteArrayInputStream(baos.toByteArray()));
+        ReferenceCountedACLCache deserializedCache = new ReferenceCountedACLCache();
+        deserializedCache.deserialize(inArchive);
+        callAddUsageNTimes(deserializedCache, aclId1, 1);
+        callAddUsageNTimes(deserializedCache, aclId2, 2);
+        deserializedCache.purgeUnused();
+
+        assertEquals(2, deserializedCache.size());
+        assertEquals(aclId1, deserializedCache.convertAcls(acl1));
+        assertEquals(aclId2, deserializedCache.convertAcls(acl2));
+        assertFalse(acl3.equals(deserializedCache.convertAcls(acl3)));
+        assertFalse(acl4.equals(deserializedCache.convertAcls(acl4)));
+        assertFalse(acl5.equals(deserializedCache.convertAcls(acl5)));
+    }
+
+    private void callAddUsageNTimes(ReferenceCountedACLCache deserializedCache, Long aclId,
int num) {
+        for (int i = 0; i < num; i++) {
+            deserializedCache.addUsage(aclId);
+        }
+    }
+
+    private Long convertACLsNTimes(ReferenceCountedACLCache cache, List<ACL> acl, int
num) {
+        if (num <= 0) {
+            return -1L;
+        }
+
+        for (int i = 0; i < num -1; i++) {
+            cache.convertAcls(acl);
+        }
+
+        return cache.convertAcls(acl);
+    }
+
+    private List<ACL> createACL(String id) {
+        List<ACL> acl1 = new ArrayList<ACL>();
+        acl1.add(new ACL(ZooDefs.Perms.ADMIN, new Id("scheme", id)));
+        return acl1;
+    }
+}
\ No newline at end of file




Mime
View raw message