zookeeper-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [zookeeper] lvfangmin commented on a change in pull request #632: [ZOOKEEPER-3150] Add tree digest check and verify data integrity when loading from disk
Date Sun, 28 Jul 2019 19:16:01 GMT
lvfangmin commented on a change in pull request #632: [ZOOKEEPER-3150] Add tree digest check
and verify data integrity when loading from disk
URL: https://github.com/apache/zookeeper/pull/632#discussion_r308014137
 
 

 ##########
 File path: zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java
 ##########
 @@ -1566,4 +1613,194 @@ private void updateWriteStat(String path, long bytes) {
         }
         ServerMetrics.getMetrics().WRITE_PER_NAMESPACE.add(namespace, path.length() + bytes);
     }
+
+    /**
+     * Add the digest to the historical list, and update the latest zxid digest.
+     */
+    private void logZxidDigest(long zxid, long digest) {
+        ZxidDigest zxidDigest = new ZxidDigest(zxid, DigestCalculator.DIGEST_VERSION, digest);
+        lastProcessedZxidDigest = zxidDigest;
+        if (zxidDigest.zxid % DIGEST_LOG_INTERVAL == 0) {
+            synchronized (digestLog) {
+                digestLog.add(zxidDigest);
+                if (digestLog.size() > DIGEST_LOG_LIMIT) {
+                    digestLog.poll();
+                }
+            }
+        }
+    }
+
+    /**
+     * Serializing the digest to snapshot, this is done after the data tree 
+     * is being serialized, so when we replay the txns and it hits this zxid 
+     * we know we should be in a non-fuzzy state, and have the same digest. 
+     *
+     * @param oa the output stream to write to 
+     * @return true if the digest is serialized successfully
+     */
+    public boolean serializeZxidDigest(OutputArchive oa) throws IOException {
+        if (!DigestCalculator.digestEnabled()) {
+            return false;
+        }
+
+        ZxidDigest zxidDigest = lastProcessedZxidDigest;
+        if (zxidDigest == null) {
+            // write an empty digest
+            zxidDigest = new ZxidDigest();
+        }
+        zxidDigest.serialize(oa);
+        return true;
+    }
+
+    /**
+     * Deserializing the zxid digest from the input stream and update the 
+     * digestFromLoadedSnapshot.
+     *
+     * @param ia the input stream to read from
+     * @return the true if it deserialized successfully
+     */
+    public boolean deserializeZxidDigest(InputArchive ia) throws IOException {
+        if (!DigestCalculator.digestEnabled()) {
+            return false;
+        }
+
+        try  {
+            ZxidDigest zxidDigest = new ZxidDigest();
+            zxidDigest.deserialize(ia);
+            if (zxidDigest.zxid > 0) {
+                digestFromLoadedSnapshot = zxidDigest;
+            }
+            return true;
+        } catch (EOFException e) {
+            LOG.warn("Got EOF exception while reading the digest, " +
+                    "likely due to the reading an older snapshot.");
+            return false;
+        }
+    }
+
+    /**
+     * Compares the actual tree's digest with that in the snapshot. 
+     * Resets digestFromLoadedSnapshot after comparision.
 
 Review comment:
   Yes, we need a way to tell if we're still in fuzzy range or not, and only start compare
and log the digest when we're not in fuzzy state..

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message