From notifications-return-957-archive-asf-public=cust-asf.ponee.io@zookeeper.apache.org Sun Jul 28 19:16:07 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id C15A6180671 for ; Sun, 28 Jul 2019 21:16:06 +0200 (CEST) Received: (qmail 3983 invoked by uid 500); 28 Jul 2019 19:16:06 -0000 Mailing-List: contact notifications-help@zookeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zookeeper.apache.org Delivered-To: mailing list notifications@zookeeper.apache.org Received: (qmail 3972 invoked by uid 99); 28 Jul 2019 19:16:06 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 28 Jul 2019 19:16:06 +0000 From: GitBox To: notifications@zookeeper.apache.org Subject: [GitHub] [zookeeper] lvfangmin commented on a change in pull request #632: [ZOOKEEPER-3150] Add tree digest check and verify data integrity when loading from disk Message-ID: <156434136101.25906.6175753739117476145.gitbox@gitbox.apache.org> Date: Sun, 28 Jul 2019 19:16:01 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit lvfangmin commented on a change in pull request #632: [ZOOKEEPER-3150] Add tree digest check and verify data integrity when loading from disk URL: https://github.com/apache/zookeeper/pull/632#discussion_r308014137 ########## File path: zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java ########## @@ -1566,4 +1613,194 @@ private void updateWriteStat(String path, long bytes) { } ServerMetrics.getMetrics().WRITE_PER_NAMESPACE.add(namespace, path.length() + bytes); } + + /** + * Add the digest to the historical list, and update the latest zxid digest. + */ + private void logZxidDigest(long zxid, long digest) { + ZxidDigest zxidDigest = new ZxidDigest(zxid, DigestCalculator.DIGEST_VERSION, digest); + lastProcessedZxidDigest = zxidDigest; + if (zxidDigest.zxid % DIGEST_LOG_INTERVAL == 0) { + synchronized (digestLog) { + digestLog.add(zxidDigest); + if (digestLog.size() > DIGEST_LOG_LIMIT) { + digestLog.poll(); + } + } + } + } + + /** + * Serializing the digest to snapshot, this is done after the data tree + * is being serialized, so when we replay the txns and it hits this zxid + * we know we should be in a non-fuzzy state, and have the same digest. + * + * @param oa the output stream to write to + * @return true if the digest is serialized successfully + */ + public boolean serializeZxidDigest(OutputArchive oa) throws IOException { + if (!DigestCalculator.digestEnabled()) { + return false; + } + + ZxidDigest zxidDigest = lastProcessedZxidDigest; + if (zxidDigest == null) { + // write an empty digest + zxidDigest = new ZxidDigest(); + } + zxidDigest.serialize(oa); + return true; + } + + /** + * Deserializing the zxid digest from the input stream and update the + * digestFromLoadedSnapshot. + * + * @param ia the input stream to read from + * @return the true if it deserialized successfully + */ + public boolean deserializeZxidDigest(InputArchive ia) throws IOException { + if (!DigestCalculator.digestEnabled()) { + return false; + } + + try { + ZxidDigest zxidDigest = new ZxidDigest(); + zxidDigest.deserialize(ia); + if (zxidDigest.zxid > 0) { + digestFromLoadedSnapshot = zxidDigest; + } + return true; + } catch (EOFException e) { + LOG.warn("Got EOF exception while reading the digest, " + + "likely due to the reading an older snapshot."); + return false; + } + } + + /** + * Compares the actual tree's digest with that in the snapshot. + * Resets digestFromLoadedSnapshot after comparision. Review comment: Yes, we need a way to tell if we're still in fuzzy range or not, and only start compare and log the digest when we're not in fuzzy state.. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services