Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 14F16200ACA for ; Thu, 19 May 2016 00:49:29 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 139A2160A15; Wed, 18 May 2016 22:49:29 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BE02C160A26 for ; Thu, 19 May 2016 00:49:27 +0200 (CEST) Received: (qmail 3264 invoked by uid 500); 18 May 2016 22:49:23 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 2300 invoked by uid 99); 18 May 2016 22:49:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 May 2016 22:49:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 427F0E0844; Wed, 18 May 2016 22:49:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: eclark@apache.org To: commits@hbase.apache.org Date: Wed, 18 May 2016 22:49:48 -0000 Message-Id: In-Reply-To: <5a847047da0a4c87828fc6fa66936f0d@git.apache.org> References: <5a847047da0a4c87828fc6fa66936f0d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [27/42] hbase git commit: HBASE-15696 Move region location cache serialization into serde archived-at: Wed, 18 May 2016 22:49:29 -0000 HBASE-15696 Move region location cache serialization into serde Summary: Use IOBuf for zk Test Plan: Added a unit test Differential Revision: https://reviews.facebook.net/D57147 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/65cd2da5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/65cd2da5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/65cd2da5 Branch: refs/heads/HBASE-14850 Commit: 65cd2da5f2a81f66635110deba17ccdfa774b3aa Parents: eabb568 Author: Elliott Clark Authored: Fri Apr 22 22:21:45 2016 -0700 Committer: Elliott Clark Committed: Wed May 18 15:48:52 2016 -0700 ---------------------------------------------------------------------- hbase-native-client/Dockerfile | 7 +- .../connection/client-dispatcher.h | 8 ++ .../connection/client-handler.cc | 7 +- hbase-native-client/connection/client-handler.h | 2 +- hbase-native-client/connection/pipeline.h | 2 +- hbase-native-client/core/client.cc | 4 +- hbase-native-client/core/client.h | 4 +- hbase-native-client/core/location-cache-test.cc | 2 +- hbase-native-client/core/location-cache.cc | 48 +++----- hbase-native-client/core/location-cache.h | 4 +- hbase-native-client/core/simple-client.cc | 6 +- hbase-native-client/serde/BUCK | 10 ++ .../serde/client-deserializer-test.cc | 4 +- .../serde/client-deserializer.cc | 4 +- .../serde/client-serializer-test.cc | 2 +- hbase-native-client/serde/client-serializer.cc | 2 +- hbase-native-client/serde/client-serializer.h | 2 +- .../serde/zk-deserializer-test.cc | 123 +++++++++++++++++++ hbase-native-client/serde/zk-deserializer.cc | 78 ++++++++++++ hbase-native-client/serde/zk-deserializer.h | 35 ++++++ hbase-native-client/utils/user-util-test.cc | 1 + 21 files changed, 297 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/Dockerfile ---------------------------------------------------------------------- diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile index 9ccc0d3..1524c56 100644 --- a/hbase-native-client/Dockerfile +++ b/hbase-native-client/Dockerfile @@ -19,12 +19,11 @@ FROM pjameson/buck-folly-watchman ARG CC=/usr/bin/gcc-5 ARG CXX=/usr/bin/g++-5 -ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread" -ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread" +ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread" +ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread" -RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \ +RUN apt-get install -y vim maven inetutils-ping python-pip && \ pip install yapf && \ - ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \ apt-get -qq clean && \ apt-get -y -qq autoremove && \ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \ http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-dispatcher.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h index c3987c9..89c7119 100644 --- a/hbase-native-client/connection/client-dispatcher.h +++ b/hbase-native-client/connection/client-dispatcher.h @@ -37,6 +37,14 @@ public: private: std::unordered_map> requests_; + // Start at some number way above what could + // be there for un-initialized call id counters. + // + // This makes it easier to make sure that the're are + // no access to un-initialized variables. + // + // uint32_t has a max of 4Billion so 10 more or less is + // not a big deal. uint32_t current_call_id_ = 10; }; } // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-handler.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc index 74b23ef..7b7794d 100644 --- a/hbase-native-client/connection/client-handler.cc +++ b/hbase-native-client/connection/client-handler.cc @@ -19,15 +19,15 @@ #include "connection/client-handler.h" -#include #include +#include #include #include "connection/request.h" #include "connection/response.h" -#include "if/RPC.pb.h" #include "if/Client.pb.h" +#include "if/RPC.pb.h" using namespace hbase; using namespace folly; @@ -60,6 +60,9 @@ void ClientHandler::read(Context *ctx, std::unique_ptr buf) { } } + +// TODO(eclark): Figure out how to handle the +// network errors that are going to come. Future ClientHandler::write(Context *ctx, Request r) { // Keep track of if we have sent the header. if (UNLIKELY(need_send_header_)) { http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-handler.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h index 38c5725..dbaf5a0 100644 --- a/hbase-native-client/connection/client-handler.h +++ b/hbase-native-client/connection/client-handler.h @@ -22,8 +22,8 @@ #include -#include "serde/client-serializer.h" #include "serde/client-deserializer.h" +#include "serde/client-serializer.h" // Forward decs. namespace hbase { http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/pipeline.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h index 68ade48..8114fab 100644 --- a/hbase-native-client/connection/pipeline.h +++ b/hbase-native-client/connection/pipeline.h @@ -18,8 +18,8 @@ */ #pragma once -#include #include +#include #include "connection/request.h" #include "connection/response.h" http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/client.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc index 893894f..266c239 100644 --- a/hbase-native-client/core/client.cc +++ b/hbase-native-client/core/client.cc @@ -21,8 +21,8 @@ #include #include -#include #include +#include #include #include @@ -36,5 +36,5 @@ using namespace hbase::pb; namespace hbase { Client::Client(string quorum_spec) - : location_cache(quorum_spec, wangle::getCPUExecutor()) {} + : location_cache_(quorum_spec, wangle::getCPUExecutor()) {} } http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/client.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h index c2dc226..b583285 100644 --- a/hbase-native-client/core/client.h +++ b/hbase-native-client/core/client.h @@ -19,8 +19,8 @@ #pragma once -#include #include +#include #include @@ -36,7 +36,7 @@ public: folly::Future get(const GetRequest &get_request); private: - LocationCache location_cache; + LocationCache location_cache_; }; } // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc index 70ca6f1..f3166fb 100644 --- a/hbase-native-client/core/location-cache-test.cc +++ b/hbase-native-client/core/location-cache-test.cc @@ -16,8 +16,8 @@ * limitations under the License. * */ -#include #include +#include #include #include "location-cache.h" http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc index 52e86e3..5925f4a 100644 --- a/hbase-native-client/core/location-cache.cc +++ b/hbase-native-client/core/location-cache.cc @@ -16,11 +16,13 @@ * limitations under the License. * */ -#include "location-cache.h" +#include "core/location-cache.h" #include +#include #include "if/ZooKeeper.pb.h" +#include "serde/zk-deserializer.h" using namespace std; using namespace folly; @@ -28,7 +30,7 @@ using namespace hbase::pb; using namespace hbase; // TODO(eclark): make this configurable on client creation -static const char META_LOCATION[] = "/hbase/meta-region-server"; +static const char META_ZNODE_NAME[] = "/hbase/meta-region-server"; LocationCache::LocationCache(string quorum_spec, shared_ptr executor) @@ -66,45 +68,25 @@ void LocationCache::RefreshMetaLocation() { } ServerName LocationCache::ReadMetaLocation() { - char contents[4096]; + auto buf = IOBuf::create(4096); + ZkDeserializer derser; + // This needs to be int rather than size_t as that's what ZK expects. - int len = sizeof(contents); + int len = buf->capacity(); // TODO(elliott): handle disconnects/reconntion as needed. - int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr); + int zk_result = + zoo_get(this->zk_, META_ZNODE_NAME, 0, + reinterpret_cast(buf->writableData()), &len, nullptr); + LOG(ERROR) << "len = " << len; if (zk_result != ZOK || len < 9) { LOG(ERROR) << "Error getting meta location."; throw runtime_error("Error getting meta location"); } - // There should be a magic number for recoverable zk - if (static_cast(contents[0]) != 255) { - LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" - << unsigned(static_cast(contents[0])); - throw runtime_error("Magic number not in znode data"); - } - // pos will keep track of skipped bytes. - int pos = 1; - // How long is the id? - int id_len = 0; - for (int i = 0; i < 4; ++i) { - id_len = id_len << 8; - id_len = id_len | static_cast(contents[pos]); - ++pos; - } - // Skip the id - pos += id_len; - // Then all protobuf's for HBase are prefixed with a magic string. - // PBUF, so we skip that. - // TODO(eclark): check to make sure that the magic string is correct - // though I am not sure that will get us much. - pos += 4; + buf->append(len); MetaRegionServer mrs; - // Try to decode the protobuf. - // If there's an error bail out. - if (mrs.ParseFromArray(contents + pos, len - pos) == false) { - LOG(ERROR) << "Error parsing Protobuf Message"; - throw runtime_error("Error parsing protobuf"); + if (derser.parse(buf.get(), &mrs) == false) { + LOG(ERROR) << "Unable to decode"; } - return mrs.server(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h index 28a1ee1..ce97c43 100644 --- a/hbase-native-client/core/location-cache.h +++ b/hbase-native-client/core/location-cache.h @@ -18,10 +18,10 @@ */ #pragma once -#include +#include #include #include -#include +#include #include #include http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/simple-client.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc index 065f70b..5e6f82e 100644 --- a/hbase-native-client/core/simple-client.cc +++ b/hbase-native-client/core/simple-client.cc @@ -22,13 +22,13 @@ #include #include -#include #include +#include -#include "core/client.h" #include "connection/connection-factory.h" -#include "if/ZooKeeper.pb.h" +#include "core/client.h" #include "if/Client.pb.h" +#include "if/ZooKeeper.pb.h" using namespace folly; using namespace std; http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/BUCK ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK index 207607f..0014c0b 100644 --- a/hbase-native-client/serde/BUCK +++ b/hbase-native-client/serde/BUCK @@ -19,10 +19,12 @@ cxx_library(name="serde", exported_headers=[ "client-serializer.h", "client-deserializer.h", + "zk-deserializer.h", ], srcs=[ "client-serializer.cc", "client-deserializer.cc", + "zk-deserializer.cc", ], deps=[ "//if:if", @@ -52,3 +54,11 @@ cxx_test(name="client-deserializer-test", ":serde", "//if:if", ], ) +cxx_test(name="zk-deserializer-test", + srcs=[ + "zk-deserializer-test.cc", + ], + deps=[ + ":serde", + "//if:if", + ], ) http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-deserializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc index bb57e50..9fef093 100644 --- a/hbase-native-client/serde/client-deserializer-test.cc +++ b/hbase-native-client/serde/client-deserializer-test.cc @@ -17,12 +17,12 @@ * */ -#include #include +#include +#include "if/Client.pb.h" #include "serde/client-deserializer.h" #include "serde/client-serializer.h" -#include "if/Client.pb.h" using namespace hbase; using folly::IOBuf; http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-deserializer.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc index 118b0d1..acca7ea 100644 --- a/hbase-native-client/serde/client-deserializer.cc +++ b/hbase-native-client/serde/client-deserializer.cc @@ -19,10 +19,10 @@ #include "serde/client-deserializer.h" -#include +#include #include #include -#include +#include using namespace hbase; http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc index b32b55d..9bf38af 100644 --- a/hbase-native-client/serde/client-serializer-test.cc +++ b/hbase-native-client/serde/client-serializer-test.cc @@ -22,9 +22,9 @@ #include -#include "serde/client-serializer.h" #include "if/HBase.pb.h" #include "if/RPC.pb.h" +#include "serde/client-serializer.h" using namespace hbase; using namespace hbase::pb; http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc index 881b6e4..09b81c8 100644 --- a/hbase-native-client/serde/client-serializer.cc +++ b/hbase-native-client/serde/client-serializer.cc @@ -18,8 +18,8 @@ */ #include "serde/client-serializer.h" -#include #include +#include #include #include http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h index 685095d..9c819fe 100644 --- a/hbase-native-client/serde/client-serializer.h +++ b/hbase-native-client/serde/client-serializer.h @@ -18,9 +18,9 @@ */ #pragma once +#include #include #include -#include // Forward namespace google { http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/zk-deserializer-test.cc b/hbase-native-client/serde/zk-deserializer-test.cc new file mode 100644 index 0000000..92d85a0 --- /dev/null +++ b/hbase-native-client/serde/zk-deserializer-test.cc @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "serde/zk-deserializer.h" + +#include +#include +#include +#include + +#include "if/ZooKeeper.pb.h" + +using namespace hbase; +using namespace hbase::pb; +using namespace folly; +using namespace std; +using namespace folly::io; + +// Test that would test if there's nothing there. +TEST(TestZkDesializer, TestThrowNoMagicNum) { + ZkDeserializer deser; + MetaRegionServer mrs; + + auto buf = IOBuf::create(100); + buf->append(100); + RWPrivateCursor c{buf.get()}; + c.write(99); + ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error); +} + +// Test if the protobuf is in a format that we can't decode +TEST(TestZkDesializer, TestBadProtoThrow) { + ZkDeserializer deser; + MetaRegionServer mrs; + string magic{"PBUF"}; + + // Set ServerName + mrs.mutable_server()->set_host_name("test"); + mrs.mutable_server()->set_port(567); + mrs.mutable_server()->set_start_code(9567); + + // One byte magic number + // four bytes for id length + // four bytes for id + // four bytes for PBUF + uint32_t start_len = 1 + 4 + 4 + 4; + // How large the protobuf will be + uint32_t pbuf_size = mrs.ByteSize(); + + auto buf = IOBuf::create(start_len + pbuf_size); + buf->append(start_len + pbuf_size); + RWPrivateCursor c{buf.get()}; + + // Write the magic number + c.write(255); + // Write the id len + c.writeBE(4); + // Write the id + c.write(13); + // Write the PBUF string + c.push(reinterpret_cast(magic.c_str()), 4); + + // Create the protobuf + MetaRegionServer out; + ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error); +} + +// Test to make sure the whole thing works. +TEST(TestZkDesializer, TestNoThrow) { + ZkDeserializer deser; + MetaRegionServer mrs; + string magic{"PBUF"}; + + // Set ServerName + mrs.mutable_server()->set_host_name("test"); + mrs.mutable_server()->set_port(567); + mrs.mutable_server()->set_start_code(9567); + + // One byte magic number + // four bytes for id length + // four bytes for id + // four bytes for PBUF + uint32_t start_len = 1 + 4 + 4 + 4; + // How large the protobuf will be + uint32_t pbuf_size = mrs.ByteSize(); + + auto buf = IOBuf::create(start_len + pbuf_size); + buf->append(start_len + pbuf_size); + RWPrivateCursor c{buf.get()}; + + // Write the magic number + c.write(255); + // Write the id len + c.writeBE(4); + // Write the id + c.write(13); + // Write the PBUF string + c.push(reinterpret_cast(magic.c_str()), 4); + + // Now write the serialized protobuf + mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len); + + // Create the protobuf + MetaRegionServer out; + ASSERT_TRUE(deser.parse(buf.get(), &out)); + ASSERT_EQ(mrs.server().host_name(), out.server().host_name()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/zk-deserializer.cc b/hbase-native-client/serde/zk-deserializer.cc new file mode 100644 index 0000000..33cf809 --- /dev/null +++ b/hbase-native-client/serde/zk-deserializer.cc @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "serde/zk-deserializer.h" + +#include +#include +#include + +using hbase::ZkDeserializer; +using std::runtime_error; +using folly::IOBuf; +using folly::io::Cursor; +using google::protobuf::Message; + +static const std::string MAGIC_STRING = "PBUF"; + +bool ZkDeserializer::parse(IOBuf *buf, Message *out) { + + // The format is like this + // 1 byte of magic number. 255 + // 4 bytes of id length. + // id_length number of bytes for the id of who put up the znode + // 4 bytes of a magic string PBUF + // Then the protobuf serialized without a varint header. + + Cursor c{buf}; + + // There should be a magic number for recoverable zk + uint8_t magic_num = c.read(); + if (magic_num != 255) { + LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" + << unsigned(magic_num); + throw runtime_error("Magic number not in znode data"); + } + // How long is the id? + uint32_t id_len = c.readBE(); + + if (id_len >= c.length()) { + LOG(ERROR) << "After skiping the if from zookeeper data there's not enough " + "left to read anything else"; + throw runtime_error("Not enough bytes to decode from zookeeper"); + } + + // Skip the id + c.skip(id_len); + + // Make sure that the magic string is there. + if (MAGIC_STRING != c.readFixedString(4)) { + LOG(ERROR) << "There was no PBUF magic string."; + throw runtime_error("No PBUF magic string in the zookpeeper data."); + } + + // Try to decode the protobuf. + // If there's an error bail out. + if (out->ParseFromArray(c.data(), c.length()) == false) { + LOG(ERROR) << "Error parsing Protobuf Message"; + throw runtime_error("Error parsing protobuf"); + } + + return true; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/serde/zk-deserializer.h b/hbase-native-client/serde/zk-deserializer.h new file mode 100644 index 0000000..aa91661 --- /dev/null +++ b/hbase-native-client/serde/zk-deserializer.h @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +namespace google { +namespace protobuf { +class Message; +} +} +namespace folly { +class IOBuf; +} + +namespace hbase { +class ZkDeserializer { +public: + bool parse(folly::IOBuf *buf, google::protobuf::Message *out); +}; +} // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/utils/user-util-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc index 2a7434f..1e72bcc 100644 --- a/hbase-native-client/utils/user-util-test.cc +++ b/hbase-native-client/utils/user-util-test.cc @@ -30,5 +30,6 @@ TEST(TestUserUtil, TestGetSomething) { UserUtil u_util; string name = u_util.user_name(); + // TODO shell out to whoami to check this. ASSERT_GT(name.length(), 0); }