From commits-return-6731-apmail-zookeeper-commits-archive=zookeeper.apache.org@zookeeper.apache.org Mon Aug 6 12:13:27 2018 Return-Path: X-Original-To: apmail-zookeeper-commits-archive@www.apache.org Delivered-To: apmail-zookeeper-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A3E7D18BDF for ; Mon, 6 Aug 2018 12:13:27 +0000 (UTC) Received: (qmail 44092 invoked by uid 500); 6 Aug 2018 12:13:27 -0000 Delivered-To: apmail-zookeeper-commits-archive@zookeeper.apache.org Received: (qmail 43884 invoked by uid 500); 6 Aug 2018 12:13:27 -0000 Mailing-List: contact commits-help@zookeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zookeeper.apache.org Delivered-To: mailing list commits@zookeeper.apache.org Received: (qmail 43544 invoked by uid 99); 6 Aug 2018 12:13:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Aug 2018 12:13:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C4EB1E0AC6; Mon, 6 Aug 2018 12:13:26 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: andor@apache.org To: commits@zookeeper.apache.org Date: Mon, 06 Aug 2018 12:13:31 -0000 Message-Id: In-Reply-To: <13bf07ba1f73429dae991fd13f50d7a5@git.apache.org> References: <13bf07ba1f73429dae991fd13f50d7a5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [06/45] zookeeper git commit: ZOOKEEPER-3030: MAVEN MIGRATION - Step 1.3 - move contrib directories http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/create_test.py ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/create_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/create_test.py new file mode 100755 index 0000000..8ab80f9 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/create_test.py @@ -0,0 +1,104 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import zookeeper, zktestbase, unittest, threading + +ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} + +class CreationTest(zktestbase.TestBase): + """Test whether we can create znodes""" + # to do: startup and teardown via scripts? + def setUp(self): + zktestbase.TestBase.setUp(self) + try: + zookeeper.delete(self.handle, "/zk-python-createtest") + zookeeper.delete(self.handle, "/zk-python-acreatetest") + except: + pass + + def test_sync_create(self): + self.assertEqual(self.connected, True) + ret = zookeeper.create(self.handle, "/zk-python-createtest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + self.assertEqual(ret, "/zk-python-createtest") + self.assertRaises(zookeeper.NoChildrenForEphemeralsException, + zookeeper.create, + self.handle, + "/zk-python-createtest/invalid-child", + "", + [ZOO_OPEN_ACL_UNSAFE], + zookeeper.EPHEMERAL) + + def test_sync_create_existing(self): + self.assertEqual(self.connected, True) + ret = zookeeper.create(self.handle, "/zk-python-createtest-existing", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + self.assertEqual(ret, "/zk-python-createtest-existing") + + self.assertRaises(zookeeper.NodeExistsException, + zookeeper.create, + self.handle, + "/zk-python-createtest-existing", + "nodecontents", + [ZOO_OPEN_ACL_UNSAFE], + zookeeper.EPHEMERAL) + + + def test_exception_paths(self): + """ + Make sure common exceptions due to API misuse are correctly propogated + """ + self.assertRaises(zookeeper.BadArgumentsException, + zookeeper.create, + self.handle, + "/zk-python-badargs-test", + "", + [ZOO_OPEN_ACL_UNSAFE], + -1) + self.assertRaises(zookeeper.InvalidACLException, + zookeeper.create, + self.handle, + "/zk-python-invalidacl-test", + "", + ZOO_OPEN_ACL_UNSAFE) # Error - not a list + + + def test_async_create(self): + self.cv = threading.Condition() + def callback(handle, rc, value): + self.cv.acquire() + self.callback_flag = True + self.rc = rc + self.cv.notify() + self.cv.release() + + self.assertEqual(self.connected, True, "Not connected!") + self.cv.acquire() + + ret = zookeeper.acreate(self.handle, "/zk-python-acreatetest", "nodecontents", + [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL, + callback ) + self.assertEqual(ret, zookeeper.OK, "acreate failed") + while not self.callback_flag: + self.cv.wait(15) + self.cv.release() + + self.assertEqual(self.callback_flag, True, "acreate timed out") + self.assertEqual(self.rc, zookeeper.OK) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/delete_test.py ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/delete_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/delete_test.py new file mode 100755 index 0000000..913b6a9 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/delete_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import zookeeper, zktestbase, unittest, threading + +class DeletionTest(zktestbase.TestBase): + """Test whether we can delete znodes""" + + def test_sync_delete(self): + ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} + self.assertEqual(self.connected, True) + ret = zookeeper.create(self.handle, "/zk-python-deletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + self.assertEqual(ret, "/zk-python-deletetest") + ret = zookeeper.delete(self.handle,"/zk-python-deletetest") + self.assertEqual(ret, zookeeper.OK) + children = zookeeper.get_children(self.handle, "/") + self.assertEqual(False, "zk-python-deletetest" in children) + + # test exception + self.assertRaises(zookeeper.NoNodeException, + zookeeper.delete, + self.handle, + "/zk-python-deletetest") + + def test_async_delete(self): + ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} + self.assertEqual(self.connected, True) + ret = zookeeper.create(self.handle, "/zk-python-adeletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + self.assertEqual(ret, "/zk-python-adeletetest") + + self.cv = threading.Condition() + self.callback_flag = False + self.rc = -1 + def callback(handle, rc): + self.cv.acquire() + self.callback_flag = True + self.cv.notify() + self.rc = rc # don't assert this here, as if the assertion fails, the test will block + self.cv.release() + + self.cv.acquire() + ret = zookeeper.adelete(self.handle,"/zk-python-adeletetest",-1,callback) + self.assertEqual(ret, zookeeper.OK, "adelete failed") + while not self.callback_flag: + self.cv.wait(15) + self.cv.release() + + self.assertEqual(self.callback_flag, True, "adelete timed out") + self.assertEqual(self.rc, zookeeper.OK) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/exists_test.py ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/exists_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/exists_test.py new file mode 100755 index 0000000..ddc6ef3 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/exists_test.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import zookeeper, zktestbase, unittest, threading + +ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} +class ExistsTest(zktestbase.TestBase): + def setUp( self ): + zktestbase.TestBase.setUp(self) + try: + zookeeper.create(self.handle, "/zk-python-existstest","existstest", [ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL) + zookeeper.create(self.handle, "/zk-python-aexiststest","existstest",[ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL) + except: + pass + + def test_sync_exists(self): + self.assertEqual(self.connected, True) + ret = zookeeper.exists(self.handle, "/zk-python-existstest", None) + self.assertNotEqual(ret, None, "/zk-python-existstest does not exist (possibly means creation failure)") + + def test_sync_nexists(self): + self.assertEqual(None, zookeeper.exists(self.handle, "/i-dont-exist", None)) + + + def test_async_exists(self): + self.cv = threading.Condition() + def callback(handle, rc, stat): + self.cv.acquire() + self.callback_flag = True + self.cv.notify() + self.cv.release() + self.rc = rc + + self.assertEqual(self.connected, True) + + self.cv.acquire() + ret = zookeeper.aexists(self.handle, "/zk-python-aexiststest", None, + callback ) + self.assertEqual(ret, zookeeper.OK) + while not self.callback_flag: + self.cv.wait(15) + self.cv.release() + + self.assertEqual(self.callback_flag, True, "aexists timed out") + self.assertEqual(self.rc, zookeeper.OK, "Return code not ok:" + zookeeper.zerror(self.rc)) + + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/get_set_test.py ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/get_set_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/get_set_test.py new file mode 100755 index 0000000..b77b3b2 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/get_set_test.py @@ -0,0 +1,211 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import zookeeper, zktestbase, unittest, threading, sys +if sys.version_info < (3,): + range = xrange + +ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} + +class GetSetTest(zktestbase.TestBase): + def setUp( self ): + zktestbase.TestBase.setUp(self) + try: + zookeeper.create(self.handle, "/zk-python-getsettest", "on",[ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + zookeeper.create(self.handle, "/zk-python-agetsettest", + "on",[ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) + except: + pass + + def test_empty_node(self): + """ + Test for a bug when instead of empty string we can get + random data from buffer malloc'ed to hold node contents. + See ZOOKEEPER-1906 for details + """ + NODE_PATH = "/zk-python-test-empty-node" + self.ensureDeleted(NODE_PATH) + zookeeper.create(self.handle, NODE_PATH, "", + [{"perms":0x1f, "scheme":"world", "id" :"anyone"}]) + (data,stat) = zookeeper.get(self.handle, NODE_PATH, None) + self.assertEqual(data, "", "Data is not empty as expected: " + data) + + def test_sync_getset(self): + self.assertEqual(self.connected, True, "Not connected!") + (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None) + self.assertEqual(data, "on", "Data is not 'on' as expected: " + data) + ret = zookeeper.set(self.handle, "/zk-python-getsettest", + "off", stat["version"]) + (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None) + self.assertEqual(data, "off", "Data is not 'off' as expected: " + data) + self.assertRaises(zookeeper.BadVersionException, + zookeeper.set, + self.handle, + "/zk-python-getsettest", + "test", + stat["version"]+1) + stat2 = zookeeper.set2(self.handle, "/zk-python-getsettest", + "set2", stat["version"]) + self.assertNotEqual(stat2, None, "set2 call failed, return should not be None") + self.assertEqual(stat2["numChildren"], 0, + "set2 call failed, numChildren not 0 in set2 call") + (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None) + self.assertEqual(data, "set2", "Data is not 'set2' as expected: " + data) + + def test_stat_deleted_node(self): + """ + Test for a bug that surfaced when trying to build a + stat object from a non-existant node. + + """ + self.ensureDeleted("/zk-python-test-deleteme") + self.assertRaises(zookeeper.NoNodeException, + zookeeper.get, + self.handle, + "/zk-python-test-deleteme") + self.cv = threading.Condition() + def callback(handle, rc, value, stat): + self.cv.acquire() + self.stat = stat + self.rc = rc + self.value = value + self.callback_flag = True + self.cv.notify() + self.cv.release() + self.cv.acquire() + zookeeper.aget(self.handle, "/zk-python-test-deleteme", None, callback) + self.cv.wait(15) + self.assertEqual(self.callback_flag, True, "aget timed out!") + self.assertEqual(self.stat, None, "Stat should be none!") + self.assertEqual(self.value, None, "Value should be none!") + + def test_sync_get_large_datanode(self): + """ + Test that we can retrieve datanode sizes up to + 1Mb with default parameters (depends on ZooKeeper server). + """ + + data = ''.join(["A" for x in range(1024*1023)]) + self.ensureDeleted("/zk-python-test-large-datanode") + zookeeper.create(self.handle, "/zk-python-test-large-datanode", data, + [{"perms":0x1f, "scheme":"world", "id" :"anyone"}]) + (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode") + self.assertEqual(len(ret), 1024*1023, + "Should have got 1Mb returned, instead got %s" % len(ret)) + (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode",None,500) + self.assertEqual(len(ret), 500, + "Should have got 500 bytes returned, instead got %s" % len(ret)) + + + + def test_async_getset(self): + self.cv = threading.Condition() + def get_callback(handle, rc, value, stat): + self.cv.acquire() + self.callback_flag = True + self.rc = rc + self.value = (value,stat) + self.cv.notify() + self.cv.release() + + def set_callback(handle, rc, stat): + self.cv.acquire() + self.callback_flag = True + self.rc = rc + self.value = stat + self.cv.notify() + self.cv.release() + + self.assertEqual(self.connected, True, "Not connected!") + + self.cv.acquire() + self.callback_flag = False + ret = zookeeper.aset(self.handle, "/zk-python-agetsettest", "off", -1, set_callback) + self.assertEqual(ret, zookeeper.OK, "aset failed") + while not self.callback_flag: + self.cv.wait(15) + self.cv.release() + self.assertEqual(self.callback_flag, True, "aset timed out") + + self.cv.acquire() + self.callback_flag = False + ret = zookeeper.aget(self.handle, "/zk-python-agetsettest", None, get_callback) + self.assertEqual(ret, zookeeper.OK, "aget failed") + self.cv.wait(15) + self.cv.release() + self.assertEqual(self.callback_flag, True, "aget timed out") + self.assertEqual(self.value[0], "off", "Data is not 'off' as expected: " + self.value[0]) + + def test_sync_getchildren(self): + self.ensureCreated("/zk-python-getchildrentest", flags=0) + self.ensureCreated("/zk-python-getchildrentest/child") + children = zookeeper.get_children(self.handle, "/zk-python-getchildrentest") + self.assertEqual(len(children), 1, "Expected to find 1 child, got " + str(len(children))) + + def test_async_getchildren(self): + self.ensureCreated("/zk-python-getchildrentest", flags=0) + self.ensureCreated("/zk-python-getchildrentest/child") + + def gc_callback(handle, rc, children): + self.cv.acquire() + self.rc = rc + self.children = children + self.callback_flag = True + self.cv.notify() + self.cv.release() + + self.cv.acquire() + self.callback_flag = False + zookeeper.aget_children(self.handle, "/zk-python-getchildrentest", None, gc_callback) + self.cv.wait(15) + self.assertEqual(self.callback_flag, True, "aget_children timed out") + self.assertEqual(self.rc, zookeeper.OK, "Return code for aget_children was not OK - %s" % zookeeper.zerror(self.rc)) + self.assertEqual(len(self.children), 1, "Expected to find 1 child, got " + str(len(self.children))) + + + def test_async_getchildren_with_watcher(self): + self.ensureCreated("/zk-python-getchildrentest", flags=0) + self.ensureCreated("/zk-python-getchildrentest/child") + + watched = [] + + def watcher(*args): + self.cv.acquire() + watched.append(args) + self.cv.notify() + self.cv.release() + + def children_callback(*args): + self.cv.acquire() + self.cv.notify() + self.cv.release() + + zookeeper.aget_children( + self.handle, "/zk-python-getchildrentest", watcher, children_callback) + + self.cv.acquire() + self.cv.wait() + self.cv.release() + + self.cv.acquire() + self.ensureCreated("/zk-python-getchildrentest/child2") + self.cv.wait(15) + self.assertTrue(watched) + +if __name__ == '__main__': + unittest.main() http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh new file mode 100755 index 0000000..18d9240 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: run_tests.sh testdir [logdir] +# logdir is optional, defaults to cwd + +# get the number of command-line arguments given +ARGC=$# + +# check to make sure enough arguments were given or exit +if [ $ARGC -lt 2 ]; then + export ZKPY_LOG_DIR="." +else + export ZKPY_LOG_DIR=$2 +fi + +# Find the build directory containing zookeeper.so +SO_PATH=`find ../../../build/ -name "zookeeper.so" | head -1` +PYTHONPATH=`dirname $SO_PATH` +LIB_PATH=../../c/.libs/:../../../build/test/test-cppunit/.libs +for test in `ls $1/*_test.py`; +do + echo "Running $test" + LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test +done http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh new file mode 100755 index 0000000..51d508f --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$1" == "x" ] +then + echo "USAGE: $0 startClean|start|stop hostPorts" + exit 2 +fi + +if [ "x$1" == "xstartClean" ] +then + if [ "x${base_dir}" == "x" ] + then + rm -rf /tmp/zkdata + else + rm -rf ${base_dir}/build/tmp + fi +fi + +if [ "x${base_dir}" == "x" ] +then +zk_base="../../../" +else +zk_base="${base_dir}" +fi + +CLASSPATH="$CLASSPATH:${zk_base}/build/classes" +CLASSPATH="$CLASSPATH:${zk_base}/conf" + +for i in "${zk_base}"/build/lib/*.jar +do + CLASSPATH="$CLASSPATH:$i" +done + +for i in "${zk_base}"/src/java/lib/*.jar +do + CLASSPATH="$CLASSPATH:$i" +done + +# Make sure nothing is left over from before +#fuser -skn tcp 22182/tcp + +case $1 in +start|startClean) + if [ "x${base_dir}" == "x" ] + then + mkdir -p /tmp/zkdata + java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 /tmp/zkdata &> /tmp/zk.log & + else + mkdir -p ${base_dir}/build/tmp/zkdata + java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 ${base_dir}/build/tmp/zkdata &> ${base_dir}/build/tmp/zk.log & + fi + sleep 5 + ;; +stop) + # Already killed above + ;; +*) + echo "Unknown command " + $1 + exit 2 +esac + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py new file mode 100755 index 0000000..8229418 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest, threading, zookeeper +ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} + +class TestBase(unittest.TestCase): + SERVER_PORT = 22182 + + def __init__(self,methodName='runTest'): + unittest.TestCase.__init__(self,methodName) + self.host = "localhost:%d" % self.SERVER_PORT + self.connected = False + self.handle = -1 + logdir = os.environ.get("ZKPY_LOG_DIR") + logfile = os.path.join(logdir, self.__class__.__name__ + ".log") + try: + f = open(logfile,"w") + zookeeper.set_log_stream(f) + except IOError: + print("Couldn't open " + logfile + " for writing") + + + def setUp(self): + self.callback_flag = False + self.cv = threading.Condition() + self.connected = False + def connection_watcher(handle, type, state, path): + self.cv.acquire() + self.connected = True + self.cv.notify() + self.cv.release() + + self.cv.acquire() + self.handle = zookeeper.init(self.host, connection_watcher) + self.cv.wait(15.0) + self.cv.release() + + if not self.connected: + raise Exception("Couldn't connect to host -", self.host) + + def newConnection(self): + cv = threading.Condition() + self.pending_connection = False + def connection_watcher(handle, type, state, path): + cv.acquire() + self.pending_connection = True + cv.notify() + cv.release() + + cv.acquire() + handle = zookeeper.init(self.host, connection_watcher) + cv.wait(15.0) + cv.release() + + if not self.pending_connection: + raise Exception("Couldn't connect to host -", self.host) + return handle + + def ensureDeleted(self,path): + self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle), "Not connected!") + try: + self.assertEqual(zookeeper.OK, zookeeper.delete(self.handle, path)) + except zookeeper.NoNodeException: + pass + + def ensureCreated(self,path,data="",flags=zookeeper.EPHEMERAL): + """ + It's possible not to get the flags you want here if the node already exists + """ + self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle), "Not connected!") + try: + self.assertEqual(path, zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], flags)) + except zookeeper.NodeExistsException: + pass + + def tearDown(self): + if self.connected: + zookeeper.close(self.handle) + + def all(self, iterable): + for element in iterable: + if not element: + return False + return True http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/Makefile.am ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/Makefile.am b/zookeeper-contrib/zookeeper-contrib-zktreeutil/Makefile.am new file mode 100644 index 0000000..36da1a5 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/Makefile.am @@ -0,0 +1,4 @@ +## Process this file with automake to produce Makefile.in + +SUBDIRS = src + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/README.txt ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/README.txt b/zookeeper-contrib/zookeeper-contrib-zktreeutil/README.txt new file mode 100644 index 0000000..43b06fa --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/README.txt @@ -0,0 +1,74 @@ +========================================== +zktreeutil - Zookeeper Tree Data Utility +Author: Anirban Roy +Organization: Yahoo Inc. +========================================== + +zktreeutil program is intended to manage and manipulate zk-tree data quickly, effi- +ciently and with ease. The utility operates on free-form ZK-tree and hence can be used +for any cluster managed by Zookeeper. Here are the basic functionalities - + +EXPORT: The whole/partial ZK-tree is exported into a XML file. This helps in +capturing a current snapshot of the data for backup/analysis. For a subtree +export, one need to specify the path to the ZK-subtree with proper option. + +IMPORT: The ZK-tree can be imported from XML into ZK cluster. This helps in priming +the new ZK cluster with static configuration. The import can be non-intrusive by +making only the additions in the existing data. The import of subtree is also +possible by optionally providing the path to the ZK-subtree. + +DIFF: Creates a diff between live ZK data vs data saved in XML file. Diff can ignore +some ZK-tree branches (possibly dynamic data) on reading the optional ignore flag +from XML file. Diffing on a ZK-subtree achieved by providing path to ZK-subtree with +diff command. + +UPDATE: Make the incremental changes into the live ZK-tree from saved XML, essentia- +lly after running the diff. + +DUMP: Dumps the ZK-tree on the standard output device reading either from live ZK +server or XML file. Like export, ZK-subtree can be dumped with optionaly +providing the path to the ZK-subtree, and till a certain depth of the (sub)tree. + +The exported ZK data into XML file can be shortened by only keeping the static ZK +nodes which are required to prime a cluster. The dynamic zk nodes (created on-the- +fly) can be ignored by setting a 'ignore' attribute at the root node of the dynamic +subtree (see tests/zk_sample.xml), possibly deleting all inner ZK nodes under that. +Once ignored, the whole subtree is ignored during DIFF, UPDATE and WRITE. + +Pre-requisites +-------------- +1. Linux system with 2.6.X kernel. +2. Zookeeper C client library (locally built at ../../c/.libs) >= 3.X.X +3. Development build libraries (rpm packages): + a. boost-devel >= 1.32.0 + b. libxml2-devel >= 2.7.3 + c. log4cxx0100-devel >= 0.10.0 + +Build instructions +------------------ +1. cd into this directory +2. autoreconf -if +3. ./configure +4. make +5. 'zktreeutil' binary created under src directory + +Limitations +----------- +Current version works with text data only, binary data will be supported in future +versions. + +Testing and usage of zktreeutil +-------------------------------- +1. Run Zookeeper server locally on port 2181 +2. export LD_LIBRARY_PATH=../../c/.libs/:/usr/local/lib/ +3. ./src/zktreeutil --help # show help +4. ./src/zktreeutil --zookeeper=localhost:2181 --import --xmlfile=tests/zk_sample.xml 2>/dev/null # import sample ZK tree +5. ./src/zktreeutil --zookeeper=localhost:2181 --dump --path=/myapp/version-1.0 2>/dev/null # dump Zk subtree +5. ./src/zktreeutil --zookeeper=localhost:2181 --dump --depth=3 2>/dev/null # dump Zk tree till certain depth +6. ./src/zktreeutil --xmlfile=zk_sample.xml -D 2>/dev/null # dump the xml data +7. Change zk_sample.xml with adding/deleting/chaging some nodes +8. ./src/zktreeutil -z localhost:2181 -F -x zk_sample.xml -p /myapp/version-1.0/configuration 2>/dev/null # take a diff of changes +9. ./src/zktreeutil -z localhost:2181 -E 2>/dev/null > zk_sample2.xml # export the mofied ZK tree +10. ./src/zktreeutil -z localhost:2181 -U -x zk_sample.xml -p /myapp/version-1.0/distributions 2>/dev/null # update with incr. changes +11. ./src/zktreeutil --zookeeper=localhost:2181 --import --force --xmlfile=zk_sample2.xml 2>/dev/null # re-prime the ZK tree + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/build.xml ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/build.xml b/zookeeper-contrib/zookeeper-contrib-zktreeutil/build.xml new file mode 100644 index 0000000..809d134 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/build.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/configure.ac ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/configure.ac b/zookeeper-contrib/zookeeper-contrib-zktreeutil/configure.ac new file mode 100644 index 0000000..b4a82a7 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/configure.ac @@ -0,0 +1,66 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.59) + +AC_INIT([zktreeutil], [1.0.0]) +AM_INIT_AUTOMAKE(foreign) + +AC_CONFIG_SRCDIR([src]) +AM_CONFIG_HEADER([config.h]) + +PACKAGE=zktreeutil +VERSION=1.0.0 + +AC_SUBST(PACKAGE) +AC_SUBST(VERSION) +BUILD_PATH="`pwd`" + +# Checks for programs. +AC_LANG_CPLUSPLUS +AC_PROG_CXX + +# Checks for libxm2. +AM_PATH_XML2(2.7.3) +XML2_INCLUDE="/usr/include/libxml2" +AC_SUBST(XML2_INCLUDE) + +# Zookeeper C client +ZOOKEEPER_PATH=${BUILD_PATH}/../../c +AC_CHECK_LIB(zookeeper_mt, main, [ZOOKEEPER="-L${ZOOKEEPER_PATH}/.libs -lzookeeper_mt"],,["-L${ZOOKEEPER_PATH}/.libs"]) +if test -z "${ZOOKEEPER}"; then + AC_ERROR("... zookeeper C client not found!") +fi + +AC_SUBST(ZOOKEEPER) +AC_SUBST(ZOOKEEPER_PATH) + +### log4cxx ### + +LOG4CXX_VERSION="0.10.0" +LOG4CXX_INCLUDE="/usr/local/include" +LOG4CXX_LIB_PATH="/usr/local/lib" +AC_CHECK_LIB(log4cxx, main, [LOG4CXX="-L${LOG4CXX_LIB_PATH} -llog4cxx"],,["-L${LOG4CXX_LIB_PATH}"]) +if test -z "${LOG4CXX}"; then + AC_ERROR("... log4cxx not found!") +fi + +AC_SUBST(LOG4CXX) +AC_SUBST(LOG4CXX_VERSION) +AC_SUBST(LOG4CXX_INCLUDE) + +# Checks for header files. +AC_HEADER_DIRENT +AC_HEADER_STDC +AC_CHECK_HEADERS([stdlib.h string.h stdio.h unistd.h boost/shared_ptr.hpp boost/algorithm/string.hpp boost/algorithm/string/split.hpp]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL +AC_C_CONST +AC_C_INLINE +AC_TYPE_SIZE_T +AC_C_VOLATILE + +AC_CONFIG_FILES([Makefile]) +AC_CONFIG_FILES([src/Makefile]) +AC_OUTPUT http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/Makefile.am ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/Makefile.am b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/Makefile.am new file mode 100644 index 0000000..641077a --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/Makefile.am @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AM_CXXFLAGS = -I${ZOOKEEPER_PATH}/include -I${ZOOKEEPER_PATH}/generated \ + -I$(top_srcdir)/include -I${LOG4CXX_INCLUDE} -I/usr/include \ + -I${XML2_INCLUDE} + +bin_PROGRAMS = zktreeutil + +zktreeutil_SOURCES = ZkAdaptor.cc ZkTreeUtil.cc ZkTreeUtilMain.cc +zktreeutil_LDADD = ${ZOOKEEPER} ${XML_LIBS} ${LOG4CXX} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/SimpleTree.h ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/SimpleTree.h b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/SimpleTree.h new file mode 100644 index 0000000..8226f05 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/SimpleTree.h @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SIMPLE_TREE_H__ +#define __SIMPLE_TREE_H__ + +#include +#include + +namespace zktreeutil +{ + using std::vector; + + /** + * \brief A simple tree data-structure template. + */ + template < class KeyType, class DataType > class SimpleTreeNode + { + private: + /** + * \brief The type representing simple-tree node smart-pointer. + */ + typedef boost::shared_ptr< SimpleTreeNode< KeyType, DataType > > SimpleTreeNodeSptr; + + public: + /** + * \brief Constructor. + * + * @param isRoot the flag indicating whether the node is root. + */ + SimpleTreeNode (bool isRoot=false) : isRoot_(isRoot) + { + } + + /** + * \brief Constructor. + * + * @param key the key stored at the tree node + * @param isRoot the flag indicating whether the node is root + */ + SimpleTreeNode (const KeyType& key, bool isRoot=false) : + isRoot_(isRoot), key_(key) + { + } + + /** + * \brief Constructor. + * + * @param key the key stored at the tree node + * @param val the value stored at the tree node + * @param isRoot the flag indicating whether the node is root + */ + SimpleTreeNode (const KeyType& key, const DataType& val, bool isRoot=false) : + isRoot_(isRoot), key_(key), val_(val) + { + } + + /** + * \brief Destructor. + */ + ~SimpleTreeNode () throw() {} + + /** + * \brief Add a child node to this node. + * + * @param node the child node to be added + */ + void addChild (const SimpleTreeNodeSptr node) { children_.push_back (node); } + + /** + * \brief Sets the key of this node. + * + * @param key the key to be set + */ + void setKey (const KeyType& key) { key_ = key; } + + /** + * \brief Sets the data of this node. + * + * @param val the value to be set + */ + void setData (const DataType& val) { val_ = val; } + + /** + * \brief Gets the key of this node. + * + * @return the key of this node + */ + KeyType getKey () const { return key_; } + + /** + * \brief Gets the data of this node. + * + * @return the value of this node + */ + DataType getData () const { return val_; } + + /** + * \brief Gets the i'th of this node. + * + * @param idx the index of the child node + * @return the child node + */ + SimpleTreeNodeSptr getChild (unsigned idx) const { return children_[idx]; } + + /** + * \brief Gets the number of children of this node. + * + * @return the number of children + */ + unsigned numChildren () const { return children_.size(); } + + /** + * \brief Indicates whether this node is root. + * + * @return 'true' if this node is root, 'false' otherwise + */ + bool isRoot () const { return isRoot_; } + + /** + * \brief Indicates whether this node is leaf node. + * + * @return 'true' if this node is leaf node, 'false' otherwise + */ + bool isLeaf () const { return !numChildren(); } + + private: + bool isRoot_; // Flag indicates if the node is root + KeyType key_; // Key of this node + DataType val_; // Value of this node + vector< SimpleTreeNodeSptr > children_; // List of children of this node + }; +} + +#endif // __SIMPLE_TREE_H__ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.cc ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.cc b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.cc new file mode 100644 index 0000000..1df175a --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.cc @@ -0,0 +1,513 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ZkAdaptor.h" +#include +#include +#include +#include +#include + +// Logger +static log4cxx::LoggerPtr zkLoggerPtr = log4cxx::Logger::getLogger ("zookeeper.core"); + +namespace zktreeutil +{ + /** + * \brief This class provides logic for checking if a request can be retried. + */ + class RetryHandler + { + public: + RetryHandler(const ZooKeeperConfig &zkConfig) : m_zkConfig(zkConfig) + { + if (zkConfig.getAutoReconnect()) + retries = 2; + else + retries = 0; + } + + /** + * \brief Attempts to fix a side effect of the given RC. + * + * @param rc the ZK error code + * @return whether the error code has been handled and the caller should + * retry an operation the caused this error + */ + bool handleRC(int rc) + { + //check if the given error code is recoverable + if (!retryOnError(rc)) + return false; + + std::cerr << "[zktreeuti] Number of retries left: " << retries << std::endl; + if (retries-- > 0) + return true; + else + return false; + } + + private: + /** + * The ZK config. + */ + const ZooKeeperConfig &m_zkConfig; + + /** + * The number of outstanding retries. + */ + int retries; + + /** + * Checks whether the given error entitles this adapter + * to retry the previous operation. + * + * @param zkErrorCode one of the ZK error code + */ + static bool retryOnError(int zkErrorCode) + { + return (zkErrorCode == ZCONNECTIONLOSS || zkErrorCode == ZOPERATIONTIMEOUT); + } + }; + + + // ======================================================================= + + ZooKeeperAdapter::ZooKeeperAdapter(ZooKeeperConfig config) throw(ZooKeeperException) : + m_zkConfig(config), + mp_zkHandle(NULL) + { + // Enforce setting up appropriate ZK log level + if (zkLoggerPtr->isDebugEnabled() +#ifdef LOG4CXX_TRACE + || zkLoggerPtr->isTraceEnabled() +#endif + ) + { + zoo_set_debug_level( ZOO_LOG_LEVEL_DEBUG ); + } else if (zkLoggerPtr->isInfoEnabled()) { + zoo_set_debug_level( ZOO_LOG_LEVEL_INFO ); + } else if (zkLoggerPtr->isWarnEnabled()) { + zoo_set_debug_level( ZOO_LOG_LEVEL_WARN ); + } else { + zoo_set_debug_level( ZOO_LOG_LEVEL_ERROR ); + } + + // Establish the connection + reconnect(); + } + + ZooKeeperAdapter::~ZooKeeperAdapter() + { + try + { + disconnect(); + } + catch (std::exception &e) + { + std::cerr << "[zktreeutil] An exception while disconnecting from ZK: " + << e.what() + << std::endl; + } + } + + void ZooKeeperAdapter::validatePath(const string &path) throw(ZooKeeperException) + { + if (path.find ("/") != 0) + { + std::ostringstream oss; + oss << "Node path must start with '/' but" "it was '" + << path + << "'"; + throw ZooKeeperException (oss.str()); + } + if (path.length() > 1) + { + if (path.rfind ("/") == path.length() - 1) + { + std::ostringstream oss; + oss << "Node path must not end with '/' but it was '" + << path + << "'"; + throw ZooKeeperException (oss.str()); + } + if (path.find( "//" ) != string::npos) + { + std::ostringstream oss; + oss << "Node path must not contain '//' but it was '" + << path + << "'"; + throw ZooKeeperException (oss.str()); + } + } + } + + void ZooKeeperAdapter::disconnect() + { + if (mp_zkHandle != NULL) + { + zookeeper_close (mp_zkHandle); + mp_zkHandle = NULL; + } + } + + void ZooKeeperAdapter::reconnect() throw(ZooKeeperException) + { + // Clear the connection state + disconnect(); + + // Establish a new connection to ZooKeeper + mp_zkHandle = zookeeper_init( m_zkConfig.getHosts().c_str(), + NULL, + m_zkConfig.getLeaseTimeout(), + 0, + NULL, + 0); + if (mp_zkHandle == NULL) + { + // Invalid handle returned + std::ostringstream oss; + oss << "Unable to connect to ZK running at '" + << m_zkConfig.getHosts() + << "'"; + throw ZooKeeperException (oss.str()); + } + + // Enter into connect loop + int64_t connWaitTime = m_zkConfig.getConnectTimeout(); + while (1) + { + int state = zoo_state (mp_zkHandle); + if (state == ZOO_CONNECTED_STATE) + { + // connected + std::cerr << "[zktreeutil] Connected! mp_zkHandle: " + << mp_zkHandle + << std::endl; + return; + } + else if ( state && state != ZOO_CONNECTING_STATE) + { + // Not connecting any more... some other issue + std::ostringstream oss; + oss << "Unable to connect to ZK running at '" + << m_zkConfig.getHosts() + << "'; state=" + << state; + throw ZooKeeperException (oss.str()); + } + + // Still connecting, wait and come back + struct timeval now; + gettimeofday( &now, NULL ); + int64_t milliSecs = -(now.tv_sec * 1000LL + now.tv_usec / 1000); + std::cerr << "[zktreeutil] About to wait 1 sec" << std::endl; + sleep (1); + gettimeofday( &now, NULL ); + milliSecs += now.tv_sec * 1000LL + now.tv_usec / 1000; + connWaitTime -= milliSecs; + // Timed out !!! + if (connWaitTime <= 0) + break; + } + + // Timed out while connecting + std::ostringstream oss; + oss << "Timed out while connecting to ZK running at '" + << m_zkConfig.getHosts() + << "'"; + throw ZooKeeperException (oss.str()); + } + + void ZooKeeperAdapter::verifyConnection() throw(ZooKeeperException) + { + // Check connection state + int state = zoo_state (mp_zkHandle); + if (state != ZOO_CONNECTED_STATE) + { + if (m_zkConfig.getAutoReconnect()) + { + // Trying to reconnect + std::cerr << "[zktreeutil] Trying to reconnect..." << std::endl; + reconnect(); + } + else + { + std::ostringstream oss; + oss << "Disconnected from ZK running at '" + << m_zkConfig.getHosts() + << "'; state=" + << state; + throw ZooKeeperException (oss.str()); + } + } + } + + bool ZooKeeperAdapter::createNode(const string &path, + const string &value, + int flags, + bool createAncestors) throw(ZooKeeperException) + { + const int MAX_PATH_LENGTH = 1024; + char realPath[MAX_PATH_LENGTH]; + realPath[0] = 0; + + int rc; + RetryHandler rh(m_zkConfig); + do + { + verifyConnection(); + rc = zoo_create( mp_zkHandle, + path.c_str(), + value.c_str(), + value.length(), + &ZOO_OPEN_ACL_UNSAFE, + flags, + realPath, + MAX_PATH_LENGTH ); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) // check return status + { + if (rc == ZNODEEXISTS) + { + //the node already exists + std::cerr << "[zktreeutil] ZK node " << path << " already exists" << std::endl; + return false; + } + else if (rc == ZNONODE && createAncestors) + { + std::cerr << "[zktreeutil] Intermediate ZK node missing in path " << path << std::endl; + //one of the ancestors doesn't exist so lets start from the root + //and make sure the whole path exists, creating missing nodes if + //necessary + for (string::size_type pos = 1; pos != string::npos; ) + { + pos = path.find( "/", pos ); + if (pos != string::npos) + { + try + { + createNode( path.substr( 0, pos ), "", 0, true ); + } + catch (ZooKeeperException &e) + { + throw ZooKeeperException( string("Unable to create " "node ") + path, rc ); + } + pos++; + } + else + { + // No more path components + return createNode( path, value, flags, false ); + } + } + } + + // Unexpected error during create + std::cerr << "[zktreeutil] Error in creating ZK node " << path << std::endl; + throw ZooKeeperException( string("Unable to create node ") + path, rc ); + } + + // Success + std::cerr << "[zktreeutil] " << realPath << " has been created" << std::endl; + return true; + } + + bool ZooKeeperAdapter::deleteNode(const string &path, + bool recursive, + int version) throw(ZooKeeperException) + { + // Validate the zk path + validatePath( path ); + + int rc; + RetryHandler rh(m_zkConfig); + do + { + verifyConnection(); + rc = zoo_delete( mp_zkHandle, path.c_str(), version ); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) //check return status + { + if (rc == ZNONODE) + { + std::cerr << "[zktreeutil] ZK Node " + << path + << " does not exist" + << std::endl; + return false; + } + if (rc == ZNOTEMPTY && recursive) + { + std::cerr << "[zktreeutil] ZK Node " + << path + << " not empty; deleting..." + << std::endl; + //get all children and delete them recursively... + vector nodeList = getNodeChildren (path); + for (vector::const_iterator i = nodeList.begin(); + i != nodeList.end(); + ++i) { + deleteNode( *i, true ); + } + //...and finally attempt to delete the node again + return deleteNode( path, false ); + } + + // Unexpected return without success + std::cerr << "[zktreeutil] Unable to delete ZK node " << path << std::endl; + throw ZooKeeperException( string("Unable to delete node ") + path, rc ); + } + + // success + std::cerr << "[zktreeutil] " << path << " has been deleted" << std::endl; + return true; + } + + vector< string > ZooKeeperAdapter::getNodeChildren (const string &path) throw (ZooKeeperException) + { + // Validate the zk path + validatePath( path ); + + String_vector children; + memset( &children, 0, sizeof(children) ); + int rc; + RetryHandler rh(m_zkConfig); + do + { + verifyConnection(); + rc = zoo_get_children( mp_zkHandle, + path.c_str(), + 0, + &children ); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) // check return code + { + std::cerr << "[zktreeutil] Error in fetching children of " << path << std::endl; + throw ZooKeeperException( string("Unable to get children of node ") + path, rc ); + } + else + { + vector< string > nodeList; + for (int i = 0; i < children.count; ++i) + { + //convert each child's path from relative to absolute + string absPath(path); + if (path != "/") + { + absPath.append( "/" ); + } + absPath.append( children.data[i] ); + nodeList.push_back( absPath ); + } + + //make sure the order is always deterministic + sort( nodeList.begin(), nodeList.end() ); + return nodeList; + } + } + + bool ZooKeeperAdapter::nodeExists(const string &path) throw(ZooKeeperException) + { + // Validate the zk path + validatePath( path ); + + struct Stat tmpStat; + struct Stat* stat = &tmpStat; + memset( stat, 0, sizeof(Stat) ); + + int rc; + RetryHandler rh(m_zkConfig); + do { + verifyConnection(); + rc = zoo_exists( mp_zkHandle, + path.c_str(), + 0, + stat ); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) + { + if (rc == ZNONODE) + return false; + // Some error + std::cerr << "[zktreeutil] Error in checking existence of " << path << std::endl; + throw ZooKeeperException( string("Unable to check existence of node ") + path, rc ); + } else { + return true; + } + } + + string ZooKeeperAdapter::getNodeData(const string &path) throw(ZooKeeperException) + { + // Validate the zk path + validatePath( path ); + + const int MAX_DATA_LENGTH = 128 * 1024; + char buffer[MAX_DATA_LENGTH]; + memset( buffer, 0, MAX_DATA_LENGTH ); + struct Stat tmpStat; + struct Stat* stat = &tmpStat; + memset( stat, 0, sizeof(Stat) ); + + int rc; + int len; + RetryHandler rh(m_zkConfig); + do { + verifyConnection(); + len = MAX_DATA_LENGTH - 1; + rc = zoo_get( mp_zkHandle, + path.c_str(), + 0, + buffer, &len, stat ); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) // checl return code + { + std::cerr << "[zktreeutil] Error in fetching value of " << path << std::endl; + throw ZooKeeperException( string("Unable to get data of node ") + path, rc ); + } + + // return data + return string( buffer, buffer + len ); + } + + void ZooKeeperAdapter::setNodeData(const string &path, + const string &value, + int version) throw(ZooKeeperException) + { + // Validate the zk path + validatePath( path ); + + int rc; + RetryHandler rh(m_zkConfig); + do { + verifyConnection(); + rc = zoo_set( mp_zkHandle, + path.c_str(), + value.c_str(), + value.length(), + version); + } while (rc != ZOK && rh.handleRC(rc)); + if (rc != ZOK) // check return code + { + std::cerr << "[zktreeutil] Error in setting value of " << path << std::endl; + throw ZooKeeperException( string("Unable to set data for node ") + path, rc ); + } + // success + } + +} /* end of 'namespace zktreeutil' */ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/b0df8fe1/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.h ---------------------------------------------------------------------- diff --git a/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.h b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.h new file mode 100644 index 0000000..4b68e28 --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zktreeutil/src/ZkAdaptor.h @@ -0,0 +1,327 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ZK_ADAPTER_H__ +#define __ZK_ADAPTER_H__ + +#include +#include + +extern "C" { +#include "zookeeper.h" +} + +namespace zktreeutil +{ + using std::string; + using std::vector; + + /** + * \brief A cluster related exception. + */ + class ZooKeeperException : public std::exception + { + public: + + /** + * \brief Constructor. + * + * @param msg the detailed message associated with this exception + */ + ZooKeeperException(const string& msg) : + m_message(msg), + m_zkErrorCode(0) {} + + /** + * \brief Constructor. + * + * @param msg the detailed message associated with this exception + * @param errorCode the ZK error code associated with this exception + */ + ZooKeeperException(const string &msg, int errorCode) : + m_zkErrorCode(errorCode) + { + char tmp[100]; + sprintf( tmp, " (ZK error code: %d)", errorCode ); + m_message = msg + tmp; + } + + /** + * \brief Destructor. + */ + ~ZooKeeperException() throw() {} + + /** + * \brief Returns detailed description of the exception. + */ + const char *what() const throw() + { + return m_message.c_str(); + } + + /** + * \brief Returns the ZK error code. + */ + int getZKErrorCode() const + { + return m_zkErrorCode; + } + + private: + + /** + * The detailed message associated with this exception. + */ + string m_message; + + /** + * The optional error code received from ZK. + */ + int m_zkErrorCode; + + }; + + /** + * \brief This class encapsulates configuration of a ZK client. + */ + class ZooKeeperConfig + { + public: + + /** + * \brief Constructor. + * + * @param hosts the comma separated list of host and port pairs of ZK nodes + * @param leaseTimeout the lease timeout (heartbeat) + * @param autoReconnect whether to allow for auto-reconnect + * @param connectTimeout the connect timeout, in milliseconds; + */ + ZooKeeperConfig(const string &hosts, + int leaseTimeout, + bool autoReconnect = true, + long long int connectTimeout = 15000) + : m_hosts(hosts), + m_leaseTimeout(leaseTimeout), + m_autoReconnect(autoReconnect), + m_connectTimeout(connectTimeout) {} + + /** + * \brief Returns the list of ZK hosts to connect to. + */ + string getHosts() const { return m_hosts; } + + /** + * \brief Returns the lease timeout. + */ + int getLeaseTimeout() const { return m_leaseTimeout; } + + /** + * \brief Returns whether {@link ZooKeeperAdapter} should attempt + * \brief to automatically reconnect in case of a connection failure. + */ + bool getAutoReconnect() const { return m_autoReconnect; } + + /** + * \brief Gets the connect timeout. + * + * @return the connect timeout + */ + long long int getConnectTimeout() const { return m_connectTimeout; } + + private: + + /** + * The host addresses of ZK nodes. + */ + const string m_hosts; + + /** + * The ZK lease timeout. + */ + const int m_leaseTimeout; + + /** + * True if this adapater should attempt to autoreconnect in case + * the current session has been dropped. + */ + const bool m_autoReconnect; + + /** + * How long to wait, in milliseconds, before a connection + * is established to ZK. + */ + const long long int m_connectTimeout; + }; + + /** + * \brief This is a wrapper around ZK C synchrounous API. + */ + class ZooKeeperAdapter + { + public: + /** + * \brief Constructor. + * Attempts to create a ZK adapter, optionally connecting + * to the ZK. Note, that if the connection is to be established + * and the given listener is NULL, some events may be lost, + * as they may arrive asynchronously before this method finishes. + * + * @param config the ZK configuration + * @throw ZooKeeperException if cannot establish connection to the given ZK + */ + ZooKeeperAdapter(ZooKeeperConfig config) throw(ZooKeeperException); + + /** + * \brief Destructor. + */ + ~ZooKeeperAdapter(); + + /** + * \brief Returns the current config. + */ + const ZooKeeperConfig &getZooKeeperConfig() const { return m_zkConfig; } + + /** + * \brief Restablishes connection to the ZK. + * If this adapter is already connected, the current connection + * will be dropped and a new connection will be established. + * + * @throw ZooKeeperException if cannot establish connection to the ZK + */ + void reconnect() throw(ZooKeeperException); + + /** + * \brief Disconnects from the ZK and unregisters {@link #mp_zkHandle}. + */ + void disconnect(); + + /** + * \brief Creates a new node identified by the given path. + * This method will optionally attempt to create all missing ancestors. + * + * @param path the absolute path name of the node to be created + * @param value the initial value to be associated with the node + * @param flags the ZK flags of the node to be created + * @param createAncestors if true and there are some missing ancestor nodes, + * this method will attempt to create them + * + * @return true if the node has been successfully created; false otherwise + * @throw ZooKeeperException if the operation has failed + */ + bool createNode(const string &path, + const string &value = "", + int flags = 0, + bool createAncestors = true) throw(ZooKeeperException); + + /** + * \brief Deletes a node identified by the given path. + * + * @param path the absolute path name of the node to be deleted + * @param recursive if true this method will attempt to remove + * all children of the given node if any exist + * @param version the expected version of the node. The function will + * fail if the actual version of the node does not match + * the expected version + * + * @return true if the node has been deleted; false otherwise + * @throw ZooKeeperException if the operation has failed + */ + bool deleteNode(const string &path, + bool recursive = false, + int version = -1) throw(ZooKeeperException); + + /** + * \brief Retrieves list of all children of the given node. + * + * @param path the absolute path name of the node for which to get children + * @return the list of absolute paths of child nodes, possibly empty + * @throw ZooKeeperException if the operation has failed + */ + vector getNodeChildren( const string &path) throw(ZooKeeperException); + + /** + * \brief Check the existence of path to a znode. + * + * @param path the absolute path name of the znode + * @return TRUE if the znode exists; FALSE otherwise + * @throw ZooKeeperException if the operation has failed + */ + bool nodeExists(const string &path) throw(ZooKeeperException); + + /** + * \brief Gets the given node's data. + * + * @param path the absolute path name of the node to get data from + * + * @return the node's data + * @throw ZooKeeperException if the operation has failed + */ + string getNodeData(const string &path) throw(ZooKeeperException); + + /** + * \brief Sets the given node's data. + * + * @param path the absolute path name of the node to get data from + * @param value the node's data to be set + * @param version the expected version of the node. The function will + * fail if the actual version of the node does not match + * the expected version + * + * @throw ZooKeeperException if the operation has failed + */ + void setNodeData(const string &path, + const string &value, + int version = -1) throw(ZooKeeperException); + + /** + * \brief Validates the given path to a node in ZK. + * + * @param the path to be validated + * + * @throw ZooKeeperException if the given path is not valid + * (for instance it doesn't start with "/") + */ + static void validatePath(const string &path) throw(ZooKeeperException); + + private: + + /** + * Verifies whether the connection is established, + * optionally auto reconnecting. + * + * @throw ZooKeeperConnection if this client is disconnected + * and auto-reconnect failed or was not allowed + */ + void verifyConnection() throw(ZooKeeperException); + + private: + + /** + * The current ZK configuration. + */ + const ZooKeeperConfig m_zkConfig; + + /** + * The current ZK session. + */ + zhandle_t *mp_zkHandle; + }; + +} /* end of 'namespace zktreeutil' */ + +#endif /* __ZK_ADAPTER_H__ */