Mailing-List: contact zookeeper-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: zookeeper-dev@<hadoop.apache.org>
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r891474 - in /hadoop/zookeeper/trunk: CHANGES.txt
 src/contrib/zkpython/src/c/pyzk_docstrings.h
 src/contrib/zkpython/src/c/zookeeper.c
 src/contrib/zkpython/src/test/get_set_test.py
Date: Wed, 16 Dec 2009 23:27:39 -0000
To: zookeeper-commits@hadoop.apache.org
From: mahadev@apache.org
Message-Id: <20091216232739.69DB723888EC@eris.apache.org>

Author: mahadev
Date: Wed Dec 16 23:27:38 2009
New Revision: 891474

URL: http://svn.apache.org/viewvc?rev=891474&view=rev
Log:
ZOOKEEPER-627. zkpython arbitrarily restricts the size of a \'get\' to 512 bytes (henry robinson via mahadev)

Modified:
    hadoop/zookeeper/trunk/CHANGES.txt
    hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/pyzk_docstrings.h
    hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/zookeeper.c
    hadoop/zookeeper/trunk/src/contrib/zkpython/src/test/get_set_test.py

Modified: hadoop/zookeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/CHANGES.txt?rev=891474&r1=891473&r2=891474&view=diff
==============================================================================
--- hadoop/zookeeper/trunk/CHANGES.txt (original)
+++ hadoop/zookeeper/trunk/CHANGES.txt Wed Dec 16 23:27:38 2009
@@ -176,6 +176,9 @@
   ZOOKEEPER-630. Trunk has duplicate ObserverTest.java files
   (henry robinson via phunt)
 
+  ZOOKEEPER-627. zkpython arbitrarily restricts the size of a 'get' to 512
+  bytes (henry robinson via mahadev)
+
 IMPROVEMENTS:
   ZOOKEEPER-473. cleanup junit tests to eliminate false positives due to
   "socket reuse" and failure to close client (phunt via mahadev)

Modified: hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/pyzk_docstrings.h
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/pyzk_docstrings.h?rev=891474&r1=891473&r2=891474&view=diff
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/pyzk_docstrings.h (original)
+++ hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/pyzk_docstrings.h Wed Dec 16 23:27:38 2009
@@ -579,7 +579,9 @@
 "\n"
 "(subsequent parameters are optional)\n"
 " watcher if not None, a watch will be set at the server to notify \n"
-"the client if the node changes.\n"
+" the client if the node changes.\n"
+" bufferlen: This value defaults to 1024*1024 - 1Mb. This method returns \n"
+" the minimum of bufferlen and the true length of the znode's data. \n"
 "RETURNS:\n"
 " the data associated with the node\n"
 "OK operation completed succesfully\n"

Modified: hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/zookeeper.c
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/zookeeper.c?rev=891474&r1=891473&r2=891474&view=diff
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/zookeeper.c (original)
+++ hadoop/zookeeper/trunk/src/contrib/zkpython/src/c/zookeeper.c Wed Dec 16 23:27:38 2009
@@ -848,33 +848,49 @@
   return build_stat(stat);
 }
 
+// As per ZK documentation, datanodes are limited
+// to 1Mb. Why not do a stat followed by a get, to 
+// determine how big the buffer should be? Because the znode
+// may get updated between calls, so we can't guarantee a 
+// complete get anyhow. 
+#define GET_BUFFER_SIZE 1024*1024
+
+// pyzoo_get has an extra parameter over the java/C equivalents.
+// If you set the fourth integer parameter buffer_len, we return 
+// min(buffer_len, datalength) bytes. This is set by default to 
+// GET_BUFFER_SIZE
 static PyObject *pyzoo_get(PyObject *self, PyObject *args)
 {
   int zkhid;
   char *path;
-  char buffer[512];
-  memset(buffer,0,sizeof(char)*512);
-  int buffer_len=512;
+  char *buffer; 
+  int buffer_len=GET_BUFFER_SIZE;
   struct Stat stat;
   PyObject *watcherfn = Py_None;
   pywatcher_t *pw = NULL;
-  if (!PyArg_ParseTuple(args, "is|O", &zkhid, &path, &watcherfn))
+  if (!PyArg_ParseTuple(args, "is|Oi", &zkhid, &path, &watcherfn, &buffer_len))
     return NULL;
   CHECK_ZHANDLE(zkhid);
-  if (watcherfn != Py_None)
-    pw = create_pywatcher( zkhid, watcherfn,0 );
+  if (watcherfn != Py_None) 
+		{
+			pw = create_pywatcher( zkhid, watcherfn,0 );
+		}
+	buffer = malloc(sizeof(char)*buffer_len);
   int err = zoo_wget(zhandles[zkhid], path, 
 		     watcherfn != Py_None ? watcher_dispatch : NULL, 
 		     pw, buffer, 
 		     &buffer_len, &stat);
+	
   PyObject *stat_dict = build_stat( &stat );
+
   if (err != ZOK)
-    {
+		{
       PyErr_SetString(err_to_exception(err), zerror(err));
       return NULL;
     }
-
-  return Py_BuildValue( "(s#,N)", buffer,buffer_len, stat_dict );
+	PyObject *ret = Py_BuildValue( "(s#,N)", buffer,buffer_len, stat_dict );
+	free(buffer);
+  return ret;
 }
 
 PyObject *pyzoo_get_acl(PyObject *self, PyObject *args)

Modified: hadoop/zookeeper/trunk/src/contrib/zkpython/src/test/get_set_test.py
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/zkpython/src/test/get_set_test.py?rev=891474&r1=891473&r2=891474&view=diff
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/zkpython/src/test/get_set_test.py (original)
+++ hadoop/zookeeper/trunk/src/contrib/zkpython/src/test/get_set_test.py Wed Dec 16 23:27:38 2009
@@ -71,6 +71,25 @@
         self.assertEqual(self.stat, None, "Stat should be none!")
         self.assertEqual(self.value, None, "Value should be none!")
 
+    def test_sync_get_large_datanode(self):
+        """
+        Test that we can retrieve datanode sizes up to
+        1Mb with default parameters (depends on ZooKeeper server).
+        """
+
+        data = ''.join(["A" for x in xrange(1024*1023)])
+        self.ensureDeleted("/zk-python-test-large-datanode")
+        zookeeper.create(self.handle, "/zk-python-test-large-datanode", data,
+                         [{"perms":0x1f, "scheme":"world", "id" :"anyone"}])
+        (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode")
+        self.assertEqual(len(ret), 1024*1023,
+                         "Should have got 1Mb returned, instead got %s" % len(ret))
+        (ret,stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode",None,500)
+        self.assertEqual(len(ret), 500,
+                         "Should have got 500 bytes returned, instead got %s" % len(ret))
+
+
+
     def test_async_getset(self):
         self.cv = threading.Condition()
         def get_callback(handle, rc, value, stat):