hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r758857 [2/3] - in /hadoop/hive/trunk: ./ service/if/ service/lib/ service/lib/py/ service/lib/py/fb303/ service/lib/py/fb303_scripts/ service/lib/py/thrift/ service/lib/py/thrift/protocol/ service/lib/py/thrift/reflection/ service/lib/py/t...
Date Thu, 26 Mar 2009 21:12:26 GMT
Added: hadoop/hive/trunk/service/lib/py/thrift/protocol/TBinaryProtocol.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/protocol/TBinaryProtocol.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/protocol/TBinaryProtocol.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/protocol/TBinaryProtocol.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,246 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+from TProtocol import *
+from struct import pack, unpack
+
+class TBinaryProtocol(TProtocolBase):
+
+  """Binary implementation of the Thrift protocol driver."""
+
+  # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be
+  # positive, converting this into a long. If we hardcode the int value
+  # instead it'll stay in 32 bit-land.
+
+  # VERSION_MASK = 0xffff0000
+  VERSION_MASK = -65536
+
+  # VERSION_1 = 0x80010000
+  VERSION_1 = -2147418112
+
+  TYPE_MASK = 0x000000ff
+
+  def __init__(self, trans, strictRead=False, strictWrite=True):
+    TProtocolBase.__init__(self, trans)
+    self.strictRead = strictRead
+    self.strictWrite = strictWrite
+
+  def writeMessageBegin(self, name, type, seqid):
+    if self.strictWrite:
+      self.writeI32(TBinaryProtocol.VERSION_1 | type)
+      self.writeString(name)
+      self.writeI32(seqid)
+    else:
+      self.writeString(name)
+      self.writeByte(type)
+      self.writeI32(seqid)
+
+  def writeMessageEnd(self):
+    pass
+
+  def writeStructBegin(self, name):
+    pass
+
+  def writeStructEnd(self):
+    pass
+
+  def writeFieldBegin(self, name, type, id):
+    self.writeByte(type)
+    self.writeI16(id)
+
+  def writeFieldEnd(self):
+    pass
+
+  def writeFieldStop(self):
+    self.writeByte(TType.STOP);
+
+  def writeMapBegin(self, ktype, vtype, size):
+    self.writeByte(ktype)
+    self.writeByte(vtype)
+    self.writeI32(size)
+
+  def writeMapEnd(self):
+    pass
+
+  def writeListBegin(self, etype, size):
+    self.writeByte(etype)
+    self.writeI32(size)
+
+  def writeListEnd(self):
+    pass
+
+  def writeSetBegin(self, etype, size):
+    self.writeByte(etype)
+    self.writeI32(size)
+
+  def writeSetEnd(self):
+    pass
+
+  def writeBool(self, bool):
+    if bool:
+      self.writeByte(1)
+    else:
+      self.writeByte(0)
+
+  def writeByte(self, byte):
+    buff = pack("!b", byte)
+    self.trans.write(buff)
+
+  def writeI16(self, i16):
+    buff = pack("!h", i16)
+    self.trans.write(buff)
+
+  def writeI32(self, i32):
+    buff = pack("!i", i32)
+    self.trans.write(buff)
+
+  def writeI64(self, i64):
+    buff = pack("!q", i64)
+    self.trans.write(buff)
+
+  def writeDouble(self, dub):
+    buff = pack("!d", dub)
+    self.trans.write(buff)
+
+  def writeString(self, str):
+    self.writeI32(len(str))
+    self.trans.write(str)
+
+  def readMessageBegin(self):
+    sz = self.readI32()
+    if sz < 0:
+      version = sz & TBinaryProtocol.VERSION_MASK
+      if version != TBinaryProtocol.VERSION_1:
+        raise TProtocolException(TProtocolException.BAD_VERSION, 'Bad version in readMessageBegin: %d' % (sz))
+      type = sz & TBinaryProtocol.TYPE_MASK
+      name = self.readString()
+      seqid = self.readI32()
+    else:
+      if self.strictRead:
+        raise TProtocolException(TProtocolException.BAD_VERSION, 'No protocol version header')
+      name = self.trans.readAll(sz)
+      type = self.readByte()
+      seqid = self.readI32()
+    return (name, type, seqid)
+
+  def readMessageEnd(self):
+    pass
+
+  def readStructBegin(self):
+    pass
+
+  def readStructEnd(self):
+    pass
+
+  def readFieldBegin(self):
+    type = self.readByte()
+    if type == TType.STOP:
+      return (None, type, 0)
+    id = self.readI16()
+    return (None, type, id)
+
+  def readFieldEnd(self):
+    pass
+
+  def readMapBegin(self):
+    ktype = self.readByte()
+    vtype = self.readByte()
+    size = self.readI32()
+    return (ktype, vtype, size)
+
+  def readMapEnd(self):
+    pass
+
+  def readListBegin(self):
+    etype = self.readByte()
+    size = self.readI32()
+    return (etype, size)
+
+  def readListEnd(self):
+    pass
+
+  def readSetBegin(self):
+    etype = self.readByte()
+    size = self.readI32()
+    return (etype, size)
+
+  def readSetEnd(self):
+    pass
+
+  def readBool(self):
+    byte = self.readByte()
+    if byte == 0:
+      return False
+    return True
+
+  def readByte(self):
+    buff = self.trans.readAll(1)
+    val, = unpack('!b', buff)
+    return val
+
+  def readI16(self):
+    buff = self.trans.readAll(2)
+    val, = unpack('!h', buff)
+    return val
+
+  def readI32(self):
+    buff = self.trans.readAll(4)
+    val, = unpack('!i', buff)
+    return val
+
+  def readI64(self):
+    buff = self.trans.readAll(8)
+    val, = unpack('!q', buff)
+    return val
+
+  def readDouble(self):
+    buff = self.trans.readAll(8)
+    val, = unpack('!d', buff)
+    return val
+
+  def readString(self):
+    len = self.readI32()
+    str = self.trans.readAll(len)
+    return str
+
+
+class TBinaryProtocolFactory:
+  def __init__(self, strictRead=False, strictWrite=True):
+    self.strictRead = strictRead
+    self.strictWrite = strictWrite
+
+  def getProtocol(self, trans):
+    prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite)
+    return prot
+
+
+class TBinaryProtocolAccelerated(TBinaryProtocol):
+
+  """C-Accelerated version of TBinaryProtocol.
+
+  This class does not override any of TBinaryProtocol's methods,
+  but the generated code recognizes it directly and will call into
+  our C module to do the encoding, bypassing this object entirely.
+  We inherit from TBinaryProtocol so that the normal TBinaryProtocol
+  encoding can happen if the fastbinary module doesn't work for some
+  reason.  (TODO(dreiss): Make this happen sanely in more cases.)
+
+  In order to take advantage of the C module, just use
+  TBinaryProtocolAccelerated instead of TBinaryProtocol.
+
+  NOTE:  This code was contributed by an external developer.
+         The internal Thrift team has reviewed and tested it,
+         but we cannot guarantee that it is production-ready.
+         Please feel free to report bugs and/or success stories
+         to the public mailing list.
+  """
+
+  pass
+
+
+class TBinaryProtocolAcceleratedFactory:
+  def getProtocol(self, trans):
+    return TBinaryProtocolAccelerated(trans)

Added: hadoop/hive/trunk/service/lib/py/thrift/protocol/TProtocol.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/protocol/TProtocol.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/protocol/TProtocol.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/protocol/TProtocol.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,192 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+from thrift.Thrift import *
+
+class TProtocolException(TException):
+
+  """Custom Protocol Exception class"""
+
+  UNKNOWN = 0
+  INVALID_DATA = 1
+  NEGATIVE_SIZE = 2
+  SIZE_LIMIT = 3
+  BAD_VERSION = 4
+
+  def __init__(self, type=UNKNOWN, message=None):
+    TException.__init__(self, message)
+    self.type = type
+
+class TProtocolBase:
+
+  """Base class for Thrift protocol driver."""
+
+  def __init__(self, trans):
+    self.trans = trans
+
+  def writeMessageBegin(self, name, type, seqid):
+    pass
+
+  def writeMessageEnd(self):
+    pass
+
+  def writeStructBegin(self, name):
+    pass
+
+  def writeStructEnd(self):
+    pass
+
+  def writeFieldBegin(self, name, type, id):
+    pass
+
+  def writeFieldEnd(self):
+    pass
+
+  def writeFieldStop(self):
+    pass
+
+  def writeMapBegin(self, ktype, vtype, size):
+    pass
+
+  def writeMapEnd(self):
+    pass
+
+  def writeListBegin(self, etype, size):
+    pass
+
+  def writeListEnd(self):
+    pass
+
+  def writeSetBegin(self, etype, size):
+    pass
+
+  def writeSetEnd(self):
+    pass
+
+  def writeBool(self, bool):
+    pass
+
+  def writeByte(self, byte):
+    pass
+
+  def writeI16(self, i16):
+    pass
+
+  def writeI32(self, i32):
+    pass
+
+  def writeI64(self, i64):
+    pass
+
+  def writeDouble(self, dub):
+    pass
+
+  def writeString(self, str):
+    pass
+
+  def readMessageBegin(self):
+    pass
+
+  def readMessageEnd(self):
+    pass
+
+  def readStructBegin(self):
+    pass
+
+  def readStructEnd(self):
+    pass
+
+  def readFieldBegin(self):
+    pass
+
+  def readFieldEnd(self):
+    pass
+
+  def readMapBegin(self):
+    pass
+
+  def readMapEnd(self):
+    pass
+
+  def readListBegin(self):
+    pass
+
+  def readListEnd(self):
+    pass
+
+  def readSetBegin(self):
+    pass
+
+  def readSetEnd(self):
+    pass
+
+  def readBool(self):
+    pass
+
+  def readByte(self):
+    pass
+
+  def readI16(self):
+    pass
+
+  def readI32(self):
+    pass
+
+  def readI64(self):
+    pass
+
+  def readDouble(self):
+    pass
+
+  def readString(self):
+    pass
+
+  def skip(self, type):
+    if type == TType.STOP:
+      return
+    elif type == TType.BOOL:
+      self.readBool()
+    elif type == TType.BYTE:
+      self.readByte()
+    elif type == TType.I16:
+      self.readI16()
+    elif type == TType.I32:
+      self.readI32()
+    elif type == TType.I64:
+      self.readI64()
+    elif type == TType.DOUBLE:
+      self.readDouble()
+    elif type == TType.STRING:
+      self.readString()
+    elif type == TType.STRUCT:
+      name = self.readStructBegin()
+      while True:
+        (name, type, id) = self.readFieldBegin()
+        if type == TType.STOP:
+          break
+        self.skip(type)
+        self.readFieldEnd()
+      self.readStructEnd()
+    elif type == TType.MAP:
+      (ktype, vtype, size) = self.readMapBegin()
+      for i in range(size):
+        self.skip(ktype)
+        self.skip(vtype)
+      self.readMapEnd()
+    elif type == TType.SET:
+      (etype, size) = self.readSetBegin()
+      for i in range(size):
+        self.skip(etype)
+      self.readSetEnd()
+    elif type == TType.LIST:
+      (etype, size) = self.readListBegin()
+      for i in range(size):
+        self.skip(etype)
+      self.readListEnd()
+
+class TProtocolFactory:
+  def getProtocol(self, trans):
+    pass

Added: hadoop/hive/trunk/service/lib/py/thrift/protocol/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/protocol/__init__.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/protocol/__init__.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/protocol/__init__.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,7 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary']

Added: hadoop/hive/trunk/service/lib/py/thrift/protocol/fastbinary.c
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/protocol/fastbinary.c?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/protocol/fastbinary.c (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/protocol/fastbinary.c Thu Mar 26 21:12:24 2009
@@ -0,0 +1,1171 @@
+// Copyright (c) 2006- Facebook
+// Distributed under the Thrift Software License
+//
+// See accompanying file LICENSE or visit the Thrift site at:
+// http://developers.facebook.com/thrift/
+//
+// NOTE:  This code was contributed by an external developer.
+//        The internal Thrift team has reviewed and tested it,
+//        but we cannot guarantee that it is production-ready.
+//        Please feel free to report bugs and/or success stories
+//        to the public mailing list.
+
+#include <Python.h>
+#include "cStringIO.h"
+#include <stdbool.h>
+#include <stdint.h>
+#include <netinet/in.h>
+
+// TODO(dreiss): defval appears to be unused.  Look into removing it.
+// TODO(dreiss): Make parse_spec_args recursive, and cache the output
+//               permanently in the object.  (Malloc and orphan.)
+// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
+//               Can cStringIO let us work with a BufferedTransport?
+// TODO(dreiss): Don't ignore the rv from cwrite (maybe).
+
+/* ====== BEGIN UTILITIES ====== */
+
+#define INIT_OUTBUF_SIZE 128
+
+// Stolen out of TProtocol.h.
+// It would be a huge pain to have both get this from one place.
+typedef enum TType {
+  T_STOP       = 0,
+  T_VOID       = 1,
+  T_BOOL       = 2,
+  T_BYTE       = 3,
+  T_I08        = 3,
+  T_I16        = 6,
+  T_I32        = 8,
+  T_U64        = 9,
+  T_I64        = 10,
+  T_DOUBLE     = 4,
+  T_STRING     = 11,
+  T_UTF7       = 11,
+  T_STRUCT     = 12,
+  T_MAP        = 13,
+  T_SET        = 14,
+  T_LIST       = 15,
+  T_UTF8       = 16,
+  T_UTF16      = 17
+} TType;
+
+#ifndef __BYTE_ORDER
+# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
+#  define __BYTE_ORDER BYTE_ORDER
+#  define __LITTLE_ENDIAN LITTLE_ENDIAN
+#  define __BIG_ENDIAN BIG_ENDIAN
+# else
+#  error "Cannot determine endianness"
+# endif
+#endif
+
+// Same comment as the enum.  Sorry.
+#if __BYTE_ORDER == __BIG_ENDIAN
+# define ntohll(n) (n)
+# define htonll(n) (n)
+#elif __BYTE_ORDER == __LITTLE_ENDIAN
+# if defined(__GNUC__) && defined(__GLIBC__)
+#  include <byteswap.h>
+#  define ntohll(n) bswap_64(n)
+#  define htonll(n) bswap_64(n)
+# else /* GNUC & GLIBC */
+#  define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) )
+#  define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) )
+# endif /* GNUC & GLIBC */
+#else /* __BYTE_ORDER */
+# error "Can't define htonll or ntohll!"
+#endif
+
+// Doing a benchmark shows that interning actually makes a difference, amazingly.
+#define INTERN_STRING(value) _intern_ ## value
+
+#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() )
+#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) )
+
+// Py_ssize_t was not defined before Python 2.5
+#if (PY_VERSION_HEX < 0x02050000)
+typedef int Py_ssize_t;
+#endif
+
+/**
+ * A cache of the spec_args for a set or list,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+  TType element_type;
+  PyObject* typeargs;
+} SetListTypeArgs;
+
+/**
+ * A cache of the spec_args for a map,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+  TType ktag;
+  TType vtag;
+  PyObject* ktypeargs;
+  PyObject* vtypeargs;
+} MapTypeArgs;
+
+/**
+ * A cache of the spec_args for a struct,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+  PyObject* klass;
+  PyObject* spec;
+} StructTypeArgs;
+
+/**
+ * A cache of the item spec from a struct specification,
+ * so we don't have to keep calling PyTuple_GET_ITEM.
+ */
+typedef struct {
+  int tag;
+  TType type;
+  PyObject* attrname;
+  PyObject* typeargs;
+  PyObject* defval;
+} StructItemSpec;
+
+/**
+ * A cache of the two key attributes of a CReadableTransport,
+ * so we don't have to keep calling PyObject_GetAttr.
+ */
+typedef struct {
+  PyObject* stringiobuf;
+  PyObject* refill_callable;
+} DecodeBuffer;
+
+/** Pointer to interned string to speed up attribute lookup. */
+static PyObject* INTERN_STRING(cstringio_buf);
+/** Pointer to interned string to speed up attribute lookup. */
+static PyObject* INTERN_STRING(cstringio_refill);
+
+static inline bool
+check_ssize_t_32(Py_ssize_t len) {
+  // error from getting the int
+  if (INT_CONV_ERROR_OCCURRED(len)) {
+    return false;
+  }
+  if (!CHECK_RANGE(len, 0, INT32_MAX)) {
+    PyErr_SetString(PyExc_OverflowError, "string size out of range");
+    return false;
+  }
+  return true;
+}
+
+static inline bool
+parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) {
+  long val = PyInt_AsLong(o);
+
+  if (INT_CONV_ERROR_OCCURRED(val)) {
+    return false;
+  }
+  if (!CHECK_RANGE(val, min, max)) {
+    PyErr_SetString(PyExc_OverflowError, "int out of range");
+    return false;
+  }
+
+  *ret = (int32_t) val;
+  return true;
+}
+
+
+/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */
+
+static bool
+parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) {
+  if (PyTuple_Size(typeargs) != 2) {
+    PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args");
+    return false;
+  }
+
+  dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
+  if (INT_CONV_ERROR_OCCURRED(dest->element_type)) {
+    return false;
+  }
+
+  dest->typeargs = PyTuple_GET_ITEM(typeargs, 1);
+
+  return true;
+}
+
+static bool
+parse_map_args(MapTypeArgs* dest, PyObject* typeargs) {
+  if (PyTuple_Size(typeargs) != 4) {
+    PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map");
+    return false;
+  }
+
+  dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0));
+  if (INT_CONV_ERROR_OCCURRED(dest->ktag)) {
+    return false;
+  }
+
+  dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2));
+  if (INT_CONV_ERROR_OCCURRED(dest->vtag)) {
+    return false;
+  }
+
+  dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1);
+  dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3);
+
+  return true;
+}
+
+static bool
+parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) {
+  if (PyTuple_Size(typeargs) != 2) {
+    PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args");
+    return false;
+  }
+
+  dest->klass = PyTuple_GET_ITEM(typeargs, 0);
+  dest->spec = PyTuple_GET_ITEM(typeargs, 1);
+
+  return true;
+}
+
+static int
+parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
+
+  // i'd like to use ParseArgs here, but it seems to be a bottleneck.
+  if (PyTuple_Size(spec_tuple) != 5) {
+    PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple");
+    return false;
+  }
+
+  dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0));
+  if (INT_CONV_ERROR_OCCURRED(dest->tag)) {
+    return false;
+  }
+
+  dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1));
+  if (INT_CONV_ERROR_OCCURRED(dest->type)) {
+    return false;
+  }
+
+  dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2);
+  dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3);
+  dest->defval = PyTuple_GET_ITEM(spec_tuple, 4);
+  return true;
+}
+
+/* ====== END UTILITIES ====== */
+
+
+/* ====== BEGIN WRITING FUNCTIONS ====== */
+
+/* --- LOW-LEVEL WRITING FUNCTIONS --- */
+
+static void writeByte(PyObject* outbuf, int8_t val) {
+  int8_t net = val;
+  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t));
+}
+
+static void writeI16(PyObject* outbuf, int16_t val) {
+  int16_t net = (int16_t)htons(val);
+  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t));
+}
+
+static void writeI32(PyObject* outbuf, int32_t val) {
+  int32_t net = (int32_t)htonl(val);
+  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t));
+}
+
+static void writeI64(PyObject* outbuf, int64_t val) {
+  int64_t net = (int64_t)htonll(val);
+  PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t));
+}
+
+static void writeDouble(PyObject* outbuf, double dub) {
+  // Unfortunately, bitwise_cast doesn't work in C.  Bad C!
+  union {
+    double f;
+    int64_t t;
+  } transfer;
+  transfer.f = dub;
+  writeI64(outbuf, transfer.t);
+}
+
+
+/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */
+
+static int
+output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) {
+  /*
+   * Refcounting Strategy:
+   *
+   * We assume that elements of the thrift_spec tuple are not going to be
+   * mutated, so we don't ref count those at all. Other than that, we try to
+   * keep a reference to all the user-created objects while we work with them.
+   * output_val assumes that a reference is already held. The *caller* is
+   * responsible for handling references
+   */
+
+  switch (type) {
+
+  case T_BOOL: {
+    int v = PyObject_IsTrue(value);
+    if (v == -1) {
+      return false;
+    }
+
+    writeByte(output, (int8_t) v);
+    break;
+  }
+  case T_I08: {
+    int32_t val;
+
+    if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) {
+      return false;
+    }
+
+    writeByte(output, (int8_t) val);
+    break;
+  }
+  case T_I16: {
+    int32_t val;
+
+    if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) {
+      return false;
+    }
+
+    writeI16(output, (int16_t) val);
+    break;
+  }
+  case T_I32: {
+    int32_t val;
+
+    if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) {
+      return false;
+    }
+
+    writeI32(output, val);
+    break;
+  }
+  case T_I64: {
+    int64_t nval = PyLong_AsLongLong(value);
+
+    if (INT_CONV_ERROR_OCCURRED(nval)) {
+      return false;
+    }
+
+    if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) {
+      PyErr_SetString(PyExc_OverflowError, "int out of range");
+      return false;
+    }
+
+    writeI64(output, nval);
+    break;
+  }
+
+  case T_DOUBLE: {
+    double nval = PyFloat_AsDouble(value);
+    if (nval == -1.0 && PyErr_Occurred()) {
+      return false;
+    }
+
+    writeDouble(output, nval);
+    break;
+  }
+
+  case T_STRING: {
+    Py_ssize_t len = PyString_Size(value);
+
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    writeI32(output, (int32_t) len);
+    PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len);
+    break;
+  }
+
+  case T_LIST:
+  case T_SET: {
+    Py_ssize_t len;
+    SetListTypeArgs parsedargs;
+    PyObject *item;
+    PyObject *iterator;
+
+    if (!parse_set_list_args(&parsedargs, typeargs)) {
+      return false;
+    }
+
+    len = PyObject_Length(value);
+
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    writeByte(output, parsedargs.element_type);
+    writeI32(output, (int32_t) len);
+
+    iterator =  PyObject_GetIter(value);
+    if (iterator == NULL) {
+      return false;
+    }
+
+    while ((item = PyIter_Next(iterator))) {
+      if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) {
+        Py_DECREF(item);
+        Py_DECREF(iterator);
+        return false;
+      }
+      Py_DECREF(item);
+    }
+
+    Py_DECREF(iterator);
+
+    if (PyErr_Occurred()) {
+      return false;
+    }
+
+    break;
+  }
+
+  case T_MAP: {
+    PyObject *k, *v;
+    Py_ssize_t pos = 0;
+    Py_ssize_t len;
+
+    MapTypeArgs parsedargs;
+
+    len = PyDict_Size(value);
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    if (!parse_map_args(&parsedargs, typeargs)) {
+      return false;
+    }
+
+    writeByte(output, parsedargs.ktag);
+    writeByte(output, parsedargs.vtag);
+    writeI32(output, len);
+
+    // TODO(bmaurer): should support any mapping, not just dicts
+    while (PyDict_Next(value, &pos, &k, &v)) {
+      // TODO(dreiss): Think hard about whether these INCREFs actually
+      //               turn any unsafe scenarios into safe scenarios.
+      Py_INCREF(k);
+      Py_INCREF(v);
+
+      if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs)
+          || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) {
+        Py_DECREF(k);
+        Py_DECREF(v);
+        return false;
+      }
+    }
+    break;
+  }
+
+  // TODO(dreiss): Consider breaking this out as a function
+  //               the way we did for decode_struct.
+  case T_STRUCT: {
+    StructTypeArgs parsedargs;
+    Py_ssize_t nspec;
+    Py_ssize_t i;
+
+    if (!parse_struct_args(&parsedargs, typeargs)) {
+      return false;
+    }
+
+    nspec = PyTuple_Size(parsedargs.spec);
+
+    if (nspec == -1) {
+      return false;
+    }
+
+    for (i = 0; i < nspec; i++) {
+      StructItemSpec parsedspec;
+      PyObject* spec_tuple;
+      PyObject* instval = NULL;
+
+      spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i);
+      if (spec_tuple == Py_None) {
+        continue;
+      }
+
+      if (!parse_struct_item_spec (&parsedspec, spec_tuple)) {
+        return false;
+      }
+
+      instval = PyObject_GetAttr(value, parsedspec.attrname);
+
+      if (!instval) {
+        return false;
+      }
+
+      if (instval == Py_None) {
+        Py_DECREF(instval);
+        continue;
+      }
+
+      writeByte(output, (int8_t) parsedspec.type);
+      writeI16(output, parsedspec.tag);
+
+      if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) {
+        Py_DECREF(instval);
+        return false;
+      }
+
+      Py_DECREF(instval);
+    }
+
+    writeByte(output, (int8_t)T_STOP);
+    break;
+  }
+
+  case T_STOP:
+  case T_VOID:
+  case T_UTF16:
+  case T_UTF8:
+  case T_U64:
+  default:
+    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+    return false;
+
+  }
+
+  return true;
+}
+
+
+/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */
+
+static PyObject *
+encode_binary(PyObject *self, PyObject *args) {
+  PyObject* enc_obj;
+  PyObject* type_args;
+  PyObject* buf;
+  PyObject* ret = NULL;
+
+  if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) {
+    return NULL;
+  }
+
+  buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE);
+  if (output_val(buf, enc_obj, T_STRUCT, type_args)) {
+    ret = PycStringIO->cgetvalue(buf);
+  }
+
+  Py_DECREF(buf);
+  return ret;
+}
+
+/* ====== END WRITING FUNCTIONS ====== */
+
+
+/* ====== BEGIN READING FUNCTIONS ====== */
+
+/* --- LOW-LEVEL READING FUNCTIONS --- */
+
+static void
+free_decodebuf(DecodeBuffer* d) {
+  Py_XDECREF(d->stringiobuf);
+  Py_XDECREF(d->refill_callable);
+}
+
+static bool
+decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) {
+  dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf));
+  if (!dest->stringiobuf) {
+    return false;
+  }
+
+  if (!PycStringIO_InputCheck(dest->stringiobuf)) {
+    free_decodebuf(dest);
+    PyErr_SetString(PyExc_TypeError, "expecting stringio input");
+    return false;
+  }
+
+  dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill));
+
+  if(!dest->refill_callable) {
+    free_decodebuf(dest);
+    return false;
+  }
+
+  if (!PyCallable_Check(dest->refill_callable)) {
+    free_decodebuf(dest);
+    PyErr_SetString(PyExc_TypeError, "expecting callable");
+    return false;
+  }
+
+  return true;
+}
+
+static bool readBytes(DecodeBuffer* input, char** output, int len) {
+  int read;
+
+  // TODO(dreiss): Don't fear the malloc.  Think about taking a copy of
+  //               the partial read instead of forcing the transport
+  //               to prepend it to its buffer.
+
+  read = PycStringIO->cread(input->stringiobuf, output, len);
+
+  if (read == len) {
+    return true;
+  } else if (read == -1) {
+    return false;
+  } else {
+    PyObject* newiobuf;
+
+    // using building functions as this is a rare codepath
+    newiobuf = PyObject_CallFunction(
+        input->refill_callable, "s#i", *output, read, len, NULL);
+    if (newiobuf == NULL) {
+      return false;
+    }
+
+    // must do this *AFTER* the call so that we don't deref the io buffer
+    Py_CLEAR(input->stringiobuf);
+    input->stringiobuf = newiobuf;
+
+    read = PycStringIO->cread(input->stringiobuf, output, len);
+
+    if (read == len) {
+      return true;
+    } else if (read == -1) {
+      return false;
+    } else {
+      // TODO(dreiss): This could be a valid code path for big binary blobs.
+      PyErr_SetString(PyExc_TypeError,
+          "refill claimed to have refilled the buffer, but didn't!!");
+      return false;
+    }
+  }
+}
+
+static int8_t readByte(DecodeBuffer* input) {
+  char* buf;
+  if (!readBytes(input, &buf, sizeof(int8_t))) {
+    return -1;
+  }
+
+  return *(int8_t*) buf;
+}
+
+static int16_t readI16(DecodeBuffer* input) {
+  char* buf;
+  if (!readBytes(input, &buf, sizeof(int16_t))) {
+    return -1;
+  }
+
+  return (int16_t) ntohs(*(int16_t*) buf);
+}
+
+static int32_t readI32(DecodeBuffer* input) {
+  char* buf;
+  if (!readBytes(input, &buf, sizeof(int32_t))) {
+    return -1;
+  }
+  return (int32_t) ntohl(*(int32_t*) buf);
+}
+
+
+static int64_t readI64(DecodeBuffer* input) {
+  char* buf;
+  if (!readBytes(input, &buf, sizeof(int64_t))) {
+    return -1;
+  }
+
+  return (int64_t) ntohll(*(int64_t*) buf);
+}
+
+static double readDouble(DecodeBuffer* input) {
+  union {
+    int64_t f;
+    double t;
+  } transfer;
+
+  transfer.f = readI64(input);
+  if (transfer.f == -1) {
+    return -1;
+  }
+  return transfer.t;
+}
+
+static bool
+checkTypeByte(DecodeBuffer* input, TType expected) {
+  TType got = readByte(input);
+  if (INT_CONV_ERROR_OCCURRED(got)) {
+    return false;
+  }
+
+  if (expected != got) {
+    PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field");
+    return false;
+  }
+  return true;
+}
+
+static bool
+skip(DecodeBuffer* input, TType type) {
+#define SKIPBYTES(n) \
+  do { \
+    if (!readBytes(input, &dummy_buf, (n))) { \
+      return false; \
+    } \
+  } while(0)
+
+  char* dummy_buf;
+
+  switch (type) {
+
+  case T_BOOL:
+  case T_I08: SKIPBYTES(1); break;
+  case T_I16: SKIPBYTES(2); break;
+  case T_I32: SKIPBYTES(4); break;
+  case T_I64:
+  case T_DOUBLE: SKIPBYTES(8); break;
+
+  case T_STRING: {
+    // TODO(dreiss): Find out if these check_ssize_t32s are really necessary.
+    int len = readI32(input);
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+    SKIPBYTES(len);
+    break;
+  }
+
+  case T_LIST:
+  case T_SET: {
+    TType etype;
+    int len, i;
+
+    etype = readByte(input);
+    if (etype == -1) {
+      return false;
+    }
+
+    len = readI32(input);
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    for (i = 0; i < len; i++) {
+      if (!skip(input, etype)) {
+        return false;
+      }
+    }
+    break;
+  }
+
+  case T_MAP: {
+    TType ktype, vtype;
+    int len, i;
+
+    ktype = readByte(input);
+    if (ktype == -1) {
+      return false;
+    }
+
+    vtype = readByte(input);
+    if (vtype == -1) {
+      return false;
+    }
+
+    len = readI32(input);
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    for (i = 0; i < len; i++) {
+      if (!(skip(input, ktype) && skip(input, vtype))) {
+        return false;
+      }
+    }
+    break;
+  }
+
+  case T_STRUCT: {
+    while (true) {
+      TType type;
+
+      type = readByte(input);
+      if (type == -1) {
+        return false;
+      }
+
+      if (type == T_STOP)
+        break;
+
+      SKIPBYTES(2); // tag
+      if (!skip(input, type)) {
+        return false;
+      }
+    }
+    break;
+  }
+
+  case T_STOP:
+  case T_VOID:
+  case T_UTF16:
+  case T_UTF8:
+  case T_U64:
+  default:
+    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+    return false;
+
+  }
+
+  return true;
+
+#undef SKIPBYTES
+}
+
+
+/* --- HELPER FUNCTION FOR DECODE_VAL --- */
+
+static PyObject*
+decode_val(DecodeBuffer* input, TType type, PyObject* typeargs);
+
+static bool
+decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) {
+  int spec_seq_len = PyTuple_Size(spec_seq);
+  if (spec_seq_len == -1) {
+    return false;
+  }
+
+  while (true) {
+    TType type;
+    int16_t tag;
+    PyObject* item_spec;
+    PyObject* fieldval = NULL;
+    StructItemSpec parsedspec;
+
+    type = readByte(input);
+    if (type == -1) {
+      return false;
+    }
+    if (type == T_STOP) {
+      break;
+    }
+    tag = readI16(input);
+    if (INT_CONV_ERROR_OCCURRED(tag)) {
+      return false;
+    }
+    if (tag >= 0 && tag < spec_seq_len) {
+      item_spec = PyTuple_GET_ITEM(spec_seq, tag);
+    } else {
+      item_spec = Py_None;
+    }
+
+    if (item_spec == Py_None) {
+      if (!skip(input, type)) {
+        return false;
+      } else {
+        continue;
+      }
+    }
+
+    if (!parse_struct_item_spec(&parsedspec, item_spec)) {
+      return false;
+    }
+    if (parsedspec.type != type) {
+      PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading");
+      return false;
+    }
+
+    fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs);
+    if (fieldval == NULL) {
+      return false;
+    }
+
+    if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) {
+      Py_DECREF(fieldval);
+      return false;
+    }
+    Py_DECREF(fieldval);
+  }
+  return true;
+}
+
+
+/* --- MAIN RECURSIVE INPUT FUCNTION --- */
+
+// Returns a new reference.
+static PyObject*
+decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
+  switch (type) {
+
+  case T_BOOL: {
+    int8_t v = readByte(input);
+    if (INT_CONV_ERROR_OCCURRED(v)) {
+      return NULL;
+    }
+
+    switch (v) {
+    case 0: Py_RETURN_FALSE;
+    case 1: Py_RETURN_TRUE;
+    // Don't laugh.  This is a potentially serious issue.
+    default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL;
+    }
+    break;
+  }
+  case T_I08: {
+    int8_t v = readByte(input);
+    if (INT_CONV_ERROR_OCCURRED(v)) {
+      return NULL;
+    }
+
+    return PyInt_FromLong(v);
+  }
+  case T_I16: {
+    int16_t v = readI16(input);
+    if (INT_CONV_ERROR_OCCURRED(v)) {
+      return NULL;
+    }
+    return PyInt_FromLong(v);
+  }
+  case T_I32: {
+    int32_t v = readI32(input);
+    if (INT_CONV_ERROR_OCCURRED(v)) {
+      return NULL;
+    }
+    return PyInt_FromLong(v);
+  }
+
+  case T_I64: {
+    int64_t v = readI64(input);
+    if (INT_CONV_ERROR_OCCURRED(v)) {
+      return NULL;
+    }
+    // TODO(dreiss): Find out if we can take this fastpath always when
+    //               sizeof(long) == sizeof(long long).
+    if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) {
+      return PyInt_FromLong((long) v);
+    }
+
+    return PyLong_FromLongLong(v);
+  }
+
+  case T_DOUBLE: {
+    double v = readDouble(input);
+    if (v == -1.0 && PyErr_Occurred()) {
+      return false;
+    }
+    return PyFloat_FromDouble(v);
+  }
+
+  case T_STRING: {
+    Py_ssize_t len = readI32(input);
+    char* buf;
+    if (!readBytes(input, &buf, len)) {
+      return NULL;
+    }
+
+    return PyString_FromStringAndSize(buf, len);
+  }
+
+  case T_LIST:
+  case T_SET: {
+    SetListTypeArgs parsedargs;
+    int32_t len;
+    PyObject* ret = NULL;
+    int i;
+
+    if (!parse_set_list_args(&parsedargs, typeargs)) {
+      return NULL;
+    }
+
+    if (!checkTypeByte(input, parsedargs.element_type)) {
+      return NULL;
+    }
+
+    len = readI32(input);
+    if (!check_ssize_t_32(len)) {
+      return NULL;
+    }
+
+    ret = PyList_New(len);
+    if (!ret) {
+      return NULL;
+    }
+
+    for (i = 0; i < len; i++) {
+      PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs);
+      if (!item) {
+        Py_DECREF(ret);
+        return NULL;
+      }
+      PyList_SET_ITEM(ret, i, item);
+    }
+
+    // TODO(dreiss): Consider biting the bullet and making two separate cases
+    //               for list and set, avoiding this post facto conversion.
+    if (type == T_SET) {
+      PyObject* setret;
+#if (PY_VERSION_HEX < 0x02050000)
+      // hack needed for older versions
+      setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL);
+#else
+      // official version
+      setret = PySet_New(ret);
+#endif
+      Py_DECREF(ret);
+      return setret;
+    }
+    return ret;
+  }
+
+  case T_MAP: {
+    int32_t len;
+    int i;
+    MapTypeArgs parsedargs;
+    PyObject* ret = NULL;
+
+    if (!parse_map_args(&parsedargs, typeargs)) {
+      return NULL;
+    }
+
+    if (!checkTypeByte(input, parsedargs.ktag)) {
+      return NULL;
+    }
+    if (!checkTypeByte(input, parsedargs.vtag)) {
+      return NULL;
+    }
+
+    len = readI32(input);
+    if (!check_ssize_t_32(len)) {
+      return false;
+    }
+
+    ret = PyDict_New();
+    if (!ret) {
+      goto error;
+    }
+
+    for (i = 0; i < len; i++) {
+      PyObject* k = NULL;
+      PyObject* v = NULL;
+      k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs);
+      if (k == NULL) {
+        goto loop_error;
+      }
+      v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs);
+      if (v == NULL) {
+        goto loop_error;
+      }
+      if (PyDict_SetItem(ret, k, v) == -1) {
+        goto loop_error;
+      }
+
+      Py_DECREF(k);
+      Py_DECREF(v);
+      continue;
+
+      // Yuck!  Destructors, anyone?
+      loop_error:
+      Py_XDECREF(k);
+      Py_XDECREF(v);
+      goto error;
+    }
+
+    return ret;
+
+    error:
+    Py_XDECREF(ret);
+    return NULL;
+  }
+
+  case T_STRUCT: {
+    StructTypeArgs parsedargs;
+    if (!parse_struct_args(&parsedargs, typeargs)) {
+      return NULL;
+    }
+
+    PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL);
+    if (!ret) {
+      return NULL;
+    }
+
+    if (!decode_struct(input, ret, parsedargs.spec)) {
+      Py_DECREF(ret);
+      return NULL;
+    }
+
+    return ret;
+  }
+
+  case T_STOP:
+  case T_VOID:
+  case T_UTF16:
+  case T_UTF8:
+  case T_U64:
+  default:
+    PyErr_SetString(PyExc_TypeError, "Unexpected TType");
+    return NULL;
+  }
+}
+
+
+/* --- TOP-LEVEL WRAPPER FOR INPUT -- */
+
+static PyObject*
+decode_binary(PyObject *self, PyObject *args) {
+  PyObject* output_obj = NULL;
+  PyObject* transport = NULL;
+  PyObject* typeargs = NULL;
+  StructTypeArgs parsedargs;
+  DecodeBuffer input = {};
+
+  if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
+    return NULL;
+  }
+
+  if (!parse_struct_args(&parsedargs, typeargs)) {
+    return NULL;
+  }
+
+  if (!decode_buffer_from_obj(&input, transport)) {
+    return NULL;
+  }
+
+  if (!decode_struct(&input, output_obj, parsedargs.spec)) {
+    free_decodebuf(&input);
+    return NULL;
+  }
+
+  free_decodebuf(&input);
+
+  Py_RETURN_NONE;
+}
+
+/* ====== END READING FUNCTIONS ====== */
+
+
+/* -- PYTHON MODULE SETUP STUFF --- */
+
+static PyMethodDef ThriftFastBinaryMethods[] = {
+
+  {"encode_binary",  encode_binary, METH_VARARGS, ""},
+  {"decode_binary",  decode_binary, METH_VARARGS, ""},
+
+  {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initfastbinary(void) {
+#define INIT_INTERN_STRING(value) \
+  do { \
+    INTERN_STRING(value) = PyString_InternFromString(#value); \
+    if(!INTERN_STRING(value)) return; \
+  } while(0)
+
+  INIT_INTERN_STRING(cstringio_buf);
+  INIT_INTERN_STRING(cstringio_refill);
+#undef INIT_INTERN_STRING
+
+  PycString_IMPORT;
+  if (PycStringIO == NULL) return;
+
+  (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
+}

Added: hadoop/hive/trunk/service/lib/py/thrift/reflection/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/reflection/__init__.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/reflection/__init__.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/reflection/__init__.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,8 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+__all__ = ['limited']
+

Added: hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/__init__.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/__init__.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/__init__.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1 @@
+__all__ = ['ttypes', 'constants']

Added: hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/constants.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/constants.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/constants.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/constants.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,9 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+from thrift.Thrift import *
+from ttypes import *
+

Added: hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/ttypes.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/ttypes.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/ttypes.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/reflection/limited/ttypes.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,535 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+from thrift.Thrift import *
+
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+try:
+  from thrift.protocol import fastbinary
+except:
+  fastbinary = None
+
+
+class TTypeTag:
+  T_VOID = 1
+  T_BOOL = 2
+  T_BYTE = 3
+  T_I16 = 6
+  T_I32 = 8
+  T_I64 = 10
+  T_DOUBLE = 4
+  T_STRING = 11
+  T_STRUCT = 12
+  T_MAP = 13
+  T_SET = 14
+  T_LIST = 15
+  T_ENUM = 101
+  T_NOT_REFLECTED = 102
+
+class SimpleType:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'ttype', None, None, ), # 1
+    (2, TType.STRING, 'name', None, None, ), # 2
+  )
+
+  def __init__(self, d=None):
+    self.ttype = None
+    self.name = None
+    if isinstance(d, dict):
+      if 'ttype' in d:
+        self.ttype = d['ttype']
+      if 'name' in d:
+        self.name = d['name']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.ttype = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRING:
+          self.name = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('SimpleType')
+    if self.ttype != None:
+      oprot.writeFieldBegin('ttype', TType.I32, 1)
+      oprot.writeI32(self.ttype)
+      oprot.writeFieldEnd()
+    if self.name != None:
+      oprot.writeFieldBegin('name', TType.STRING, 2)
+      oprot.writeString(self.name)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class ContainerType:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'ttype', None, None, ), # 1
+    (2, TType.STRUCT, 'subtype1', (SimpleType, SimpleType.thrift_spec), None, ), # 2
+    (3, TType.STRUCT, 'subtype2', (SimpleType, SimpleType.thrift_spec), None, ), # 3
+  )
+
+  def __init__(self, d=None):
+    self.ttype = None
+    self.subtype1 = None
+    self.subtype2 = None
+    if isinstance(d, dict):
+      if 'ttype' in d:
+        self.ttype = d['ttype']
+      if 'subtype1' in d:
+        self.subtype1 = d['subtype1']
+      if 'subtype2' in d:
+        self.subtype2 = d['subtype2']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I32:
+          self.ttype = iprot.readI32();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRUCT:
+          self.subtype1 = SimpleType()
+          self.subtype1.read(iprot)
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRUCT:
+          self.subtype2 = SimpleType()
+          self.subtype2.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('ContainerType')
+    if self.ttype != None:
+      oprot.writeFieldBegin('ttype', TType.I32, 1)
+      oprot.writeI32(self.ttype)
+      oprot.writeFieldEnd()
+    if self.subtype1 != None:
+      oprot.writeFieldBegin('subtype1', TType.STRUCT, 2)
+      self.subtype1.write(oprot)
+      oprot.writeFieldEnd()
+    if self.subtype2 != None:
+      oprot.writeFieldBegin('subtype2', TType.STRUCT, 3)
+      self.subtype2.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class ThriftType:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.BOOL, 'is_container', None, None, ), # 1
+    (2, TType.STRUCT, 'simple_type', (SimpleType, SimpleType.thrift_spec), None, ), # 2
+    (3, TType.STRUCT, 'container_type', (ContainerType, ContainerType.thrift_spec), None, ), # 3
+  )
+
+  def __init__(self, d=None):
+    self.is_container = None
+    self.simple_type = None
+    self.container_type = None
+    if isinstance(d, dict):
+      if 'is_container' in d:
+        self.is_container = d['is_container']
+      if 'simple_type' in d:
+        self.simple_type = d['simple_type']
+      if 'container_type' in d:
+        self.container_type = d['container_type']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.BOOL:
+          self.is_container = iprot.readBool();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRUCT:
+          self.simple_type = SimpleType()
+          self.simple_type.read(iprot)
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRUCT:
+          self.container_type = ContainerType()
+          self.container_type.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('ThriftType')
+    if self.is_container != None:
+      oprot.writeFieldBegin('is_container', TType.BOOL, 1)
+      oprot.writeBool(self.is_container)
+      oprot.writeFieldEnd()
+    if self.simple_type != None:
+      oprot.writeFieldBegin('simple_type', TType.STRUCT, 2)
+      self.simple_type.write(oprot)
+      oprot.writeFieldEnd()
+    if self.container_type != None:
+      oprot.writeFieldBegin('container_type', TType.STRUCT, 3)
+      self.container_type.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class Argument:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I16, 'key', None, None, ), # 1
+    (2, TType.STRING, 'name', None, None, ), # 2
+    (3, TType.STRUCT, 'type', (ThriftType, ThriftType.thrift_spec), None, ), # 3
+  )
+
+  def __init__(self, d=None):
+    self.key = None
+    self.name = None
+    self.type = None
+    if isinstance(d, dict):
+      if 'key' in d:
+        self.key = d['key']
+      if 'name' in d:
+        self.name = d['name']
+      if 'type' in d:
+        self.type = d['type']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.I16:
+          self.key = iprot.readI16();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRING:
+          self.name = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRUCT:
+          self.type = ThriftType()
+          self.type.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('Argument')
+    if self.key != None:
+      oprot.writeFieldBegin('key', TType.I16, 1)
+      oprot.writeI16(self.key)
+      oprot.writeFieldEnd()
+    if self.name != None:
+      oprot.writeFieldBegin('name', TType.STRING, 2)
+      oprot.writeString(self.name)
+      oprot.writeFieldEnd()
+    if self.type != None:
+      oprot.writeFieldBegin('type', TType.STRUCT, 3)
+      self.type.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class Method:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRING, 'name', None, None, ), # 1
+    (2, TType.STRUCT, 'return_type', (ThriftType, ThriftType.thrift_spec), None, ), # 2
+    (3, TType.LIST, 'arguments', (TType.STRUCT,(Argument, Argument.thrift_spec)), None, ), # 3
+  )
+
+  def __init__(self, d=None):
+    self.name = None
+    self.return_type = None
+    self.arguments = None
+    if isinstance(d, dict):
+      if 'name' in d:
+        self.name = d['name']
+      if 'return_type' in d:
+        self.return_type = d['return_type']
+      if 'arguments' in d:
+        self.arguments = d['arguments']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.name = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRUCT:
+          self.return_type = ThriftType()
+          self.return_type.read(iprot)
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.LIST:
+          self.arguments = []
+          (_etype3, _size0) = iprot.readListBegin()
+          for _i4 in xrange(_size0):
+            _elem5 = Argument()
+            _elem5.read(iprot)
+            self.arguments.append(_elem5)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('Method')
+    if self.name != None:
+      oprot.writeFieldBegin('name', TType.STRING, 1)
+      oprot.writeString(self.name)
+      oprot.writeFieldEnd()
+    if self.return_type != None:
+      oprot.writeFieldBegin('return_type', TType.STRUCT, 2)
+      self.return_type.write(oprot)
+      oprot.writeFieldEnd()
+    if self.arguments != None:
+      oprot.writeFieldBegin('arguments', TType.LIST, 3)
+      oprot.writeListBegin(TType.STRUCT, len(self.arguments))
+      for iter6 in self.arguments:
+        iter6.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class Service:
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRING, 'name', None, None, ), # 1
+    (2, TType.LIST, 'methods', (TType.STRUCT,(Method, Method.thrift_spec)), None, ), # 2
+    (3, TType.BOOL, 'fully_reflected', None, None, ), # 3
+  )
+
+  def __init__(self, d=None):
+    self.name = None
+    self.methods = None
+    self.fully_reflected = None
+    if isinstance(d, dict):
+      if 'name' in d:
+        self.name = d['name']
+      if 'methods' in d:
+        self.methods = d['methods']
+      if 'fully_reflected' in d:
+        self.fully_reflected = d['fully_reflected']
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.name = iprot.readString();
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.LIST:
+          self.methods = []
+          (_etype10, _size7) = iprot.readListBegin()
+          for _i11 in xrange(_size7):
+            _elem12 = Method()
+            _elem12.read(iprot)
+            self.methods.append(_elem12)
+          iprot.readListEnd()
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.BOOL:
+          self.fully_reflected = iprot.readBool();
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('Service')
+    if self.name != None:
+      oprot.writeFieldBegin('name', TType.STRING, 1)
+      oprot.writeString(self.name)
+      oprot.writeFieldEnd()
+    if self.methods != None:
+      oprot.writeFieldBegin('methods', TType.LIST, 2)
+      oprot.writeListBegin(TType.STRUCT, len(self.methods))
+      for iter13 in self.methods:
+        iter13.write(oprot)
+      oprot.writeListEnd()
+      oprot.writeFieldEnd()
+    if self.fully_reflected != None:
+      oprot.writeFieldBegin('fully_reflected', TType.BOOL, 3)
+      oprot.writeBool(self.fully_reflected)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def __str__(self):
+    return str(self.__dict__)
+
+  def __repr__(self):
+    return repr(self.__dict__)
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+

Added: hadoop/hive/trunk/service/lib/py/thrift/server/THttpServer.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/server/THttpServer.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/server/THttpServer.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/server/THttpServer.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,50 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+import BaseHTTPServer
+
+from thrift.server import TServer
+from thrift.transport import TTransport
+
+class THttpServer(TServer.TServer):
+  """A simple HTTP-based Thrift server
+
+  This class is not very performant, but it is useful (for example) for
+  acting as a mock version of an Apache-based PHP Thrift endpoint."""
+
+  def __init__(self, processor, server_address,
+      inputProtocolFactory, outputProtocolFactory = None):
+    """Set up protocol factories and HTTP server.
+
+    See BaseHTTPServer for server_address.
+    See TServer for protocol factories."""
+
+    if outputProtocolFactory is None:
+      outputProtocolFactory = inputProtocolFactory
+
+    TServer.TServer.__init__(self, processor, None, None, None,
+        inputProtocolFactory, outputProtocolFactory)
+
+    thttpserver = self
+
+    class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler):
+      def do_POST(self):
+        # Don't care about the request path.
+        self.send_response(200)
+        self.send_header("content-type", "application/x-thrift")
+        self.end_headers()
+
+        itrans = TTransport.TFileObjectTransport(self.rfile)
+        otrans = TTransport.TFileObjectTransport(self.wfile)
+        iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
+        oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
+        thttpserver.processor.process(iprot, oprot)
+        otrans.flush()
+
+    self.httpd = BaseHTTPServer.HTTPServer(server_address, RequestHander)
+
+  def serve(self):
+    self.httpd.serve_forever()

Added: hadoop/hive/trunk/service/lib/py/thrift/server/TNonblockingServer.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/server/TNonblockingServer.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/server/TNonblockingServer.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/server/TNonblockingServer.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,291 @@
+"""Implementation of non-blocking server.
+
+The main idea of the server is reciving and sending requests
+only from main thread.
+
+It also makes thread pool server in tasks terms, not connections.
+"""
+import threading
+import socket
+import Queue
+import select
+import struct
+import logging
+
+from thrift.transport import TTransport
+from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory
+
+__all__ = ['TNonblockingServer']
+
+class Worker(threading.Thread):
+    """Worker is a small helper to process incoming connection."""
+    def __init__(self, queue):
+        threading.Thread.__init__(self)
+        self.queue = queue
+
+    def run(self):
+        """Process queries from task queue, stop if processor is None."""
+        while True:
+            try:
+                processor, iprot, oprot, otrans, callback = self.queue.get()
+                if processor is None:
+                    break
+                processor.process(iprot, oprot)
+                callback(True, otrans.getvalue())
+            except Exception:
+                logging.exception("Exception while processing request")
+                callback(False, '')
+
+WAIT_LEN = 0
+WAIT_MESSAGE = 1
+WAIT_PROCESS = 2
+SEND_ANSWER = 3
+CLOSED = 4
+
+def locked(func):
+    "Decorator which locks self.lock."
+    def nested(self, *args, **kwargs):
+        self.lock.acquire()
+        try:
+            return func(self, *args, **kwargs)
+        finally:
+            self.lock.release()
+    return nested
+
+def socket_exception(func):
+    "Decorator close object on socket.error."
+    def read(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except socket.error:
+            self.close()
+    return read
+
+class Connection:
+    """Basic class is represented connection.
+    
+    It can be in state:
+        WAIT_LEN --- connection is reading request len.
+        WAIT_MESSAGE --- connection is reading request.
+        WAIT_PROCESS --- connection has just read whole request and 
+            waits for call ready routine.
+        SEND_ANSWER --- connection is sending answer string (including length
+            of answer).
+        CLOSED --- socket was closed and connection should be deleted.
+    """
+    def __init__(self, new_socket, wake_up):
+        self.socket = new_socket
+        self.socket.setblocking(False)
+        self.status = WAIT_LEN
+        self.len = 0
+        self.message = ''
+        self.lock = threading.Lock()
+        self.wake_up = wake_up
+
+    def _read_len(self):
+        """Reads length of request.
+        
+        It's really paranoic routine and it may be replaced by 
+        self.socket.recv(4)."""
+        read = self.socket.recv(4 - len(self.message))
+        if len(read) == 0:
+            # if we read 0 bytes and self.message is empty, it means client close 
+            # connection
+            if len(self.message) != 0:
+                logging.error("can't read frame size from socket")
+            self.close()
+            return
+        self.message += read
+        if len(self.message) == 4:
+            self.len, = struct.unpack('!i', self.message)
+            if self.len < 0:
+                logging.error("negative frame size, it seems client"\
+                    " doesn't use FramedTransport")
+                self.close()
+            elif self.len == 0:
+                logging.error("empty frame, it's really strange")
+                self.close()
+            else:
+                self.message = ''
+                self.status = WAIT_MESSAGE
+
+    @socket_exception
+    def read(self):
+        """Reads data from stream and switch state."""
+        assert self.status in (WAIT_LEN, WAIT_MESSAGE)
+        if self.status == WAIT_LEN:
+            self._read_len()
+            # go back to the main loop here for simplicity instead of
+            # falling through, even though there is a good chance that
+            # the message is already available
+        elif self.status == WAIT_MESSAGE:
+            read = self.socket.recv(self.len - len(self.message))
+            if len(read) == 0:
+                logging.error("can't read frame from socket (get %d of %d bytes)" %
+                    (len(self.message), self.len))
+                self.close()
+                return
+            self.message += read
+            if len(self.message) == self.len:
+                self.status = WAIT_PROCESS
+
+    @socket_exception
+    def write(self):
+        """Writes data from socket and switch state."""
+        assert self.status == SEND_ANSWER
+        sent = self.socket.send(self.message)
+        if sent == len(self.message):
+            self.status = WAIT_LEN
+            self.message = ''
+            self.len = 0
+        else:
+            self.message = self.message[sent:]
+
+    @locked
+    def ready(self, all_ok, message):
+        """Callback function for switching state and waking up main thread.
+        
+        This function is the only function witch can be called asynchronous.
+        
+        The ready can switch Connection to three states:
+            WAIT_LEN if request was async.
+            SEND_ANSWER if request was processed in normal way.
+            CLOSED if request throws unexpected exception.
+        
+        The one wakes up main thread.
+        """
+        assert self.status == WAIT_PROCESS
+        if not all_ok:
+            self.close()
+            self.wake_up()
+            return
+        self.len = ''
+        self.message = struct.pack('!i', len(message)) + message
+        if len(message) == 0:
+            # it was async request, do not write answer
+            self.status = WAIT_LEN
+        else:
+            self.status = SEND_ANSWER
+        self.wake_up()
+
+    @locked
+    def is_writeable(self):
+        "Returns True if connection should be added to write list of select."
+        return self.status == SEND_ANSWER
+
+    # it's not necessary, but...
+    @locked
+    def is_readable(self):
+        "Returns True if connection should be added to read list of select."
+        return self.status in (WAIT_LEN, WAIT_MESSAGE)
+
+    @locked
+    def is_closed(self):
+        "Returns True if connection is closed."
+        return self.status == CLOSED
+
+    def fileno(self):
+        "Returns the file descriptor of the associated socket."
+        return self.socket.fileno()
+
+    def close(self):
+        "Closes connection"
+        self.status = CLOSED
+        self.socket.close()
+
+class TNonblockingServer:
+    """Non-blocking server."""
+    def __init__(self, processor, lsocket, inputProtocolFactory=None, 
+            outputProtocolFactory=None, threads=10):
+        self.processor = processor
+        self.socket = lsocket
+        self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory()
+        self.out_protocol = outputProtocolFactory or self.in_protocol
+        self.threads = int(threads)
+        self.clients = {}
+        self.tasks = Queue.Queue()
+        self._read, self._write = socket.socketpair()
+        self.prepared = False
+
+    def setNumThreads(self, num):
+        """Set the number of worker threads that should be created."""
+        # implement ThreadPool interface
+        assert not self.prepared, "You can't change number of threads for working server"
+        self.threads = num
+
+    def prepare(self):
+        """Prepares server for serve requests."""
+        self.socket.listen()
+        for _ in xrange(self.threads):
+            thread = Worker(self.tasks)
+            thread.setDaemon(True)
+            thread.start()
+        self.prepared = True
+
+    def wake_up(self):
+        """Wake up main thread.
+        
+        The server usualy waits in select call in we should terminate one.
+        The simplest way is using socketpair.
+        
+        Select always wait to read from the first socket of socketpair.
+        
+        In this case, we can just write anything to the second socket from
+        socketpair."""
+        self._write.send('1')
+
+    def _select(self):
+        """Does select on open connections."""
+        readable = [self.socket.handle.fileno(), self._read.fileno()]
+        writable = []
+        for i, connection in self.clients.items():
+            if connection.is_readable():
+                readable.append(connection.fileno())
+            if connection.is_writeable():
+                writable.append(connection.fileno())
+            if connection.is_closed():
+                del self.clients[i]
+        return select.select(readable, writable, readable)
+        
+    def handle(self):
+        """Handle requests.
+       
+        WARNING! You must call prepare BEFORE calling handle.
+        """
+        assert self.prepared, "You have to call prepare before handle"
+        rset, wset, xset = self._select()
+        for readable in rset:
+            if readable == self._read.fileno():
+                # don't care i just need to clean readable flag
+                self._read.recv(1024) 
+            elif readable == self.socket.handle.fileno():
+                client = self.socket.accept().handle
+                self.clients[client.fileno()] = Connection(client, self.wake_up)
+            else:
+                connection = self.clients[readable]
+                connection.read()
+                if connection.status == WAIT_PROCESS:
+                    itransport = TTransport.TMemoryBuffer(connection.message)
+                    otransport = TTransport.TMemoryBuffer()
+                    iprot = self.in_protocol.getProtocol(itransport)
+                    oprot = self.out_protocol.getProtocol(otransport)
+                    self.tasks.put([self.processor, iprot, oprot, 
+                                    otransport, connection.ready])
+        for writeable in wset:
+            self.clients[writeable].write()
+        for oob in xset:
+            self.clients[oob].close()
+            del self.clients[oob]
+
+    def close(self):
+        """Closes the server."""
+        for _ in xrange(self.threads):
+            self.tasks.put([None, None, None, None, None])
+        self.socket.close()
+        self.prepared = False
+        
+    def serve(self):
+        """Serve forever."""
+        self.prepare()
+        while True:
+            self.handle()

Added: hadoop/hive/trunk/service/lib/py/thrift/server/TServer.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/server/TServer.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/server/TServer.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/server/TServer.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,257 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+import logging
+import sys
+import os
+import traceback
+import threading
+import Queue
+
+from thrift.Thrift import TProcessor
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+
+class TServer:
+
+  """Base interface for a server, which must have a serve method."""
+
+  """ 3 constructors for all servers:
+  1) (processor, serverTransport)
+  2) (processor, serverTransport, transportFactory, protocolFactory)
+  3) (processor, serverTransport,
+      inputTransportFactory, outputTransportFactory,
+      inputProtocolFactory, outputProtocolFactory)"""
+  def __init__(self, *args):
+    if (len(args) == 2):
+      self.__initArgs__(args[0], args[1],
+                        TTransport.TTransportFactoryBase(),
+                        TTransport.TTransportFactoryBase(),
+                        TBinaryProtocol.TBinaryProtocolFactory(),
+                        TBinaryProtocol.TBinaryProtocolFactory())
+    elif (len(args) == 4):
+      self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3])
+    elif (len(args) == 6):
+      self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5])
+
+  def __initArgs__(self, processor, serverTransport,
+                   inputTransportFactory, outputTransportFactory,
+                   inputProtocolFactory, outputProtocolFactory):
+    self.processor = processor
+    self.serverTransport = serverTransport
+    self.inputTransportFactory = inputTransportFactory
+    self.outputTransportFactory = outputTransportFactory
+    self.inputProtocolFactory = inputProtocolFactory
+    self.outputProtocolFactory = outputProtocolFactory
+
+  def serve(self):
+    pass
+
+class TSimpleServer(TServer):
+
+  """Simple single-threaded server that just pumps around one transport."""
+
+  def __init__(self, *args):
+    TServer.__init__(self, *args)
+
+  def serve(self):
+    self.serverTransport.listen()
+    while True:
+      client = self.serverTransport.accept()
+      itrans = self.inputTransportFactory.getTransport(client)
+      otrans = self.outputTransportFactory.getTransport(client)
+      iprot = self.inputProtocolFactory.getProtocol(itrans)
+      oprot = self.outputProtocolFactory.getProtocol(otrans)
+      try:
+        while True:
+          self.processor.process(iprot, oprot)
+      except TTransport.TTransportException, tx:
+        pass
+      except Exception, x:
+        logging.exception(x)
+
+      itrans.close()
+      otrans.close()
+
+class TThreadedServer(TServer):
+
+  """Threaded server that spawns a new thread per each connection."""
+
+  def __init__(self, *args):
+    TServer.__init__(self, *args)
+
+  def serve(self):
+    self.serverTransport.listen()
+    while True:
+      try:
+        client = self.serverTransport.accept()
+        t = threading.Thread(target = self.handle, args=(client,))
+        t.start()
+      except KeyboardInterrupt:
+        raise
+      except Exception, x:
+        logging.exception(x)
+
+  def handle(self, client):
+    itrans = self.inputTransportFactory.getTransport(client)
+    otrans = self.outputTransportFactory.getTransport(client)
+    iprot = self.inputProtocolFactory.getProtocol(itrans)
+    oprot = self.outputProtocolFactory.getProtocol(otrans)
+    try:
+      while True:
+        self.processor.process(iprot, oprot)
+    except TTransport.TTransportException, tx:
+      pass
+    except Exception, x:
+      logging.exception(x)
+
+    itrans.close()
+    otrans.close()
+
+class TThreadPoolServer(TServer):
+
+  """Server with a fixed size pool of threads which service requests."""
+
+  def __init__(self, *args):
+    TServer.__init__(self, *args)
+    self.clients = Queue.Queue()
+    self.threads = 10
+
+  def setNumThreads(self, num):
+    """Set the number of worker threads that should be created"""
+    self.threads = num
+
+  def serveThread(self):
+    """Loop around getting clients from the shared queue and process them."""
+    while True:
+      try:
+        client = self.clients.get()
+        self.serveClient(client)
+      except Exception, x:
+        logging.exception(x)
+
+  def serveClient(self, client):
+    """Process input/output from a client for as long as possible"""
+    itrans = self.inputTransportFactory.getTransport(client)
+    otrans = self.outputTransportFactory.getTransport(client)
+    iprot = self.inputProtocolFactory.getProtocol(itrans)
+    oprot = self.outputProtocolFactory.getProtocol(otrans)
+    try:
+      while True:
+        self.processor.process(iprot, oprot)
+    except TTransport.TTransportException, tx:
+      pass
+    except Exception, x:
+      logging.exception(x)
+
+    itrans.close()
+    otrans.close()
+
+  def serve(self):
+    """Start a fixed number of worker threads and put client into a queue"""
+    for i in range(self.threads):
+      try:
+        t = threading.Thread(target = self.serveThread)
+        t.start()
+      except Exception, x:
+        logging.exception(x)
+
+    # Pump the socket for clients
+    self.serverTransport.listen()
+    while True:
+      try:
+        client = self.serverTransport.accept()
+        self.clients.put(client)
+      except Exception, x:
+        logging.exception(x)
+
+
+class TForkingServer(TServer):
+
+  """A Thrift server that forks a new process for each request"""
+  """
+  This is more scalable than the threaded server as it does not cause
+  GIL contention.
+
+  Note that this has different semantics from the threading server.
+  Specifically, updates to shared variables will no longer be shared.
+  It will also not work on windows.
+
+  This code is heavily inspired by SocketServer.ForkingMixIn in the
+  Python stdlib.
+  """
+
+  def __init__(self, *args):
+    TServer.__init__(self, *args)
+    self.children = []
+
+  def serve(self):
+    def try_close(file):
+      try:
+        file.close()
+      except IOError, e:
+        logging.warning(e, exc_info=True)
+
+
+    self.serverTransport.listen()
+    while True:
+      client = self.serverTransport.accept()
+      try:
+        pid = os.fork()
+
+        if pid: # parent
+          # add before collect, otherwise you race w/ waitpid
+          self.children.append(pid)
+          self.collect_children()
+
+          # Parent must close socket or the connection may not get
+          # closed promptly
+          itrans = self.inputTransportFactory.getTransport(client)
+          otrans = self.outputTransportFactory.getTransport(client)
+          try_close(itrans)
+          try_close(otrans)
+        else:
+          itrans = self.inputTransportFactory.getTransport(client)
+          otrans = self.outputTransportFactory.getTransport(client)
+
+          iprot = self.inputProtocolFactory.getProtocol(itrans)
+          oprot = self.outputProtocolFactory.getProtocol(otrans)
+
+          ecode = 0
+          try:
+            try:
+              while True:
+                self.processor.process(iprot, oprot)
+            except TTransport.TTransportException, tx:
+              pass
+            except Exception, e:
+              logging.exception(e)
+              ecode = 1
+          finally:
+            try_close(itrans)
+            try_close(otrans)
+
+          os._exit(ecode)
+
+      except TTransport.TTransportException, tx:
+        pass
+      except Exception, x:
+        logging.exception(x)
+
+
+  def collect_children(self):
+    while self.children:
+      try:
+        pid, status = os.waitpid(0, os.WNOHANG)
+      except os.error:
+        pid = None
+
+      if pid:
+        self.children.remove(pid)
+      else:
+        break
+
+

Added: hadoop/hive/trunk/service/lib/py/thrift/server/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/server/__init__.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/server/__init__.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/server/__init__.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,7 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+__all__ = ['TServer', 'TNonblockingServer']

Added: hadoop/hive/trunk/service/lib/py/thrift/transport/THttpClient.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/transport/THttpClient.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/transport/THttpClient.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/transport/THttpClient.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,57 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+from TTransport import *
+from cStringIO import StringIO
+
+import httplib
+
+class THttpClient(TTransportBase):
+
+  """Http implementation of TTransport base."""
+
+  def __init__(self, host, port, uri):
+    self.host = host
+    self.port = port
+    self.uri = uri
+    self.__wbuf = StringIO()
+    self.__http = None
+
+  def open(self):
+    self.__http = httplib.HTTP(self.host, self.port)
+
+  def close(self):
+    self.__http.close()
+    self.__http = None
+
+  def isOpen(self):
+    return self.__http != None
+
+  def read(self, sz):
+    return self.__http.file.read(sz)
+
+  def write(self, buf):
+    self.__wbuf.write(buf)
+
+  def flush(self):
+    # Pull data out of buffer
+    data = self.__wbuf.getvalue()
+    self.__wbuf = StringIO()
+
+    # HTTP request
+    self.__http.putrequest('POST', self.uri)
+
+    # Write headers
+    self.__http.putheader('Host', self.host)
+    self.__http.putheader('Content-Type', 'application/x-thrift')
+    self.__http.putheader('Content-Length', str(len(data)))
+    self.__http.endheaders()
+
+    # Write payload
+    self.__http.send(data)
+
+    # Get reply to flush the request
+    self.code, self.message, self.headers = self.__http.getreply()

Added: hadoop/hive/trunk/service/lib/py/thrift/transport/TSocket.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/transport/TSocket.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/transport/TSocket.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/transport/TSocket.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,134 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+from TTransport import *
+import os
+import errno
+import socket
+
+class TSocketBase(TTransportBase):
+  def _resolveAddr(self):
+    if self._unix_socket is not None:
+      return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)]
+    else:
+      return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG)
+
+  def close(self):
+    if self.handle:
+      self.handle.close()
+      self.handle = None
+
+class TSocket(TSocketBase):
+  """Socket implementation of TTransport base."""
+
+  def __init__(self, host='localhost', port=9090, unix_socket=None):
+    """Initialize a TSocket
+
+    @param host(str)  The host to connect to.
+    @param port(int)  The (TCP) port to connect to.
+    @param unix_socket(str)  The filename of a unix socket to connect to.
+                             (host and port will be ignored.)
+    """
+
+    self.host = host
+    self.port = port
+    self.handle = None
+    self._unix_socket = unix_socket
+    self._timeout = None
+
+  def setHandle(self, h):
+    self.handle = h
+
+  def isOpen(self):
+    return self.handle != None
+
+  def setTimeout(self, ms):
+    if ms is None:
+      self._timeout = None
+    else:
+      self._timeout = ms/1000.0
+
+    if (self.handle != None):
+      self.handle.settimeout(self._timeout)
+
+  def open(self):
+    try:
+      res0 = self._resolveAddr()
+      for res in res0:
+        self.handle = socket.socket(res[0], res[1])
+        self.handle.settimeout(self._timeout)
+        try:
+          self.handle.connect(res[4])
+        except socket.error, e:
+          if res is not res0[-1]:
+            continue
+          else:
+            raise e
+        break
+    except socket.error, e:
+      if self._unix_socket:
+        message = 'Could not connect to socket %s' % self._unix_socket
+      else:
+        message = 'Could not connect to %s:%d' % (self.host, self.port)
+      raise TTransportException(TTransportException.NOT_OPEN, message)
+
+  def read(self, sz):
+    buff = self.handle.recv(sz)
+    if len(buff) == 0:
+      raise TTransportException('TSocket read 0 bytes')
+    return buff
+
+  def write(self, buff):
+    sent = 0
+    have = len(buff)
+    while sent < have:
+      plus = self.handle.send(buff)
+      if plus == 0:
+        raise TTransportException('TSocket sent 0 bytes')
+      sent += plus
+      buff = buff[plus:]
+
+  def flush(self):
+    pass
+
+class TServerSocket(TServerTransportBase, TSocketBase):
+  """Socket implementation of TServerTransport base."""
+
+  def __init__(self, port=9090, unix_socket=None):
+    self.host = None
+    self.port = port
+    self._unix_socket = unix_socket
+    self.handle = None
+
+  def listen(self):
+    res0 = self._resolveAddr()
+    for res in res0:
+      if res[0] is socket.AF_INET6 or res is res0[-1]:
+        break
+
+    # We need remove the old unix socket if the file exists and
+    # nobody is listening on it.
+    if self._unix_socket:
+      tmp = socket.socket(res[0], res[1])
+      try:
+        tmp.connect(res[4])
+      except socket.error, err:
+        eno, message = err.args
+        if eno == errno.ECONNREFUSED:
+          os.unlink(res[4])
+
+    self.handle = socket.socket(res[0], res[1])
+    self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    if hasattr(self.handle, 'set_timeout'):
+      self.handle.set_timeout(None)
+    self.handle.bind(res[4])
+    self.handle.listen(128)
+
+  def accept(self):
+    client, addr = self.handle.accept()
+    result = TSocket()
+    result.setHandle(client)
+    return result



Mime
View raw message