avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ph...@apache.org
Subject svn commit: r987543 - in /avro/trunk: CHANGES.txt lang/py/build.xml lang/py/src/avro/datafile.py lang/py/src/avro/io.py lang/py/src/avro/protocol.py lang/py/test/test_protocol.py lang/py/test/test_schema.py
Date Fri, 20 Aug 2010 15:22:12 GMT
Author: philz
Date: Fri Aug 20 15:22:10 2010
New Revision: 987543

URL: http://svn.apache.org/viewvc?rev=987543&view=rev
Log:
AVRO-618. Avro doesn't work with python 2.4

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/py/build.xml
    avro/trunk/lang/py/src/avro/datafile.py
    avro/trunk/lang/py/src/avro/io.py
    avro/trunk/lang/py/src/avro/protocol.py
    avro/trunk/lang/py/test/test_protocol.py
    avro/trunk/lang/py/test/test_schema.py

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Aug 20 15:22:10 2010
@@ -129,6 +129,8 @@ Avro 1.4.0 (unreleased)
 
   BUG FIXES
 
+    AVRO-618. Avro doesn't work with python 2.4 (philz)
+
     AVRO-502. Memory leak from parsing JSON schema.
     (Robert G. Jakabosky via massie)
 

Modified: avro/trunk/lang/py/build.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/build.xml?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/build.xml (original)
+++ avro/trunk/lang/py/build.xml Fri Aug 20 15:22:10 2010
@@ -28,6 +28,8 @@
   <property name="top.build" value="${basedir}/../../build"/>
   <property name="interop.data.dir" value="${top.build}/interop/data"/>
 
+  <property name="python" value="python"/>
+
   <!-- Python implementation directories -->
   <property name="build.dir" value="${basedir}/build"/>
   <property name="src.dir" value="${basedir}/src"/>
@@ -111,7 +113,7 @@
           depends="build">
     <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
 	     classpathref="java.classpath"/>
-    <py-test python="python" pythonpathref="test.path" >
+    <py-test python="${python}" pythonpathref="test.path" >
       <fileset dir="${build.dir}/test">
         <include name="test_*.py"/>
         <exclude name="test_datafile_interop.py"/>
@@ -125,7 +127,7 @@
           depends="build">
     <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
 	     classpathref="java.classpath"/>
-    <py-test python="python" pythonpathref="test.path" >
+    <py-test python="${python}" pythonpathref="test.path" >
       <fileset dir="${build.dir}/test">
         <include name="test_datafile_interop.py"/>
       </fileset>
@@ -137,7 +139,7 @@
           description="Generate Python interop data files."
           depends="build">
     <mkdir dir="${interop.data.dir}"/>
-    <exec executable="python">
+    <exec executable="${python}">
       <env key="PYTHONPATH" value="$PYTHONPATH:${build.dir}/src"/>
       <arg value="${build.dir}/test/gen_interop_data.py"/>
       <arg value="${share.dir}/test/schemas/interop.avsc"/>
@@ -150,7 +152,7 @@
           description="Build source distribution"
           depends="build">
     <mkdir dir="${dist.dir}"/>
-    <exec executable="python" failonerror="true">
+    <exec executable="${python}" failonerror="true">
       <arg value="${build.dir}/setup.py"/>
       <arg value="sdist"/>
       <arg value="--dist-dir=${dist.dir}"/>

Modified: avro/trunk/lang/py/src/avro/datafile.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/datafile.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/datafile.py (original)
+++ avro/trunk/lang/py/src/avro/datafile.py Fri Aug 20 15:22:10 2010
@@ -17,7 +17,6 @@
 Read/Write Avro File Object Containers.
 """
 import zlib
-import uuid
 try:
   from cStringIO import StringIO
 except ImportError:
@@ -65,7 +64,7 @@ class DataFileException(schema.AvroExcep
 class DataFileWriter(object):
   @staticmethod
   def generate_sync_marker():
-    return uuid.uuid4().bytes
+    return generate_sixteen_random_bytes()
 
   # TODO(hammer): make 'encoder' a metadata property
   def __init__(self, writer, datum_writer, writers_schema=None, codec='null'):
@@ -322,3 +321,11 @@ class DataFileReader(object):
   def close(self):
     """Close this reader."""
     self.reader.close()
+
+def generate_sixteen_random_bytes():
+  try:
+    import os
+    return os.urandom(16)
+  except:
+    import random
+    return [ chr(random.randrange(256)) for i in range(16) ]

Modified: avro/trunk/lang/py/src/avro/io.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/io.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/io.py (original)
+++ avro/trunk/lang/py/src/avro/io.py Fri Aug 20 15:22:10 2010
@@ -38,6 +38,7 @@ uses the following mapping:
 """
 import struct
 from avro import schema
+import sys
 
 #
 # Constants
@@ -49,10 +50,22 @@ LONG_MIN_VALUE = -(1 << 63)
 LONG_MAX_VALUE = (1 << 63) - 1
 
 # TODO(hammer): shouldn't ! be < for little-endian (according to spec?)
-STRUCT_INT = struct.Struct('!I')     # big-endian unsigned int
-STRUCT_LONG = struct.Struct('!Q')    # big-endian unsigned long long
-STRUCT_FLOAT = struct.Struct('!f')   # big-endian float
-STRUCT_DOUBLE = struct.Struct('!d')  # big-endian double
+if sys.version_info >= (2, 5, 0):
+  struct_class = struct.Struct
+else:
+  class SimpleStruct(object):
+    def __init__(self, format):
+      self.format = format
+    def pack(self, *args):
+      return struct.pack(self.format, *args)
+    def unpack(self, *args):
+      return struct.unpack(self.format, *args)
+  struct_class = SimpleStruct
+
+STRUCT_INT = struct_class('!I')     # big-endian unsigned int
+STRUCT_LONG = struct_class('!Q')    # big-endian unsigned long long
+STRUCT_FLOAT = struct_class('!f')   # big-endian float
+STRUCT_DOUBLE = struct_class('!d')  # big-endian double
 
 #
 # Exceptions

Modified: avro/trunk/lang/py/src/avro/protocol.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/protocol.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/src/avro/protocol.py (original)
+++ avro/trunk/lang/py/src/avro/protocol.py Fri Aug 20 15:22:10 2010
@@ -17,9 +17,9 @@
 Protocol implementation.
 """
 try:
-  import hashlib
+  from hashlib import md5
 except ImportError:
-  import md5
+  from md5 import md5
 try:
   import json
 except ImportError:
@@ -100,10 +100,7 @@ class Protocol(object):
       self.set_prop('types', self._parse_types(types, type_names))
     if messages is not None:
       self.set_prop('messages', self._parse_messages(messages, type_names))
-    if hashlib:
-      self._md5 = hashlib.md5(str(self)).digest()
-    else:
-      self._md5 = md5.new(str(self)).digest()
+    self._md5 = md5(str(self)).digest()
 
   # read-only properties
   name = property(lambda self: self.get_prop('name'))

Modified: avro/trunk/lang/py/test/test_protocol.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_protocol.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/test/test_protocol.py (original)
+++ avro/trunk/lang/py/test/test_protocol.py Fri Aug 20 15:22:10 2010
@@ -318,12 +318,13 @@ class TestProtocol(unittest.TestCase):
     num_correct = 0
     for example in EXAMPLES:
       try:
-        protocol.parse(example.protocol_string)
-        if example.valid: num_correct += 1
-        debug_msg = "%s: PARSE SUCCESS" % example.name
-      except:
-        if not example.valid: num_correct += 1
-        debug_msg = "%s: PARSE FAILURE" % example.name
+        try:
+          protocol.parse(example.protocol_string)
+          if example.valid: num_correct += 1
+          debug_msg = "%s: PARSE SUCCESS" % example.name
+        except:
+          if not example.valid: num_correct += 1
+          debug_msg = "%s: PARSE FAILURE" % example.name
       finally:
         print debug_msg
 
@@ -345,11 +346,12 @@ class TestProtocol(unittest.TestCase):
     for example in VALID_EXAMPLES:
       protocol_data = protocol.parse(example.protocol_string)
       try:
-        protocol.parse(str(protocol_data))
-        debug_msg = "%s: STRING CAST SUCCESS" % example.name
-        num_correct += 1
-      except:
-        debug_msg = "%s: STRING CAST FAILURE" % example.name
+        try:
+          protocol.parse(str(protocol_data))
+          debug_msg = "%s: STRING CAST SUCCESS" % example.name
+          num_correct += 1
+        except:
+          debug_msg = "%s: STRING CAST FAILURE" % example.name
       finally:
         print debug_msg
 
@@ -372,16 +374,17 @@ class TestProtocol(unittest.TestCase):
     num_correct = 0
     for example in VALID_EXAMPLES:
       try:
-        original_protocol = protocol.parse(example.protocol_string)
-        round_trip_protocol = protocol.parse(str(original_protocol))
-
-        if original_protocol == round_trip_protocol:
-          num_correct += 1
-          debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
-        else:       
+        try:
+          original_protocol = protocol.parse(example.protocol_string)
+          round_trip_protocol = protocol.parse(str(original_protocol))
+
+          if original_protocol == round_trip_protocol:
+            num_correct += 1
+            debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
+          else:       
+            debug_msg = "%s: ROUND TRIP FAILURE" % example.name
+        except:
           debug_msg = "%s: ROUND TRIP FAILURE" % example.name
-      except:
-        debug_msg = "%s: ROUND TRIP FAILURE" % example.name
       finally:
         print debug_msg
 

Modified: avro/trunk/lang/py/test/test_schema.py
URL: http://svn.apache.org/viewvc/avro/trunk/lang/py/test/test_schema.py?rev=987543&r1=987542&r2=987543&view=diff
==============================================================================
--- avro/trunk/lang/py/test/test_schema.py (original)
+++ avro/trunk/lang/py/test/test_schema.py Fri Aug 20 15:22:10 2010
@@ -275,12 +275,19 @@ class TestSchema(unittest.TestCase):
     correct = 0
     for example in EXAMPLES:
       try:
-        schema.parse(example.schema_string)
-        if example.valid: correct += 1
-        debug_msg = "%s: PARSE SUCCESS" % example.name
-      except:
-        if not example.valid: correct += 1
-        debug_msg = "%s: PARSE FAILURE" % example.name
+        try:
+          schema.parse(example.schema_string)
+          if example.valid:
+            correct += 1
+          else:
+            self.fail("Invalid schema was parsed: " + example.schema_string)
+          debug_msg = "%s: PARSE SUCCESS" % example.name
+        except:
+          if not example.valid: 
+            correct += 1
+          else:
+            self.fail("Valid schema failed to parse: " + example.schema_string)
+          debug_msg = "%s: PARSE FAILURE" % example.name
       finally:
         print debug_msg
 
@@ -298,11 +305,12 @@ class TestSchema(unittest.TestCase):
     for example in VALID_EXAMPLES:
       schema_data = schema.parse(example.schema_string)
       try:
-        schema.parse(str(schema_data))
-        debug_msg = "%s: STRING CAST SUCCESS" % example.name
-        correct += 1
-      except:
-        debug_msg = "%s: STRING CAST FAILURE" % example.name
+        try:
+          schema.parse(str(schema_data))
+          debug_msg = "%s: STRING CAST SUCCESS" % example.name
+          correct += 1
+        except:
+          debug_msg = "%s: STRING CAST FAILURE" % example.name
       finally:
         print debug_msg
 
@@ -321,15 +329,16 @@ class TestSchema(unittest.TestCase):
     correct = 0
     for example in VALID_EXAMPLES:
       try:
-        original_schema = schema.parse(example.schema_string)
-        round_trip_schema = schema.parse(str(original_schema))
-        if original_schema == round_trip_schema:
-          correct += 1
-          debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
-        else:       
+        try:
+          original_schema = schema.parse(example.schema_string)
+          round_trip_schema = schema.parse(str(original_schema))
+          if original_schema == round_trip_schema:
+            correct += 1
+            debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
+          else:       
+            debug_msg = "%s: ROUND TRIP FAILURE" % example.name
+        except:
           debug_msg = "%s: ROUND TRIP FAILURE" % example.name
-      except:
-        debug_msg = "%s: ROUND TRIP FAILURE" % example.name
       finally:
         print debug_msg
 



Mime
View raw message