avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ham...@apache.org
Subject svn commit: r908770 - in /hadoop/avro/trunk: CHANGES.txt build.sh lang/py/build.xml lang/py/test/gen_interop_data.py lang/py/test/test_datafile_interop.py
Date Thu, 11 Feb 2010 00:07:41 GMT
Author: hammer
Date: Thu Feb 11 00:07:31 2010
New Revision: 908770

URL: http://svn.apache.org/viewvc?rev=908770&view=rev
Log:
AVRO-399: Make data file interop tests work with
          the Python implementation (hammer)


Added:
    hadoop/avro/trunk/lang/py/test/gen_interop_data.py
    hadoop/avro/trunk/lang/py/test/test_datafile_interop.py
Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/build.sh
    hadoop/avro/trunk/lang/py/build.xml

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=908770&r1=908769&r2=908770&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Thu Feb 11 00:07:31 2010
@@ -351,6 +351,8 @@
     AVRO-411, AVRO-413. Add Ruby data file interop tests. (Jeff Hodges
     via cutting)
 
+    AVRO-399. Make data file interop tests work with the Python implementation (hammer)
+
   BUG FIXES
  
     AVRO-176. Safeguard against bad istreams before reading. (sbanacho)

Modified: hadoop/avro/trunk/build.sh
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/build.sh?rev=908770&r1=908769&r2=908770&view=diff
==============================================================================
--- hadoop/avro/trunk/build.sh (original)
+++ hadoop/avro/trunk/build.sh Thu Feb 11 00:07:31 2010
@@ -48,14 +48,14 @@
 
 	# create interop test data
 	(cd lang/java; ant interop-data-generate)
-	#(cd lang/py; ant interop-data-generate)
+	(cd lang/py; ant interop-data-generate)
 	(cd lang/c; ./build.sh interop-data-generate)
 	#(cd lang/c++; make interop-data-generate)
 	(cd lang/ruby; rake generate_interop)
 
 	# run interop data tests
 	(cd lang/java; ant interop-data-test)
-	#(cd lang/py; ant interop-data-test)
+	(cd lang/py; ant interop-data-test)
 	(cd lang/c; ./build.sh interop-data-test)
 	#(cd lang/c++; make interop-data-test)
 	(cd lang/ruby; rake interop)

Modified: hadoop/avro/trunk/lang/py/build.xml
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/py/build.xml?rev=908770&r1=908769&r2=908770&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/py/build.xml (original)
+++ hadoop/avro/trunk/lang/py/build.xml Thu Feb 11 00:07:31 2010
@@ -23,6 +23,7 @@
 
   <property name="share.dir" value="${basedir}/../../share"/>
   <property name="dist.dir" value="${basedir}/../../dist"/>
+  <property name="top.build" value="${basedir}/../../build"/>
 
   <loadresource property="version">
     <file file="${share.dir}/VERSION.txt"/>
@@ -56,10 +57,32 @@
     <py-test python="python" pythonpathref="test.path" >
       <fileset dir="${basedir}/test">
         <include name="test_*.py"/>
+        <exclude name="test_datafile_interop.py"/>
       </fileset>
     </py-test>
   </target>
 
+  <target name="interop-data-test" description="Run python interop data tests">
+    <taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask"
+	     classpathref="java.classpath"/>
+    <py-test python="python" pythonpathref="test.path" >
+      <fileset dir="${basedir}/test">
+        <include name="test_datafile_interop.py"/>
+      </fileset>
+    </py-test>
+  </target>
+
+  <target name="interop-data-generate"
+          description="Generate Python interop data files.">
+    <mkdir dir="${top.build}/interop/data"/>
+    <exec executable="python">
+      <env key="PYTHONPATH" value="$PYTHONPATH:${basedir}/src"/>
+      <arg value="${basedir}/test/gen_interop_data.py"/>
+      <arg value="${share.dir}/test/schemas/interop.avsc"/>
+      <arg value="${top.build}/interop/data/py.avro"/>
+    </exec>
+  </target>
+
   <target name="doc" description="Generate python api docs">
     <taskdef name="py-doc" classname="org.pyant.tasks.PythonDocTask"
 	     classpathref="java.classpath"/>

Added: hadoop/avro/trunk/lang/py/test/gen_interop_data.py
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/py/test/gen_interop_data.py?rev=908770&view=auto
==============================================================================
--- hadoop/avro/trunk/lang/py/test/gen_interop_data.py (added)
+++ hadoop/avro/trunk/lang/py/test/gen_interop_data.py Thu Feb 11 00:07:31 2010
@@ -0,0 +1,30 @@
+import sys
+from avro import schema
+from avro import io
+from avro import datafile
+
+DATUM = {
+  'intField': 12,
+  'longField': 15234324L,
+  'stringField': unicode('hey'),
+  'boolField': True,
+  'floatField': 1234.0,
+  'doubleField': -1234.0,
+  'bytesField': '12312adf',
+  'nullField': None,
+  'arrayField': [5.0, 0.0, 12.0],
+  'mapField': {'a': {'label': 'a'}, 'bee': {'label': 'cee'}},
+  'unionField': 12.0,
+  'enumField': 'C',
+  'fixedField': '1019181716151413',
+  'recordField': {'label': 'blah', 'children': [{'label': 'inner', 'children': []}]},
+}
+
+if __name__ == "__main__":
+  interop_schema = schema.parse(open(sys.argv[1], 'r').read())
+  writer = open(sys.argv[2], 'wb')
+  datum_writer = io.DatumWriter()
+  # NB: not using compression
+  dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
+  dfw.append(DATUM)
+  dfw.close()

Added: hadoop/avro/trunk/lang/py/test/test_datafile_interop.py
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/py/test/test_datafile_interop.py?rev=908770&view=auto
==============================================================================
--- hadoop/avro/trunk/lang/py/test/test_datafile_interop.py (added)
+++ hadoop/avro/trunk/lang/py/test/test_datafile_interop.py Thu Feb 11 00:07:31 2010
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+from avro import io
+from avro import datafile
+
+INTEROP_DATA_DIR = os.path.join(os.path.dirname(__file__), '../../../build/interop/data')
+
+class TestDataFileInterop(unittest.TestCase):
+  def test_interop(self):
+    print ''
+    print 'TEST INTEROP'
+    print '============'
+    print ''
+    for f in os.listdir(INTEROP_DATA_DIR):
+      print 'READING %s' % f
+      print ''
+
+      # read data in binary from file
+      reader = open(os.path.join(INTEROP_DATA_DIR, f), 'rb')
+      datum_reader = io.DatumReader()
+      dfr = datafile.DataFileReader(reader, datum_reader)
+      for datum in dfr:
+        assert datum is not None
+
+if __name__ == '__main__':
+  unittest.main()



Mime
View raw message