avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marti...@apache.org
Subject svn commit: r1467548 - in /avro/trunk: CHANGES.txt build.sh lang/java/avro/pom.xml lang/java/avro/src/test/java/org/apache/avro/RandomData.java lang/php/test/InterOpTest.php lang/ruby/interop/test_interop.rb
Date Sat, 13 Apr 2013 04:43:26 GMT
Author: martinkl
Date: Sat Apr 13 04:43:26 2013
New Revision: 1467548

URL: http://svn.apache.org/r1467548
Log:
AVRO-1287. Add data file with deflate codec to the interoperability test suite.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/build.sh
    avro/trunk/lang/java/avro/pom.xml
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java
    avro/trunk/lang/php/test/InterOpTest.php
    avro/trunk/lang/ruby/interop/test_interop.rb

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Sat Apr 13 04:43:26 2013
@@ -22,6 +22,9 @@ Trunk (not yet released)
     AVRO-1259. Java: Improve Trevni's encoding of sparse columns.
     (cutting)
 
+    AVRO-1287. Add data file with deflate codec to the interoperability
+    test suite. (martinkl)
+
   BUG FIXES
 
     AVRO-1266. Java: Fix mapred.AvroMultipleOutputs to support multiple

Modified: avro/trunk/build.sh
URL: http://svn.apache.org/viewvc/avro/trunk/build.sh?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/build.sh (original)
+++ avro/trunk/build.sh Sat Apr 13 04:43:26 2013
@@ -51,7 +51,7 @@ case "$target" in
 
 	# create interop test data
         mkdir -p build/interop/data
-	(cd lang/java/avro; mvn exec:java -P interop-data-generate)
+	(cd lang/java/avro; mvn -P interop-data-generate generate-resources)
 	(cd lang/py; ant interop-data-generate)
 	(cd lang/c; ./build.sh interop-data-generate)
 	#(cd lang/c++; make interop-data-generate)

Modified: avro/trunk/lang/java/avro/pom.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/pom.xml?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/pom.xml (original)
+++ avro/trunk/lang/java/avro/pom.xml Sat Apr 13 04:43:26 2013
@@ -78,25 +78,49 @@
   <profiles>
     <profile>
       <id>interop-data-generate</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
       <build>
-        <pluginManagement>
-          <plugins>
-            <plugin>
-              <groupId>org.codehaus.mojo</groupId>
-              <artifactId>exec-maven-plugin</artifactId>
-              <!-- default configuration is to generate random data for interop tests
-->
-              <configuration>
-                <mainClass>org.apache.avro.RandomData</mainClass>
-                <classpathScope>test</classpathScope>
-                <arguments>
-                  <argument>../../../share/test/schemas/interop.avsc</argument>
-                  <argument>../../../build/interop/data/java.avro</argument>
-                  <argument>100</argument>
-                </arguments>
-              </configuration>
-            </plugin>
-          </plugins>
-        </pluginManagement>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <!-- Generate random data for interop tests, using null codec -->
+              <execution>
+                <id>interop-generate-null-codec</id>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <mainClass>org.apache.avro.RandomData</mainClass>
+                  <classpathScope>test</classpathScope>
+                  <arguments>
+                    <argument>../../../share/test/schemas/interop.avsc</argument>
+                    <argument>../../../build/interop/data/java.avro</argument>
+                    <argument>100</argument>
+                  </arguments>
+                </configuration>
+                <goals><goal>java</goal></goals>
+              </execution>
+              <!-- Generate random data for interop tests, using deflate codec -->
+              <execution>
+                <id>interop-generate-deflate-codec</id>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <mainClass>org.apache.avro.RandomData</mainClass>
+                  <classpathScope>test</classpathScope>
+                  <arguments>
+                    <argument>../../../share/test/schemas/interop.avsc</argument>
+                    <argument>../../../build/interop/data/java_deflate.avro</argument>
+                    <argument>100</argument>
+                    <argument>deflate</argument>
+                  </arguments>
+                </configuration>
+                <goals><goal>java</goal></goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
       </build>
     </profile>
   </profiles>

Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/RandomData.java Sat Apr 13 04:43:26
2013
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.avro.file.CodecFactory;
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.generic.GenericArray;
 import org.apache.avro.generic.GenericData;
@@ -123,14 +124,15 @@ public class RandomData implements Itera
   }
 
   public static void main(String[] args) throws Exception {
-    if(args.length != 3) {
-      System.out.println("Usage: RandomData <schemafile> <outputfile> <count>");
+    if(args.length < 3 || args.length > 4) {
+      System.out.println("Usage: RandomData <schemafile> <outputfile> <count>
[codec]");
       System.exit(-1);
     }
     Schema sch = Schema.parse(new File(args[0]));
     DataFileWriter<Object> writer =
-      new DataFileWriter<Object>(new GenericDatumWriter<Object>())
-      .create(sch, new File(args[1]));
+      new DataFileWriter<Object>(new GenericDatumWriter<Object>());
+    writer.setCodec(CodecFactory.fromString(args.length >= 4 ? args[3] : "null"));
+    writer.create(sch, new File(args[1]));
     try {
       for (Object datum : new RandomData(sch, Integer.parseInt(args[2]))) {
         writer.append(datum);

Modified: avro/trunk/lang/php/test/InterOpTest.php
URL: http://svn.apache.org/viewvc/avro/trunk/lang/php/test/InterOpTest.php?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/php/test/InterOpTest.php (original)
+++ avro/trunk/lang/php/test/InterOpTest.php Sat Apr 13 04:43:26 2013
@@ -37,8 +37,13 @@ class InterOpTest extends PHPUnit_Framew
     $data_files = array();
     if (!($dh = opendir($data_dir)))
       die("Could not open data dir '$data_dir'\n");
+
+    /* TODO This currently only tries to read files of the form 'language.avro',
+     * but not 'language_deflate.avro' as the PHP implementation is not yet
+     * able to read deflate data files. When deflate support is added, change
+     * this to match *.avro. */
     while ($file = readdir($dh))
-      if (0 < preg_match('/.*\.avro$/', $file))
+      if (0 < preg_match('/^[a-z]+\.avro$/', $file))
         $data_files []= join(DIRECTORY_SEPARATOR, array($data_dir, $file));
     closedir($dh);
 

Modified: avro/trunk/lang/ruby/interop/test_interop.rb
URL: http://svn.apache.org/viewvc/avro/trunk/lang/ruby/interop/test_interop.rb?rev=1467548&r1=1467547&r2=1467548&view=diff
==============================================================================
--- avro/trunk/lang/ruby/interop/test_interop.rb (original)
+++ avro/trunk/lang/ruby/interop/test_interop.rb Sat Apr 13 04:43:26 2013
@@ -23,7 +23,10 @@ class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))
   SHARE = HERE + '/../../../share'
   SCHEMAS = SHARE + '/test/schemas'
-  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|  
+
+  # TODO this currently ignores data files with deflate codec.
+  # Remove this restriction when deflate support is added.
+  Dir[HERE + '/../../../build/interop/data/*'].reject{|fn| fn =~ /_deflate/ }.each do |fn|
 
     define_method("test_read_#{File.basename(fn, 'avro')}") do
       projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
 



Mime
View raw message