avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1409542 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/generic/ lang/java/avro/src/main/java/org/apache/avro/util/ lang/java/avro/src/test/java/org/apache/avro/ lang/java/avro/src/test/java/org/apache/avro/generic/ lang/...
Date Wed, 14 Nov 2012 23:21:31 GMT
Author: cutting
Date: Wed Nov 14 23:21:29 2012
New Revision: 1409542

URL: http://svn.apache.org/viewvc?rev=1409542&view=rev
Log:
AVRO-1199. Java: Fix SortedKeyValueFile to copy the key used to compare against next.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
    avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
    avro/trunk/lang/java/tools/src/test/compiler/output/Player.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Nov 14 23:21:29 2012
@@ -40,6 +40,9 @@ Trunk (not yet released)
     AVRO-1200. DatumWriter can write malformed data if collection is
     modified during write. (tomwhite)
 
+    AVRO-1199. Java: Fix SortedKeyValueFile to copy the key used to
+    compare against next.  Also improve GenericData#deepCopy() to be
+    generic, so that its return type matches its parameter type. (cutting)
 
 Avro 1.7.2 (20 October 2012)
 

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java Wed Nov
14 23:21:29 2012
@@ -767,7 +767,7 @@ public class GenericData {
    * @return a deep copy of the given value.
    */
   @SuppressWarnings({ "rawtypes", "unchecked" })
-  public Object deepCopy(Schema schema, Object value) {
+  public <T> T deepCopy(Schema schema, T value) {
     if (value == null) {
       return null;
     }
@@ -779,9 +779,9 @@ public class GenericData {
         for (Object obj : arrayValue) {
           arrayCopy.add(deepCopy(schema.getElementType(), obj));
         }
-        return arrayCopy;
+        return (T)arrayCopy;
       case BOOLEAN:
-        return new Boolean((Boolean)value);
+        return (T)new Boolean((Boolean)value);
       case BYTES:
         ByteBuffer byteBufferValue = (ByteBuffer) value;
         int start = byteBufferValue.position();
@@ -789,20 +789,20 @@ public class GenericData {
         byte[] bytesCopy = new byte[length];
         byteBufferValue.get(bytesCopy, 0, length);
         byteBufferValue.position(start);
-        return ByteBuffer.wrap(bytesCopy, 0, length);
+        return (T)ByteBuffer.wrap(bytesCopy, 0, length);
       case DOUBLE:
-        return new Double((Double)value);
+        return (T)new Double((Double)value);
       case ENUM:
         // Enums are immutable; shallow copy will suffice
         return value;
       case FIXED:
-        return createFixed(null, ((GenericFixed) value).bytes(), schema);
+        return (T)createFixed(null, ((GenericFixed) value).bytes(), schema);
       case FLOAT:
-        return new Float((Float)value);
+        return (T)new Float((Float)value);
       case INT:
-        return new Integer((Integer)value);
+        return (T)new Integer((Integer)value);
       case LONG:
-        return new Long((Long)value);
+        return (T)new Long((Long)value);
       case MAP:
         Map<CharSequence, Object> mapValue = (Map) value;
         Map<CharSequence, Object> mapCopy = 
@@ -811,7 +811,7 @@ public class GenericData {
           mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())),
               deepCopy(schema.getValueType(), entry.getValue()));
         }
-        return mapCopy;
+        return (T)mapCopy;
       case NULL:
         return null;
       case RECORD:
@@ -821,11 +821,11 @@ public class GenericData {
           recordCopy.put(field.pos(), 
               deepCopy(field.schema(), recordValue.get(field.pos())));
         }
-        return recordCopy;
+        return (T)recordCopy;
       case STRING:
         // Strings are immutable
         if (value instanceof String) {
-          return value;
+          return (T)value;
         }
         
         // Some CharSequence subclasses are mutable, so we still need to make 
@@ -833,9 +833,9 @@ public class GenericData {
         else if (value instanceof Utf8) {
           // Utf8 copy constructor is more efficient than converting 
           // to string and then back to Utf8
-          return new Utf8((Utf8)value);
+          return (T)new Utf8((Utf8)value);
         }
-        return new Utf8(value.toString());
+        return (T)new Utf8(value.toString());
       case UNION:
         return deepCopy(
             schema.getTypes().get(resolveUnion(schema, value)), value);

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/util/Utf8.java Wed Nov 14 23:21:29
2012
@@ -83,6 +83,14 @@ public class Utf8 implements Comparable<
     return this;
   }
 
+  /** Set to the contents of a String. */
+  public Utf8 set(String string) {
+    this.bytes = getBytesFor(string);
+    this.length = bytes.length;
+    this.string = string;
+    return this;
+  }
+
   @Override
   public String toString() {
     if (this.length == 0) return "";

Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/FooBarSpecificRecord.java Wed
Nov 14 23:21:29 2012
@@ -96,11 +96,11 @@ public class FooBarSpecificRecord extend
     private Builder(org.apache.avro.FooBarSpecificRecord other) {
             super(org.apache.avro.FooBarSpecificRecord.SCHEMA$);
       if (isValidValue(fields()[0], other.id)) {
-        this.id = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.id);
+        this.id = data().deepCopy(fields()[0].schema(), other.id);
         fieldSetFlags()[0] = true;
       }
       if (isValidValue(fields()[1], other.relatedids)) {
-        this.relatedids = (java.util.List<java.lang.Integer>) data().deepCopy(fields()[1].schema(),
other.relatedids);
+        this.relatedids = data().deepCopy(fields()[1].schema(), other.relatedids);
         fieldSetFlags()[1] = true;
       }
     }

Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericData.java Wed
Nov 14 23:21:29 2012
@@ -374,8 +374,7 @@ public class TestGenericData {
     GenericRecord record = new GenericData.Record(schema);
     record.put(byte_field.name(), buffer);
     
-    GenericRecord copy =
-      (GenericRecord) GenericData.get().deepCopy(schema, record);
+    GenericRecord copy = GenericData.get().deepCopy(schema, record);
     ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name());
 
     assertEquals(buffer, buffer_copy);

Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
(original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
Wed Nov 14 23:21:29 2012
@@ -159,7 +159,7 @@ public class ${this.mangle($schema.getNa
       super(#if ($schema.getNamespace())$schema.getNamespace().#end${this.mangle($schema.getName())}.SCHEMA$)#end;
 #foreach ($field in $schema.getFields())
       if (isValidValue(fields()[$field.pos()], other.${this.mangle($field.name(), $schema.isError())}))
{
-        this.${this.mangle($field.name(), $schema.isError())} = (${this.javaType($field.schema())})
data().deepCopy(fields()[$field.pos()].schema(), other.${this.mangle($field.name(), $schema.isError())});
+        this.${this.mangle($field.name(), $schema.isError())} = data().deepCopy(fields()[$field.pos()].schema(),
other.${this.mangle($field.name(), $schema.isError())});
         fieldSetFlags()[$field.pos()] = true;
       }
 #end

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/file/SortedKeyValueFile.java
Wed Nov 14 23:21:29 2012
@@ -536,7 +536,7 @@ public class SortedKeyValueFile {
         throw new IllegalArgumentException("Records must be inserted in sorted key order."
             + " Attempted to insert key " + key + " after " + mPreviousKey + ".");
       }
-      mPreviousKey = key;
+      mPreviousKey = GenericData.get().deepCopy(mKeySchema, key);
 
       // Construct the data record.
       AvroKeyValue<K, V> dataRecord

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/file/TestSortedKeyValueFile.java
Wed Nov 14 23:21:29 2012
@@ -32,6 +32,7 @@ import org.apache.avro.hadoop.io.AvroKey
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.file.FileReader;
 import org.apache.avro.file.DataFileReader;
+import org.apache.avro.util.Utf8;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.junit.Rule;
@@ -61,9 +62,11 @@ public class TestSortedKeyValueFile {
     SortedKeyValueFile.Writer<CharSequence, CharSequence> writer
         = new SortedKeyValueFile.Writer<CharSequence, CharSequence>(options);
 
+    Utf8 key = new Utf8();                        // re-use key, to test copied
+
     try {
-      writer.append("banana", "Banana");
-      writer.append("apple", "Apple");  // Ruh, roh!
+      writer.append(key.set("banana"), "Banana");
+      writer.append(key.set("apple"), "Apple");  // Ruh, roh!
     } finally {
       writer.close();
     }

Modified: avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output/Player.java?rev=1409542&r1=1409541&r2=1409542&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output/Player.java (original)
+++ avro/trunk/lang/java/tools/src/test/compiler/output/Player.java Wed Nov 14 23:21:29 2012
@@ -150,19 +150,19 @@ public class Player extends org.apache.a
     private Builder(avro.examples.baseball.Player other) {
             super(avro.examples.baseball.Player.SCHEMA$);
       if (isValidValue(fields()[0], other.number)) {
-        this.number = (java.lang.Integer) data().deepCopy(fields()[0].schema(), other.number);
+        this.number = data().deepCopy(fields()[0].schema(), other.number);
         fieldSetFlags()[0] = true;
       }
       if (isValidValue(fields()[1], other.first_name)) {
-        this.first_name = (java.lang.CharSequence) data().deepCopy(fields()[1].schema(),
other.first_name);
+        this.first_name = data().deepCopy(fields()[1].schema(), other.first_name);
         fieldSetFlags()[1] = true;
       }
       if (isValidValue(fields()[2], other.last_name)) {
-        this.last_name = (java.lang.CharSequence) data().deepCopy(fields()[2].schema(), other.last_name);
+        this.last_name = data().deepCopy(fields()[2].schema(), other.last_name);
         fieldSetFlags()[2] = true;
       }
       if (isValidValue(fields()[3], other.position)) {
-        this.position = (java.util.List<avro.examples.baseball.Position>) data().deepCopy(fields()[3].schema(),
other.position);
+        this.position = data().deepCopy(fields()[3].schema(), other.position);
         fieldSetFlags()[3] = true;
       }
     }



Mime
View raw message