Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 54167 invoked from network); 10 Apr 2008 04:34:32 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 10 Apr 2008 04:34:32 -0000 Received: (qmail 4139 invoked by uid 500); 10 Apr 2008 04:34:32 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 4104 invoked by uid 500); 10 Apr 2008 04:34:32 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 4095 invoked by uid 99); 10 Apr 2008 04:34:32 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Apr 2008 21:34:32 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Apr 2008 04:33:49 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 7DE5D1A9838; Wed, 9 Apr 2008 21:34:08 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r646643 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/io/serializer/ src/test/org/apache/hadoop/io/ src/test/org/apache/hadoop/io/serializer/ Date: Thu, 10 Apr 2008 04:34:07 -0000 To: core-commits@hadoop.apache.org From: cdouglas@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080410043408.7DE5D1A9838@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cdouglas Date: Wed Apr 9 21:34:07 2008 New Revision: 646643 URL: http://svn.apache.org/viewvc?rev=646643&view=rev Log: HADOOP-3208. Fix WritableDeserializer to set the Configuration on deserialized Writables. Contributed by Enis Soztutar. Added: hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/ hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/TestWritableSerialization.java Modified: hadoop/core/trunk/CHANGES.txt hadoop/core/trunk/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java hadoop/core/trunk/src/test/org/apache/hadoop/io/TestGenericWritable.java Modified: hadoop/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=646643&r1=646642&r2=646643&view=diff ============================================================================== --- hadoop/core/trunk/CHANGES.txt (original) +++ hadoop/core/trunk/CHANGES.txt Wed Apr 9 21:34:07 2008 @@ -551,6 +551,9 @@ HADOOP-3220. Safemode message corrected. (shv) + HADOOP-3208. Fix WritableDeserializer to set the Configuration on + deserialized Writables. (Enis Soztutar via cdouglas) + Release 0.16.3 - Unreleased BUG FIXES Modified: hadoop/core/trunk/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java?rev=646643&r1=646642&r2=646643&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/io/serializer/WritableSerialization.java Wed Apr 9 21:34:07 2008 @@ -24,6 +24,8 @@ import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; @@ -32,14 +34,17 @@ * {@link Writable#write(java.io.DataOutput)} and * {@link Writable#readFields(java.io.DataInput)}. */ -public class WritableSerialization implements Serialization { +public class WritableSerialization extends Configured + implements Serialization { - static class WritableDeserializer implements Deserializer { + static class WritableDeserializer extends Configured + implements Deserializer { private Class writableClass; private DataInputStream dataIn; - public WritableDeserializer(Class c) { + public WritableDeserializer(Configuration conf, Class c) { + setConf(conf); this.writableClass = c; } @@ -54,7 +59,8 @@ public Writable deserialize(Writable w) throws IOException { Writable writable; if (w == null) { - writable = (Writable) ReflectionUtils.newInstance(writableClass, null); + writable + = (Writable) ReflectionUtils.newInstance(writableClass, getConf()); } else { writable = w; } @@ -95,7 +101,7 @@ } public Deserializer getDeserializer(Class c) { - return new WritableDeserializer(c); + return new WritableDeserializer(getConf(), c); } public Serializer getSerializer(Class c) { Modified: hadoop/core/trunk/src/test/org/apache/hadoop/io/TestGenericWritable.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/TestGenericWritable.java?rev=646643&r1=646642&r2=646643&view=diff ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/io/TestGenericWritable.java (original) +++ hadoop/core/trunk/src/test/org/apache/hadoop/io/TestGenericWritable.java Wed Apr 9 21:34:07 2008 @@ -22,11 +22,11 @@ import java.io.DataOutput; import java.io.IOException; +import junit.framework.TestCase; + import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import junit.framework.TestCase; - /** * TestCase for {@link GenericWritable} class. * @see TestWritable#testWritable(Writable) @@ -34,9 +34,10 @@ public class TestGenericWritable extends TestCase { private Configuration conf; - private static final String CONF_TEST_KEY = "test.generic.writable"; - private static final String CONF_TEST_VALUE = "dummy"; + public static final String CONF_TEST_KEY = "test.generic.writable"; + public static final String CONF_TEST_VALUE = "dummy"; + @Override protected void setUp() throws Exception { super.setUp(); conf = new Configuration(); @@ -53,6 +54,7 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, foo); } + @Override public boolean equals(Object obj) { if (!(obj instanceof Foo)) return false; @@ -75,6 +77,7 @@ public void setConf(Configuration conf) { this.conf = conf; } + @Override public boolean equals(Object obj) { if (!(obj instanceof Bar)) return false; @@ -84,12 +87,14 @@ /** Dummy class for testing {@link GenericWritable} */ public static class Baz extends Bar { + @Override public void readFields(DataInput in) throws IOException { super.readFields(in); //needs a configuration parameter assertEquals("Configuration is not set for the wrapped object", CONF_TEST_VALUE, getConf().get(CONF_TEST_KEY)); } + @Override public void write(DataOutput out) throws IOException { super.write(out); } @@ -97,10 +102,12 @@ /** Dummy class for testing {@link GenericWritable} */ public static class FooGenericWritable extends GenericWritable { + @Override @SuppressWarnings("unchecked") protected Class[] getTypes() { return new Class[] {Foo.class, Bar.class, Baz.class}; } + @Override public boolean equals(Object obj) { if(! (obj instanceof FooGenericWritable)) return false; Added: hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/TestWritableSerialization.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/TestWritableSerialization.java?rev=646643&view=auto ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/TestWritableSerialization.java (added) +++ hadoop/core/trunk/src/test/org/apache/hadoop/io/serializer/TestWritableSerialization.java Wed Apr 9 21:34:07 2008 @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer; + +import static org.apache.hadoop.io.TestGenericWritable.CONF_TEST_KEY; +import static org.apache.hadoop.io.TestGenericWritable.CONF_TEST_VALUE; +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.TestGenericWritable.Baz; +import org.apache.hadoop.io.TestGenericWritable.FooGenericWritable; +import org.apache.hadoop.util.GenericsUtil; + +public class TestWritableSerialization extends TestCase { + + private static final Configuration conf = new Configuration(); + + static { + conf.set("io.serializations" + , "org.apache.hadoop.io.serializer.WritableSerialization"); + } + + public void testWritableSerialization() throws Exception { + Text before = new Text("test writable"); + testSerialization(conf, before); + } + + + public void testWritableConfigurable() throws Exception { + + //set the configuration parameter + conf.set(CONF_TEST_KEY, CONF_TEST_VALUE); + + //reuse TestGenericWritable inner classes to test + //writables that also implement Configurable. + FooGenericWritable generic = new FooGenericWritable(); + generic.setConf(conf); + Baz baz = new Baz(); + generic.set(baz); + Baz result = testSerialization(conf, baz); + assertNotNull(result.getConf()); + } + + /** + * A utility that tests serialization/deserialization. + * @param the class of the item + * @param conf configuration to use, "io.serializations" is read to + * determine the serialization + * @param before item to (de)serialize + * @return deserialized item + */ + public static K testSerialization(Configuration conf, K before) + throws Exception { + + SerializationFactory factory = new SerializationFactory(conf); + Serializer serializer + = factory.getSerializer(GenericsUtil.getClass(before)); + Deserializer deserializer + = factory.getDeserializer(GenericsUtil.getClass(before)); + + DataOutputBuffer out = new DataOutputBuffer(); + serializer.open(out); + serializer.serialize(before); + serializer.close(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(out.getData(), out.getLength()); + deserializer.open(in); + K after = deserializer.deserialize(null); + deserializer.close(); + + assertEquals(before, after); + return after; + } + +}