hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject svn commit: r644630 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/util/ src/test/org/apache/hadoop/io/ src/test/org/apache/hadoop/util/
Date Fri, 04 Apr 2008 07:13:38 GMT
Author: enis
Date: Fri Apr  4 00:13:35 2008
New Revision: 644630

URL: http://svn.apache.org/viewvc?rev=644630&view=rev
Log:
HADOOP-3048.  A new Interface and a default implementation to convert and restore serializaztions
of objects to strings. Contributed by Enis Soztutar.

Added:
    hadoop/core/trunk/src/java/org/apache/hadoop/io/DefaultStringifier.java
    hadoop/core/trunk/src/java/org/apache/hadoop/io/Stringifier.java
    hadoop/core/trunk/src/java/org/apache/hadoop/util/GenericsUtil.java
    hadoop/core/trunk/src/test/org/apache/hadoop/io/TestDefaultStringifier.java
    hadoop/core/trunk/src/test/org/apache/hadoop/util/TestGenericsUtil.java
Modified:
    hadoop/core/trunk/CHANGES.txt

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=644630&r1=644629&r2=644630&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Apr  4 00:13:35 2008
@@ -111,6 +111,9 @@
     HADOOP-3001. Add job counters that measure the number of bytes
     read and written to HDFS, S3, KFS, and local file systems. (omalley)
 
+    HADOOP-3048.  A new Interface and a default implementation to convert 
+    and restore serializaztions of objects to strings. (enis)
+
   IMPROVEMENTS
 
     HADOOP-2655. Copy on write for data and metadata files in the 

Added: hadoop/core/trunk/src/java/org/apache/hadoop/io/DefaultStringifier.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/DefaultStringifier.java?rev=644630&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/io/DefaultStringifier.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/io/DefaultStringifier.java Fri Apr  4 00:13:35
2008
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.ArrayList;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.Serialization;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.hadoop.util.GenericsUtil;
+
+/**
+ * DefaultStringifier is the default implementation of the {@link Stringifier}
+ * interface which stringifies the objects using base64 encoding of the
+ * serialized version of the objects. The {@link Serializer} and
+ * {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ * <br>
+ * DefaultStringifier offers convenience methods to store/load objects to/from
+ * the configuration.
+ * 
+ * @param <T> the class of the objects to stringify
+ */
+public class DefaultStringifier<T> implements Stringifier<T> {
+
+  private static final String SEPARATOR = ",";
+
+  private Serializer<T> serializer;
+
+  private Deserializer<T> deserializer;
+
+  private DataInputBuffer inBuf;
+
+  private DataOutputBuffer outBuf;
+
+  public DefaultStringifier(Configuration conf, Class<T> c) {
+
+    SerializationFactory factory = new SerializationFactory(conf);
+    this.serializer = factory.getSerializer(c);
+    this.deserializer = factory.getDeserializer(c);
+    this.inBuf = new DataInputBuffer();
+    this.outBuf = new DataOutputBuffer();
+    try {
+      serializer.open(outBuf);
+      deserializer.open(inBuf);
+    } catch (IOException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
+  public T fromString(String str) throws IOException {
+    try {
+      byte[] bytes = Base64.decodeBase64(str.getBytes("UTF-8"));
+      inBuf.reset(bytes, bytes.length);
+      T restored = deserializer.deserialize(null);
+      return restored;
+    } catch (UnsupportedCharsetException ex) {
+      throw new IOException(ex.toString());
+    }
+  }
+
+  public String toString(T obj) throws IOException {
+    outBuf.reset();
+    serializer.serialize(obj);
+    byte[] buf = new byte[outBuf.getLength()];
+    System.arraycopy(outBuf.getData(), 0, buf, 0, buf.length);
+    return new String(Base64.encodeBase64(buf));
+  }
+
+  public void close() throws IOException {
+    inBuf.close();
+    outBuf.close();
+    deserializer.close();
+    serializer.close();
+  }
+
+  /**
+   * Stores the item in the configuration with the given keyName.
+   * 
+   * @param <K>  the class of the item
+   * @param conf the configuration to store
+   * @param item the object to be stored
+   * @param keyName the name of the key to use
+   * @throws IOException : forwards Exceptions from the underlying 
+   * {@link Serialization} classes. 
+   */
+  public static <K> void store(Configuration conf, K item, String keyName)
+  throws IOException {
+
+    DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
+        GenericsUtil.getClass(item));
+    conf.set(keyName, stringifier.toString(item));
+    stringifier.close();
+  }
+
+  /**
+   * Restores the object from the configuration.
+   * 
+   * @param <K> the class of the item
+   * @param conf the configuration to use
+   * @param keyName the name of the key to use
+   * @param itemClass the class of the item
+   * @return restored object
+   * @throws IOException : forwards Exceptions from the underlying 
+   * {@link Serialization} classes.
+   */
+  public static <K> K load(Configuration conf, String keyName,
+      Class<K> itemClass) throws IOException {
+    DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
+        itemClass);
+    try {
+      String itemStr = conf.get(keyName);
+      return stringifier.fromString(itemStr);
+    } finally {
+      stringifier.close();
+    }
+  }
+
+  /**
+   * Stores the array of items in the configuration with the given keyName.
+   * 
+   * @param <K> the class of the item
+   * @param conf the configuration to use 
+   * @param items the objects to be stored
+   * @param keyName the name of the key to use
+   * @throws IndexOutOfBoundsException if the items array is empty
+   * @throws IOException : forwards Exceptions from the underlying 
+   * {@link Serialization} classes.         
+   */
+  public static <K> void storeArray(Configuration conf, K[] items,
+      String keyName) throws IOException {
+
+    DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
+        GenericsUtil.getClass(items[0]));
+    try {
+      StringBuilder builder = new StringBuilder();
+      for (K item : items) {
+        builder.append(stringifier.toString(item)).append(SEPARATOR);
+      }
+      conf.set(keyName, builder.toString());
+    }
+    finally {
+      stringifier.close();
+    }
+  }
+
+  /**
+   * Restores the array of objects from the configuration.
+   * 
+   * @param <K> the class of the item
+   * @param conf the configuration to use
+   * @param keyName the name of the key to use
+   * @param itemClass the class of the item
+   * @return restored object
+   * @throws IOException : forwards Exceptions from the underlying 
+   * {@link Serialization} classes.
+   */
+  public static <K> K[] loadArray(Configuration conf, String keyName,
+      Class<K> itemClass) throws IOException {
+    DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
+        itemClass);
+    try {
+      String itemStr = conf.get(keyName);
+      ArrayList<K> list = new ArrayList<K>();
+      String[] parts = itemStr.split(SEPARATOR);
+
+      for (String part : parts) {
+        if (!part.equals(""))
+          list.add(stringifier.fromString(part));
+      }
+
+      return GenericsUtil.toArray(itemClass, list);
+    }
+    finally {
+      stringifier.close();
+    }
+  }
+
+}

Added: hadoop/core/trunk/src/java/org/apache/hadoop/io/Stringifier.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/Stringifier.java?rev=644630&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/io/Stringifier.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/io/Stringifier.java Fri Apr  4 00:13:35 2008
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+
+/**
+ * Stringifier interface offers two methods to convert an object 
+ * to a string representation and restore the object given its 
+ * string representation.
+ * @param <T> the class of the objects to stringify
+ */
+public interface Stringifier<T> extends java.io.Closeable {
+
+  /**
+   * Converts the object to a string representation
+   * @param obj the object to convert
+   * @return the string representation of the object
+   * @throws IOException if the object cannot be converted
+   */
+  public String toString(T obj)  throws IOException;
+  
+  /**
+   * Restores the object from its string representation.
+   * @param str the string representation of the object
+   * @return restored object
+   * @throws IOException if the object cannot be restored
+   */
+  public T fromString(String str) throws IOException;
+  
+  
+  /** 
+   * Closes this object. 
+   * @throws IOException if an I/O error occurs 
+   * */
+  public void close() throws IOException;
+  
+}

Added: hadoop/core/trunk/src/java/org/apache/hadoop/util/GenericsUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/util/GenericsUtil.java?rev=644630&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/util/GenericsUtil.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/util/GenericsUtil.java Fri Apr  4 00:13:35
2008
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.lang.reflect.Array;
+import java.util.List;
+
+/**
+ * Contains utility methods for dealing with Java Generics. 
+ */
+public class GenericsUtil {
+
+  /**
+   * Returns the Class object (of type <code>Class&lt;T&gt;</code>) of
the  
+   * argument of type <code>T</code>. 
+   * @param <T> The type of the argument
+   * @param t the object to get it class
+   * @return <code>Class&lt;T&gt;</code>
+   */
+  public static <T> Class<T> getClass(T t) {
+    @SuppressWarnings("unchecked")
+    Class<T> clazz = (Class<T>)t.getClass();
+    return clazz;
+  }
+
+  /**
+   * Converts the given <code>List&lt;T&gt;</code> to a an array of 
+   * <code>T[]</code>.
+   * @param c the Class object of the items in the list
+   * @param list the list to convert
+   */
+  public static <T> T[] toArray(Class<T> c, List<T> list)
+  {
+    @SuppressWarnings("unchecked")
+    T[] ta= (T[])Array.newInstance(c, list.size());
+
+    for (int i= 0; i<list.size(); i++)
+      ta[i]= list.get(i);
+    return ta;
+  }
+
+
+  /**
+   * Converts the given <code>List&lt;T&gt;</code> to a an array of 
+   * <code>T[]</code>. 
+   * @param list the list to convert
+   * @throws ArrayIndexOutOfBoundsException if the list is empty. 
+   * Use {@link #toArray(Class, List)} if the list may be empty.
+   */
+  public static <T> T[] toArray(List<T> list) {
+    return toArray(getClass(list.get(0)), list);
+  }
+
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/io/TestDefaultStringifier.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/TestDefaultStringifier.java?rev=644630&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/io/TestDefaultStringifier.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/io/TestDefaultStringifier.java Fri Apr  4
00:13:35 2008
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+public class TestDefaultStringifier extends TestCase {
+
+  private static Configuration conf = new Configuration();
+  private static final Log LOG = LogFactory.getLog(TestDefaultStringifier.class);
+
+  private char[] alphabet = "abcdefghijklmnopqrstuvwxyz".toCharArray();
+
+  public void testWithWritable() throws Exception {
+
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.WritableSerialization");
+
+    LOG.info("Testing DefaultStringifier with Text");
+
+    Random random = new Random();
+
+    //test with a Text
+    for(int i=0;i<10;i++) {
+      //generate a random string
+      StringBuilder builder = new StringBuilder();
+      int strLen = random.nextInt(40);
+      for(int j=0; j< strLen; j++) {
+        builder.append(alphabet[random.nextInt(alphabet.length)]);
+      }
+      Text text = new Text(builder.toString());
+      DefaultStringifier<Text> stringifier = new DefaultStringifier<Text>(conf,
Text.class);
+
+      String str = stringifier.toString(text);
+      Text claimedText = stringifier.fromString(str);
+      LOG.info("Object: " + text);
+      LOG.info("String representation of the object: " + str);
+      assertEquals(text, claimedText);
+    }
+  }
+
+  public void testWithJavaSerialization() throws Exception {
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
+
+    LOG.info("Testing DefaultStringifier with Serializable Integer");
+
+    //Integer implements Serializable
+    Integer testInt = Integer.valueOf(42);
+    DefaultStringifier<Integer> stringifier = new DefaultStringifier<Integer>(conf,
Integer.class);
+
+    String str = stringifier.toString(testInt);
+    Integer claimedInt = stringifier.fromString(str);
+    LOG.info("String representation of the object: " + str);
+
+    assertEquals(testInt, claimedInt);
+  }
+
+  public void testStoreLoad() throws IOException {
+
+    LOG.info("Testing DefaultStringifier#store() and #load()");
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.WritableSerialization");
+    Text text = new Text("uninteresting test string");
+    String keyName = "test.defaultstringifier.key1";
+
+    DefaultStringifier.store(conf,text, keyName);
+
+    Text claimedText = DefaultStringifier.load(conf, keyName, Text.class);
+    assertEquals("DefaultStringifier#load() or #store() might be flawed"
+        , text, claimedText);
+
+  }
+
+  public void testStoreLoadArray() throws IOException {
+    LOG.info("Testing DefaultStringifier#storeArray() and #loadArray()");
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
+
+    String keyName = "test.defaultstringifier.key2";
+
+    Integer[] array = new Integer[] {1,2,3,4,5};
+
+
+    DefaultStringifier.storeArray(conf, array, keyName);
+
+    Integer[] claimedArray = DefaultStringifier.<Integer>loadArray(conf, keyName, Integer.class);
+    for (int i = 0; i < array.length; i++) {
+      assertEquals("two arrays are not equal", array[i], claimedArray[i]);
+    }
+
+  }
+
+}

Added: hadoop/core/trunk/src/test/org/apache/hadoop/util/TestGenericsUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/util/TestGenericsUtil.java?rev=644630&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/util/TestGenericsUtil.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/util/TestGenericsUtil.java Fri Apr  4 00:13:35
2008
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+public class TestGenericsUtil extends TestCase {
+
+  public void testToArray() {
+    
+    //test a list of size 10
+    List<Integer> list = new ArrayList<Integer>(); 
+    
+    for(int i=0; i<10; i++) {
+      list.add(i);
+    }
+    
+    Integer[] arr = GenericsUtil.toArray(list);
+    
+    for (int i = 0; i < arr.length; i++) {
+      assertEquals(list.get(i), arr[i]);
+    }
+  }
+  
+  public void testWithEmptyList() {
+    try {
+      List<String> list = new ArrayList<String>();
+      String[] arr = GenericsUtil.toArray(list);
+      fail("Empty array should throw exception");
+      System.out.println(arr); //use arr so that compiler will not complain
+      
+    }catch (IndexOutOfBoundsException ex) {
+      //test case is successful
+    }
+  }
+ 
+  public void testWithEmptyList2() {
+    List<String> list = new ArrayList<String>();
+    //this method should not throw IndexOutOfBoundsException
+    String[] arr = GenericsUtil.<String>toArray(String.class, list);
+    
+    assertEquals(0, arr.length);
+  }
+  
+  /** This class uses generics */
+  private class GenericClass<T> {
+    T dummy;
+    List<T> list = new ArrayList<T>();
+    
+    void add(T item) {
+      list.add(item);
+    }
+    
+    T[] funcThatUsesToArray() {
+      T[] arr = GenericsUtil.toArray(list);
+      return arr;
+    }
+  }
+  
+  public void testWithGenericClass() {
+    
+    GenericClass<String> testSubject = new GenericClass<String>();
+    
+    testSubject.add("test1");
+    testSubject.add("test2");
+    
+    try {
+      //this cast would fail, if we had not used GenericsUtil.toArray, since the 
+      //rmethod would return Object[] rather than String[]
+      String[] arr = testSubject.funcThatUsesToArray();
+      
+      assertEquals("test1", arr[0]);
+      assertEquals("test2", arr[1]);
+      
+    }catch (ClassCastException ex) {
+      fail("GenericsUtil#toArray() is not working for generic classes");
+    }
+    
+  }
+  
+  public void testGetClass() {
+    
+    //test with Integer
+    Integer x = new Integer(42); 
+    Class<Integer> c = GenericsUtil.getClass(x);
+    assertEquals(Integer.class, c);
+    
+    //test with GenericClass<Integer>
+    GenericClass<Integer> testSubject = new GenericClass<Integer>();
+    Class<GenericClass<Integer>> c2 = GenericsUtil.getClass(testSubject);
+    assertEquals(GenericClass.class, c2);
+  }
+  
+}



Mime
View raw message