hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r681238 - in /hadoop/core/trunk: CHANGES.txt src/core/org/apache/hadoop/io/Text.java src/test/org/apache/hadoop/io/TestText.java
Date Wed, 30 Jul 2008 22:59:03 GMT
Author: omalley
Date: Wed Jul 30 15:59:02 2008
New Revision: 681238

URL: http://svn.apache.org/viewvc?rev=681238&view=rev
Log:
HADOOP-3863. Use a thread-local string encoder rather than a static one
that is protected by a lock. (acmurthy via omalley)

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/io/Text.java
    hadoop/core/trunk/src/test/org/apache/hadoop/io/TestText.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=681238&r1=681237&r2=681238&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Jul 30 15:59:02 2008
@@ -137,6 +137,9 @@
     datanode in the pipeline needs to verify the checksum. Saves around
     30% CPU on intermediate datanodes. (rangadi)
 
+    HADOOP-3863. Use a thread-local string encoder rather than a static one
+    that is protected by a lock. (acmurthy via omalley)
+
   BUG FIXES
 
     HADOOP-3563.  Refactor the distributed upgrade code so that it is 

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/io/Text.java?rev=681238&r1=681237&r2=681238&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/io/Text.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/io/Text.java Wed Jul 30 15:59:02 2008
@@ -47,15 +47,24 @@
 public class Text implements WritableComparable {
   private static final Log LOG= LogFactory.getLog("org.apache.hadoop.io.Text");
   
-  private static final CharsetDecoder DECODER = 
-    Charset.forName("UTF-8").newDecoder().
-    onMalformedInput(CodingErrorAction.REPORT).
-    onUnmappableCharacter(CodingErrorAction.REPORT);
-  private static final CharsetEncoder ENCODER = 
-    Charset.forName("UTF-8").newEncoder().
-    onMalformedInput(CodingErrorAction.REPORT).
-    onUnmappableCharacter(CodingErrorAction.REPORT);
-
+  private static ThreadLocal<CharsetEncoder> ENCODER_FACTORY =
+    new ThreadLocal<CharsetEncoder>() {
+      protected CharsetEncoder initialValue() {
+        return Charset.forName("UTF-8").newEncoder().
+               onMalformedInput(CodingErrorAction.REPORT).
+               onUnmappableCharacter(CodingErrorAction.REPORT);
+    }
+  };
+  
+  private static ThreadLocal<CharsetDecoder> DECODER_FACTORY =
+    new ThreadLocal<CharsetDecoder>() {
+    protected CharsetDecoder initialValue() {
+      return Charset.forName("UTF-8").newDecoder().
+             onMalformedInput(CodingErrorAction.REPORT).
+             onUnmappableCharacter(CodingErrorAction.REPORT);
+    }
+  };
+  
   private static final byte [] EMPTY_BYTES = new byte[0];
   
   private byte[] bytes;
@@ -349,21 +358,19 @@
   
   private static String decode(ByteBuffer utf8, boolean replace) 
     throws CharacterCodingException {
-    synchronized(DECODER) {
-      if (replace) {
-        DECODER.onMalformedInput(
-                                 java.nio.charset.CodingErrorAction.REPLACE);
-        DECODER.onUnmappableCharacter(CodingErrorAction.REPLACE);
-      }
-      String str = DECODER.decode(utf8).toString();
-      // set decoder back to its default value: REPORT
-      if (replace) {
-        DECODER.onMalformedInput(CodingErrorAction.REPORT);
-        DECODER.onUnmappableCharacter(CodingErrorAction.REPORT);
-      }
-      return str;
+    CharsetDecoder decoder = DECODER_FACTORY.get();
+    if (replace) {
+      decoder.onMalformedInput(
+          java.nio.charset.CodingErrorAction.REPLACE);
+      decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+    }
+    String str = decoder.decode(utf8).toString();
+    // set decoder back to its default value: REPORT
+    if (replace) {
+      decoder.onMalformedInput(CodingErrorAction.REPORT);
+      decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
     }
-
+    return str;
   }
 
   /**
@@ -390,18 +397,18 @@
    */
   public static ByteBuffer encode(String string, boolean replace)
     throws CharacterCodingException {
-    synchronized(ENCODER) {
-      if (replace) {
-        ENCODER.onMalformedInput(CodingErrorAction.REPLACE);
-        ENCODER.onUnmappableCharacter(CodingErrorAction.REPLACE);
-      }
-      ByteBuffer bytes=ENCODER.encode(CharBuffer.wrap(string.toCharArray()));
-      if (replace) {
-        ENCODER.onMalformedInput(CodingErrorAction.REPORT);
-        ENCODER.onUnmappableCharacter(CodingErrorAction.REPORT);
-      }
-      return bytes;
+    CharsetEncoder encoder = ENCODER_FACTORY.get();
+    if (replace) {
+      encoder.onMalformedInput(CodingErrorAction.REPLACE);
+      encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+    }
+    ByteBuffer bytes = 
+      encoder.encode(CharBuffer.wrap(string.toCharArray()));
+    if (replace) {
+      encoder.onMalformedInput(CodingErrorAction.REPORT);
+      encoder.onUnmappableCharacter(CodingErrorAction.REPORT);
     }
+    return bytes;
   }
 
   /** Read a UTF8 encoded string from in

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/io/TestText.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/TestText.java?rev=681238&r1=681237&r2=681238&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/io/TestText.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/io/TestText.java Wed Jul 30 15:59:02 2008
@@ -216,6 +216,42 @@
     assertEquals("modified aliased string", "abc", b.toString());
     assertEquals("appended string incorrectly", "abcdefg", a.toString());
   }
+  
+  private class ConcurrentEncodeDecodeThread extends Thread {
+    public ConcurrentEncodeDecodeThread(String name) {
+      super(name);
+    }
+
+    public void run() {
+      String name = this.getName();
+      DataOutputBuffer out = new DataOutputBuffer();
+      DataInputBuffer in = new DataInputBuffer();
+      for (int i=0; i < 1000; ++i) {
+        try {
+          out.reset();
+          WritableUtils.writeString(out, name);
+          
+          in.reset(out.getData(), out.getLength());
+          String s = WritableUtils.readString(in);
+          
+          assertEquals(name, s);
+        } catch (Exception ioe) {
+          throw new RuntimeException(ioe);
+        }
+      }
+    }
+  }
+  
+  public void testConcurrentEncodeDecode() throws Exception{
+    Thread thread1 = new ConcurrentEncodeDecodeThread("apache");
+    Thread thread2 = new ConcurrentEncodeDecodeThread("hadoop");
+    
+    thread1.start();
+    thread2.start();
+    
+    thread2.join();
+    thread2.join();
+  }
 
   public static void main(String[] args)  throws Exception
   {



Mime
View raw message