tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject tajo git commit: TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik)
Date Mon, 17 Nov 2014 11:32:29 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 6059a3dd1 -> a0625a620


TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik)

Closes #243


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a0625a62
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a0625a62
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a0625a62

Branch: refs/heads/master
Commit: a0625a620bba9781c51f2ee0f47ec0aad1c36767
Parents: 6059a3d
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Mon Nov 17 03:30:36 2014 -0800
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Mon Nov 17 03:30:36 2014 -0800

----------------------------------------------------------------------
 CHANGES                                         |  3 ++
 .../java/org/apache/tajo/datum/TextDatum.java   |  5 +-
 .../java/org/apache/tajo/util/KeyValueSet.java  |  6 +--
 .../java/org/apache/tajo/util/StringUtils.java  | 57 ++++++++++++++++++++
 .../java/org/apache/tajo/util/UnsafeUtil.java   |  4 +-
 .../org/apache/tajo/datum/TestTextDatum.java    | 14 +++++
 .../org/apache/tajo/util/TestStringUtil.java    | 24 +++++++++
 .../org/apache/tajo/client/TestTajoClient.java  |  9 ++--
 .../apache/tajo/tuple/offheap/HeapTuple.java    |  7 ++-
 .../apache/tajo/tuple/offheap/UnSafeTuple.java  |  7 ++-
 10 files changed, 118 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index e68f4a0..ef1561a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -47,6 +47,9 @@ Release 0.9.1 - unreleased
     TAJO-1169: Some older version of OpenJDK 1.6 does not get default 
     timezone id. (Jihun Kang via hyunsik)
 
+    TAJO-1177: Reduce the use of Sun proprietary API. 
+    (Jihun Kang via hyunsik)
+
   BUG FIXES
     
     TAJO-1162: to_char() returns "-00" second. (Hyoungjun Kim)

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
index 9a9b37e..8d8e3f8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
@@ -20,11 +20,12 @@ package org.apache.tajo.datum;
 
 import com.google.common.primitives.UnsignedBytes;
 import com.google.gson.annotations.Expose;
-import com.sun.tools.javac.util.Convert;
+
 import org.apache.tajo.common.TajoDataTypes;
 import org.apache.tajo.exception.InvalidCastException;
 import org.apache.tajo.exception.InvalidOperationException;
 import org.apache.tajo.util.MurmurHash;
+import org.apache.tajo.util.StringUtils;
 
 import java.nio.charset.Charset;
 import java.util.Comparator;
@@ -97,7 +98,7 @@ public class TextDatum extends Datum {
 
   @Override
   public char[] asUnicodeChars() {
-    return Convert.utf2chars(this.bytes);
+    return StringUtils.convertBytesToChars(bytes, DEFAULT_CHARSET);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
index c1e7092..8e3eb2a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java
@@ -24,7 +24,6 @@ import com.google.gson.annotations.Expose;
 import org.apache.tajo.common.ProtoObject;
 import org.apache.tajo.json.CommonGsonHelper;
 import org.apache.tajo.json.GsonObject;
-import sun.misc.FloatingDecimal;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -173,11 +172,10 @@ public class KeyValueSet implements ProtoObject<KeyValueSetProto>,
Cloneable, Gs
     if (containsKey(key)) {
       String strVal = get(key, null);
       try {
-        sun.misc.FloatingDecimal fd = FloatingDecimal.readJavaFormatString(strVal);
-        if (Float.MAX_VALUE < fd.doubleValue()) {
+        if (Float.MAX_VALUE < Double.parseDouble(strVal)) {
           throw new IllegalStateException("Parsed value is overflow in float type");
         }
-        return fd.floatValue();
+        return Float.parseFloat(strVal);
       } catch (NumberFormatException nfe) {
         throw new IllegalArgumentException("No such a config key: "  + key);
       }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 96118ac..38c0fd8 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -25,8 +25,13 @@ import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.SignalLogger;
 
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
 import java.util.Arrays;
 import java.util.BitSet;
 
@@ -310,4 +315,56 @@ public class StringUtils {
 
     return padded;
   }
+  
+  public static char[] convertBytesToChars(byte[] src, Charset charset) {
+    CharsetDecoder decoder = charset.newDecoder();
+    char[] resultArray = new char[(int) (src.length * decoder.maxCharsPerByte())];
+    
+    if (src.length != 0) {
+      ByteBuffer byteBuffer = ByteBuffer.wrap(src);
+      CharBuffer charBuffer = CharBuffer.wrap(resultArray);
+      
+      decoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
+      decoder.reset();
+      
+      CoderResult coderResult = decoder.decode(byteBuffer, charBuffer, true);
+      if (coderResult.isUnderflow()) {
+        coderResult = decoder.flush(charBuffer);
+        
+        if (coderResult.isUnderflow()) {
+          if (resultArray.length != charBuffer.position()) {
+            resultArray = Arrays.copyOf(resultArray, charBuffer.position());
+          }
+        }
+      }
+    }
+    
+    return resultArray;
+  }
+  
+  public static byte[] convertCharsToBytes(char[] src, Charset charset) {
+    CharsetEncoder encoder = charset.newEncoder();
+    byte[] resultArray = new byte[(int) (src.length * encoder.maxBytesPerChar())];
+    
+    if (src.length != 0) {
+      CharBuffer charBuffer = CharBuffer.wrap(src);
+      ByteBuffer byteBuffer = ByteBuffer.wrap(resultArray);
+      
+      encoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
+      encoder.reset();
+      
+      CoderResult coderResult = encoder.encode(charBuffer, byteBuffer, true);
+      if (coderResult.isUnderflow()) {
+        coderResult = encoder.flush(byteBuffer);
+        
+        if (coderResult.isUnderflow()) {
+          if (resultArray.length != byteBuffer.position()) {
+            resultArray = Arrays.copyOf(resultArray, byteBuffer.position());
+          }
+        }
+      }
+    }
+    
+    return resultArray;
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
index ae88f73..9d6b9b3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java
@@ -95,7 +95,7 @@ public class UnsafeUtil {
   }
 
   public static long getAddress(ByteBuffer buffer) {
-    Preconditions.checkArgument(buffer instanceof DirectBuffer, "ByteBuffer must be DirectBuffer");
+    Preconditions.checkArgument(buffer.isDirect(), "ByteBuffer must be DirectBuffer");
     return ((DirectBuffer)buffer).address();
   }
 
@@ -105,7 +105,7 @@ public class UnsafeUtil {
 
   public static void free(ByteBuffer bb) {
     Preconditions.checkNotNull(bb);
-    Preconditions.checkState(bb instanceof DirectBuffer);
+    Preconditions.checkState(bb.isDirect());
 
     Cleaner cleaner = ((DirectBuffer) bb).cleaner();
     if (cleaner != null) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
index bf48f78..2624d33 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java
@@ -96,4 +96,18 @@ public class TestTextDatum {
     Whitebox.setInternalState(Charset.class, "defaultCharset", systemCharSet);
     assertEquals(systemCharSet, Charset.defaultCharset());
   }
+  
+  @Test
+  public void testAsUnicodeChars() {
+    byte[] testStringArray = new byte[] {0x74, 0x61, 0x6A, 0x6F};
+    TextDatum test = new TextDatum(testStringArray);
+    
+    assertArrayEquals(new char[] {'t', 'a', 'j', 'o'}, test.asUnicodeChars());
+    
+    testStringArray = new byte[] {(byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, 
+        (byte) 0xA1, (byte) 0xB0};
+    test = new TextDatum(testStringArray);
+    
+    assertArrayEquals(new char[] {'\ud0c0', '\uc870'}, test.asUnicodeChars());
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
index 6c732c7..5272586 100644
--- a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
+++ b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java
@@ -18,6 +18,8 @@
 
 package org.apache.tajo.util;
 
+import java.nio.charset.Charset;
+
 import org.apache.commons.lang.CharUtils;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.junit.Test;
@@ -125,4 +127,26 @@ public class TestStringUtil {
     assertArrayEquals(textArray[1].getBytes(), bytesArray[1]);
     assertNull(bytesArray[2]);
   }
+  
+  @Test
+  public void testConvertBytesToChars() {
+    byte[] testStringArray = new byte[] { 0x74, 0x61, 0x6A, 0x6F };
+    assertArrayEquals(new char[] { 't', 'a', 'j', 'o' },
+        StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8")));
+
+    testStringArray = new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte)
0xA1, (byte) 0xB0 };
+    assertArrayEquals(new char[] { '\ud0c0', '\uc870' },
+        StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8")));
+  }
+  
+  @Test
+  public void testConvertCharsToBytes() {
+    char[] testStringArray = new char[] { 't', 'a', 'j', 'o' };
+    assertArrayEquals(new byte[] { 0x74, 0x61, 0x6A, 0x6F },
+        StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8")));
+
+    testStringArray = new char[] { '\ud0c0', '\uc870' };
+    assertArrayEquals(new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte)
0xA1, (byte) 0xB0 },
+        StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8")));
+  }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 30c94f3..6e255ff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -22,8 +22,9 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.protobuf.ServiceException;
-import com.sun.org.apache.commons.logging.Log;
-import com.sun.org.apache.commons.logging.LogFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -39,12 +40,9 @@ import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto;
 import org.apache.tajo.ipc.ClientProtos.QueryInfoProto;
 import org.apache.tajo.ipc.ClientProtos.SubQueryHistoryProto;
 import org.apache.tajo.jdbc.TajoResultSet;
-import org.apache.tajo.master.querymaster.QueryInfo;
 import org.apache.tajo.storage.StorageConstants;
 import org.apache.tajo.storage.StorageUtil;
 import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.history.QueryUnitHistory;
-import org.apache.tajo.util.history.SubQueryHistory;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -57,7 +55,6 @@ import java.sql.SQLException;
 import java.util.*;
 
 import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
 
 @Category(IntegrationTest.class)
 public class TestTajoClient {

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
index e38555c..33f9f1c 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
@@ -20,16 +20,19 @@ package org.apache.tajo.tuple.offheap;
 
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.Message;
-import com.sun.tools.javac.util.Convert;
+
 import org.apache.tajo.datum.*;
 import org.apache.tajo.exception.UnsupportedException;
 import org.apache.tajo.storage.Tuple;
 import org.apache.tajo.storage.VTuple;
 import org.apache.tajo.util.SizeOf;
+import org.apache.tajo.util.StringUtils;
 import org.apache.tajo.util.UnsafeUtil;
+
 import sun.misc.Unsafe;
 
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 
 import static org.apache.tajo.common.TajoDataTypes.DataType;
 
@@ -241,7 +244,7 @@ public class HeapTuple implements Tuple {
 
     byte [] bytes = new byte[len];
     UNSAFE.copyMemory(data, BASE_OFFSET + pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET,
len);
-    return Convert.utf2chars(bytes);
+    return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8"));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
index 138386f..b742e6d 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
@@ -21,18 +21,21 @@ package org.apache.tajo.tuple.offheap;
 import com.google.common.base.Preconditions;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.Message;
-import com.sun.tools.javac.util.Convert;
+
 import org.apache.tajo.datum.*;
 import org.apache.tajo.exception.UnsupportedException;
 import org.apache.tajo.storage.Tuple;
 import org.apache.tajo.storage.VTuple;
 import org.apache.tajo.util.SizeOf;
+import org.apache.tajo.util.StringUtils;
 import org.apache.tajo.util.UnsafeUtil;
+
 import sun.misc.Unsafe;
 import sun.nio.ch.DirectBuffer;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.charset.Charset;
 
 import static org.apache.tajo.common.TajoDataTypes.DataType;
 
@@ -278,7 +281,7 @@ public abstract class UnSafeTuple implements Tuple {
 
     byte [] bytes = new byte[len];
     UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len);
-    return Convert.utf2chars(bytes);
+    return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8"));
   }
 
   @Override


Mime
View raw message