Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A5590C4C4 for ; Mon, 17 Nov 2014 11:32:29 +0000 (UTC) Received: (qmail 38854 invoked by uid 500); 17 Nov 2014 11:32:29 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 38820 invoked by uid 500); 17 Nov 2014 11:32:29 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 38810 invoked by uid 99); 17 Nov 2014 11:32:29 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Nov 2014 11:32:29 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 4544793DB26; Mon, 17 Nov 2014 11:32:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hyunsik@apache.org To: commits@tajo.apache.org Message-Id: <4568dc10004c484f8d4dd494127e757c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tajo git commit: TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik) Date: Mon, 17 Nov 2014 11:32:29 +0000 (UTC) Repository: tajo Updated Branches: refs/heads/master 6059a3dd1 -> a0625a620 TAJO-1177: Reduce the use of Sun proprietary API. (Jihun Kang via hyunsik) Closes #243 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a0625a62 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a0625a62 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a0625a62 Branch: refs/heads/master Commit: a0625a620bba9781c51f2ee0f47ec0aad1c36767 Parents: 6059a3d Author: Hyunsik Choi Authored: Mon Nov 17 03:30:36 2014 -0800 Committer: Hyunsik Choi Committed: Mon Nov 17 03:30:36 2014 -0800 ---------------------------------------------------------------------- CHANGES | 3 ++ .../java/org/apache/tajo/datum/TextDatum.java | 5 +- .../java/org/apache/tajo/util/KeyValueSet.java | 6 +-- .../java/org/apache/tajo/util/StringUtils.java | 57 ++++++++++++++++++++ .../java/org/apache/tajo/util/UnsafeUtil.java | 4 +- .../org/apache/tajo/datum/TestTextDatum.java | 14 +++++ .../org/apache/tajo/util/TestStringUtil.java | 24 +++++++++ .../org/apache/tajo/client/TestTajoClient.java | 9 ++-- .../apache/tajo/tuple/offheap/HeapTuple.java | 7 ++- .../apache/tajo/tuple/offheap/UnSafeTuple.java | 7 ++- 10 files changed, 118 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index e68f4a0..ef1561a 100644 --- a/CHANGES +++ b/CHANGES @@ -47,6 +47,9 @@ Release 0.9.1 - unreleased TAJO-1169: Some older version of OpenJDK 1.6 does not get default timezone id. (Jihun Kang via hyunsik) + TAJO-1177: Reduce the use of Sun proprietary API. + (Jihun Kang via hyunsik) + BUG FIXES TAJO-1162: to_char() returns "-00" second. (Hyoungjun Kim) http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java index 9a9b37e..8d8e3f8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java @@ -20,11 +20,12 @@ package org.apache.tajo.datum; import com.google.common.primitives.UnsignedBytes; import com.google.gson.annotations.Expose; -import com.sun.tools.javac.util.Convert; + import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.InvalidCastException; import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.util.MurmurHash; +import org.apache.tajo.util.StringUtils; import java.nio.charset.Charset; import java.util.Comparator; @@ -97,7 +98,7 @@ public class TextDatum extends Datum { @Override public char[] asUnicodeChars() { - return Convert.utf2chars(this.bytes); + return StringUtils.convertBytesToChars(bytes, DEFAULT_CHARSET); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index c1e7092..8e3eb2a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -24,7 +24,6 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.json.GsonObject; -import sun.misc.FloatingDecimal; import java.util.HashMap; import java.util.Map; @@ -173,11 +172,10 @@ public class KeyValueSet implements ProtoObject, Cloneable, Gs if (containsKey(key)) { String strVal = get(key, null); try { - sun.misc.FloatingDecimal fd = FloatingDecimal.readJavaFormatString(strVal); - if (Float.MAX_VALUE < fd.doubleValue()) { + if (Float.MAX_VALUE < Double.parseDouble(strVal)) { throw new IllegalStateException("Parsed value is overflow in float type"); } - return fd.floatValue(); + return Float.parseFloat(strVal); } catch (NumberFormatException nfe) { throw new IllegalArgumentException("No such a config key: " + key); } http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java index 96118ac..38c0fd8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java @@ -25,8 +25,13 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.SignalLogger; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; import java.util.Arrays; import java.util.BitSet; @@ -310,4 +315,56 @@ public class StringUtils { return padded; } + + public static char[] convertBytesToChars(byte[] src, Charset charset) { + CharsetDecoder decoder = charset.newDecoder(); + char[] resultArray = new char[(int) (src.length * decoder.maxCharsPerByte())]; + + if (src.length != 0) { + ByteBuffer byteBuffer = ByteBuffer.wrap(src); + CharBuffer charBuffer = CharBuffer.wrap(resultArray); + + decoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE); + decoder.reset(); + + CoderResult coderResult = decoder.decode(byteBuffer, charBuffer, true); + if (coderResult.isUnderflow()) { + coderResult = decoder.flush(charBuffer); + + if (coderResult.isUnderflow()) { + if (resultArray.length != charBuffer.position()) { + resultArray = Arrays.copyOf(resultArray, charBuffer.position()); + } + } + } + } + + return resultArray; + } + + public static byte[] convertCharsToBytes(char[] src, Charset charset) { + CharsetEncoder encoder = charset.newEncoder(); + byte[] resultArray = new byte[(int) (src.length * encoder.maxBytesPerChar())]; + + if (src.length != 0) { + CharBuffer charBuffer = CharBuffer.wrap(src); + ByteBuffer byteBuffer = ByteBuffer.wrap(resultArray); + + encoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE); + encoder.reset(); + + CoderResult coderResult = encoder.encode(charBuffer, byteBuffer, true); + if (coderResult.isUnderflow()) { + coderResult = encoder.flush(byteBuffer); + + if (coderResult.isUnderflow()) { + if (resultArray.length != byteBuffer.position()) { + resultArray = Arrays.copyOf(resultArray, byteBuffer.position()); + } + } + } + } + + return resultArray; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java index ae88f73..9d6b9b3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/UnsafeUtil.java @@ -95,7 +95,7 @@ public class UnsafeUtil { } public static long getAddress(ByteBuffer buffer) { - Preconditions.checkArgument(buffer instanceof DirectBuffer, "ByteBuffer must be DirectBuffer"); + Preconditions.checkArgument(buffer.isDirect(), "ByteBuffer must be DirectBuffer"); return ((DirectBuffer)buffer).address(); } @@ -105,7 +105,7 @@ public class UnsafeUtil { public static void free(ByteBuffer bb) { Preconditions.checkNotNull(bb); - Preconditions.checkState(bb instanceof DirectBuffer); + Preconditions.checkState(bb.isDirect()); Cleaner cleaner = ((DirectBuffer) bb).cleaner(); if (cleaner != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java index bf48f78..2624d33 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTextDatum.java @@ -96,4 +96,18 @@ public class TestTextDatum { Whitebox.setInternalState(Charset.class, "defaultCharset", systemCharSet); assertEquals(systemCharSet, Charset.defaultCharset()); } + + @Test + public void testAsUnicodeChars() { + byte[] testStringArray = new byte[] {0x74, 0x61, 0x6A, 0x6F}; + TextDatum test = new TextDatum(testStringArray); + + assertArrayEquals(new char[] {'t', 'a', 'j', 'o'}, test.asUnicodeChars()); + + testStringArray = new byte[] {(byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, + (byte) 0xA1, (byte) 0xB0}; + test = new TextDatum(testStringArray); + + assertArrayEquals(new char[] {'\ud0c0', '\uc870'}, test.asUnicodeChars()); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java index 6c732c7..5272586 100644 --- a/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java +++ b/tajo-common/src/test/java/org/apache/tajo/util/TestStringUtil.java @@ -18,6 +18,8 @@ package org.apache.tajo.util; +import java.nio.charset.Charset; + import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringEscapeUtils; import org.junit.Test; @@ -125,4 +127,26 @@ public class TestStringUtil { assertArrayEquals(textArray[1].getBytes(), bytesArray[1]); assertNull(bytesArray[2]); } + + @Test + public void testConvertBytesToChars() { + byte[] testStringArray = new byte[] { 0x74, 0x61, 0x6A, 0x6F }; + assertArrayEquals(new char[] { 't', 'a', 'j', 'o' }, + StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8"))); + + testStringArray = new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte) 0xA1, (byte) 0xB0 }; + assertArrayEquals(new char[] { '\ud0c0', '\uc870' }, + StringUtils.convertBytesToChars(testStringArray, Charset.forName("UTF-8"))); + } + + @Test + public void testConvertCharsToBytes() { + char[] testStringArray = new char[] { 't', 'a', 'j', 'o' }; + assertArrayEquals(new byte[] { 0x74, 0x61, 0x6A, 0x6F }, + StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8"))); + + testStringArray = new char[] { '\ud0c0', '\uc870' }; + assertArrayEquals(new byte[] { (byte) 0xED, (byte) 0x83, (byte) 0x80, (byte) 0xEC, (byte) 0xA1, (byte) 0xB0 }, + StringUtils.convertCharsToBytes(testStringArray, Charset.forName("UTF-8"))); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 30c94f3..6e255ff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -22,8 +22,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.ServiceException; -import com.sun.org.apache.commons.logging.Log; -import com.sun.org.apache.commons.logging.LogFactory; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -39,12 +40,9 @@ import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto; import org.apache.tajo.ipc.ClientProtos.QueryInfoProto; import org.apache.tajo.ipc.ClientProtos.SubQueryHistoryProto; import org.apache.tajo.jdbc.TajoResultSet; -import org.apache.tajo.master.querymaster.QueryInfo; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.history.QueryUnitHistory; -import org.apache.tajo.util.history.SubQueryHistory; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -57,7 +55,6 @@ import java.sql.SQLException; import java.util.*; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; @Category(IntegrationTest.class) public class TestTajoClient { http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java index e38555c..33f9f1c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java +++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java @@ -20,16 +20,19 @@ package org.apache.tajo.tuple.offheap; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; -import com.sun.tools.javac.util.Convert; + import org.apache.tajo.datum.*; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.SizeOf; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.UnsafeUtil; + import sun.misc.Unsafe; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import static org.apache.tajo.common.TajoDataTypes.DataType; @@ -241,7 +244,7 @@ public class HeapTuple implements Tuple { byte [] bytes = new byte[len]; UNSAFE.copyMemory(data, BASE_OFFSET + pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); - return Convert.utf2chars(bytes); + return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8")); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/a0625a62/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java index 138386f..b742e6d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java +++ b/tajo-storage/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java @@ -21,18 +21,21 @@ package org.apache.tajo.tuple.offheap; import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; -import com.sun.tools.javac.util.Convert; + import org.apache.tajo.datum.*; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.SizeOf; +import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.UnsafeUtil; + import sun.misc.Unsafe; import sun.nio.ch.DirectBuffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.Charset; import static org.apache.tajo.common.TajoDataTypes.DataType; @@ -278,7 +281,7 @@ public abstract class UnSafeTuple implements Tuple { byte [] bytes = new byte[len]; UNSAFE.copyMemory(null, pos, bytes, UnsafeUtil.ARRAY_BYTE_BASE_OFFSET, len); - return Convert.utf2chars(bytes); + return StringUtils.convertBytesToChars(bytes, Charset.forName("UTF-8")); } @Override