Return-Path: X-Original-To: apmail-spark-commits-archive@minotaur.apache.org Delivered-To: apmail-spark-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 09464184AD for ; Wed, 19 Aug 2015 03:40:04 +0000 (UTC) Received: (qmail 51085 invoked by uid 500); 19 Aug 2015 03:40:04 -0000 Delivered-To: apmail-spark-commits-archive@spark.apache.org Received: (qmail 51058 invoked by uid 500); 19 Aug 2015 03:40:03 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 51049 invoked by uid 99); 19 Aug 2015 03:40:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Aug 2015 03:40:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BCA8AE08EA; Wed, 19 Aug 2015 03:40:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: davies@apache.org To: commits@spark.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: spark git commit: [SPARK-10095] [SQL] use public API of BigInteger Date: Wed, 19 Aug 2015 03:40:03 +0000 (UTC) Repository: spark Updated Branches: refs/heads/master bf32c1f7f -> 270ee6777 [SPARK-10095] [SQL] use public API of BigInteger In UnsafeRow, we use the private field of BigInteger for better performance, but it actually didn't contribute much (3% in one benchmark) to end-to-end runtime, and make it not portable (may fail on other JVM implementations). So we should use the public API instead. cc rxin Author: Davies Liu Closes #8286 from davies/portable_decimal. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/270ee677 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/270ee677 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/270ee677 Branch: refs/heads/master Commit: 270ee677750a1f2adaf24b5816857194e61782ff Parents: bf32c1f Author: Davies Liu Authored: Tue Aug 18 20:39:59 2015 -0700 Committer: Davies Liu Committed: Tue Aug 18 20:39:59 2015 -0700 ---------------------------------------------------------------------- .../sql/catalyst/expressions/UnsafeRow.java | 29 ++++++-------------- .../catalyst/expressions/UnsafeRowWriters.java | 9 ++---- .../java/org/apache/spark/unsafe/Platform.java | 18 ------------ 3 files changed, 11 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/270ee677/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java index 7fd9477..6c02004 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java @@ -273,14 +273,13 @@ public final class UnsafeRow extends MutableRow { } else { final BigInteger integer = value.toJavaBigDecimal().unscaledValue(); - final int[] mag = (int[]) Platform.getObjectVolatile(integer, - Platform.BIG_INTEGER_MAG_OFFSET); - assert(mag.length <= 4); + byte[] bytes = integer.toByteArray(); + assert(bytes.length <= 16); // Write the bytes to the variable length portion. Platform.copyMemory( - mag, Platform.INT_ARRAY_OFFSET, baseObject, baseOffset + cursor, mag.length * 4); - setLong(ordinal, (cursor << 32) | ((long) (((integer.signum() + 1) << 8) + mag.length))); + bytes, Platform.BYTE_ARRAY_OFFSET, baseObject, baseOffset + cursor, bytes.length); + setLong(ordinal, (cursor << 32) | ((long) bytes.length)); } } } @@ -375,8 +374,6 @@ public final class UnsafeRow extends MutableRow { return Platform.getDouble(baseObject, getFieldOffset(ordinal)); } - private static byte[] EMPTY = new byte[0]; - @Override public Decimal getDecimal(int ordinal, int precision, int scale) { if (isNullAt(ordinal)) { @@ -385,20 +382,10 @@ public final class UnsafeRow extends MutableRow { if (precision <= Decimal.MAX_LONG_DIGITS()) { return Decimal.apply(getLong(ordinal), precision, scale); } else { - long offsetAndSize = getLong(ordinal); - long offset = offsetAndSize >>> 32; - int signum = ((int) (offsetAndSize & 0xfff) >> 8); - assert signum >=0 && signum <= 2 : "invalid signum " + signum; - int size = (int) (offsetAndSize & 0xff); - int[] mag = new int[size]; - Platform.copyMemory( - baseObject, baseOffset + offset, mag, Platform.INT_ARRAY_OFFSET, size * 4); - - // create a BigInteger using signum and mag - BigInteger v = new BigInteger(0, EMPTY); // create the initial object - Platform.putInt(v, Platform.BIG_INTEGER_SIGNUM_OFFSET, signum - 1); - Platform.putObjectVolatile(v, Platform.BIG_INTEGER_MAG_OFFSET, mag); - return Decimal.apply(new BigDecimal(v, scale), precision, scale); + byte[] bytes = getBinary(ordinal); + BigInteger bigInteger = new BigInteger(bytes); + BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); + return Decimal.apply(javaDecimal, precision, scale); } } http://git-wip-us.apache.org/repos/asf/spark/blob/270ee677/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java index 005351f..2f43db6 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java @@ -71,16 +71,13 @@ public class UnsafeRowWriters { } final BigInteger integer = input.toJavaBigDecimal().unscaledValue(); - int signum = integer.signum() + 1; - final int[] mag = (int[]) Platform.getObjectVolatile( - integer, Platform.BIG_INTEGER_MAG_OFFSET); - assert(mag.length <= 4); + byte[] bytes = integer.toByteArray(); // Write the bytes to the variable length portion. Platform.copyMemory( - mag, Platform.INT_ARRAY_OFFSET, base, target.getBaseOffset() + cursor, mag.length * 4); + bytes, Platform.BYTE_ARRAY_OFFSET, base, target.getBaseOffset() + cursor, bytes.length); // Set the fixed length portion. - target.setLong(ordinal, (((long) cursor) << 32) | ((long) ((signum << 8) + mag.length))); + target.setLong(ordinal, (((long) cursor) << 32) | (long) bytes.length); return SIZE; } http://git-wip-us.apache.org/repos/asf/spark/blob/270ee677/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java ---------------------------------------------------------------------- diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index 18343ef..1c16da9 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -18,7 +18,6 @@ package org.apache.spark.unsafe; import java.lang.reflect.Field; -import java.math.BigInteger; import sun.misc.Unsafe; @@ -34,10 +33,6 @@ public final class Platform { public static final int DOUBLE_ARRAY_OFFSET; - // Support for resetting final fields while deserializing - public static final long BIG_INTEGER_SIGNUM_OFFSET; - public static final long BIG_INTEGER_MAG_OFFSET; - public static int getInt(Object object, long offset) { return _UNSAFE.getInt(object, offset); } @@ -150,24 +145,11 @@ public final class Platform { INT_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(int[].class); LONG_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(long[].class); DOUBLE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(double[].class); - - long signumOffset = 0; - long magOffset = 0; - try { - signumOffset = _UNSAFE.objectFieldOffset(BigInteger.class.getDeclaredField("signum")); - magOffset = _UNSAFE.objectFieldOffset(BigInteger.class.getDeclaredField("mag")); - } catch (Exception ex) { - // should not happen - } - BIG_INTEGER_SIGNUM_OFFSET = signumOffset; - BIG_INTEGER_MAG_OFFSET = magOffset; } else { BYTE_ARRAY_OFFSET = 0; INT_ARRAY_OFFSET = 0; LONG_ARRAY_OFFSET = 0; DOUBLE_ARRAY_OFFSET = 0; - BIG_INTEGER_SIGNUM_OFFSET = 0; - BIG_INTEGER_MAG_OFFSET = 0; } } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org