Return-Path: X-Original-To: apmail-parquet-commits-archive@minotaur.apache.org Delivered-To: apmail-parquet-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5858D176AB for ; Mon, 27 Apr 2015 23:11:59 +0000 (UTC) Received: (qmail 48082 invoked by uid 500); 27 Apr 2015 23:11:59 -0000 Delivered-To: apmail-parquet-commits-archive@parquet.apache.org Received: (qmail 48001 invoked by uid 500); 27 Apr 2015 23:11:59 -0000 Mailing-List: contact commits-help@parquet.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@parquet.incubator.apache.org Delivered-To: mailing list commits@parquet.incubator.apache.org Received: (qmail 47386 invoked by uid 99); 27 Apr 2015 23:11:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Apr 2015 23:11:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9E9AEE17F5; Mon, 27 Apr 2015 23:11:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: blue@apache.org To: commits@parquet.apache.org Date: Mon, 27 Apr 2015 23:12:15 -0000 Message-Id: <48b582e26bd94b26b566c239dde758a6@git.apache.org> In-Reply-To: <190ba0de36204468a8a5a0e9d143ae5d@git.apache.org> References: <190ba0de36204468a8a5a0e9d143ae5d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/51] [partial] parquet-mr git commit: PARQUET-23: Rename to org.apache.parquet. http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java new file mode 100644 index 0000000..c0acd11 --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java @@ -0,0 +1,725 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader; +import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter; + +// TODO: rework the whole thing. It does not need to use streams at all +/** + * provides the correct implementation of a bitpacking based on the width in bits + * + * @author Julien Le Dem + * + */ +public class BitPacking { + + /** + * to writes ints to a stream packed to only the needed bits. + * there is no guarantee of corecteness if ints larger than the max size are written + * + * @author Julien Le Dem + * + */ + abstract public static class BitPackingWriter { + /** + * will write the bits to the underlying stream aligned on the buffer size + * @param val the value to encode + * @throws IOException + */ + abstract public void write(int val) throws IOException; + + /** + * will flush the buffer to the underlying stream (and pad with 0s) + * @throws IOException + */ + abstract public void finish() throws IOException; + } + + /** + * to read back what has been written with the corresponding writer + * + * @author Julien Le Dem + * + */ + abstract public static class BitPackingReader { + + /** + * + * @return and int decoded from the underlying stream + * @throws IOException + */ + abstract public int read() throws IOException; + } + + private BitPacking() { + } + + /** + * @param bitLength the width in bits of the integers to write + * @param out the stream to write the bytes to + * @return the correct implementation for the width + */ + public static BitPackingWriter getBitPackingWriter(int bitLength, OutputStream out) { + switch (bitLength) { + case 0: + return new ZeroBitPackingWriter(); + case 1: + return new OneBitPackingWriter(out); + case 2: + return new TwoBitPackingWriter(out); + case 3: + return new ThreeBitPackingWriter(out); + case 4: + return new FourBitPackingWriter(out); + case 5: + return new FiveBitPackingWriter(out); + case 6: + return new SixBitPackingWriter(out); + case 7: + return new SevenBitPackingWriter(out); + case 8: + return new EightBitPackingWriter(out); + default: + throw new UnsupportedOperationException("only support up to 8 for now"); + } + } + + /** + * + * @param bitLength the width in bits of the integers to read + * @param inthe stream to read the bytes from + * @return the correct implementation for the width + */ + public static BitPackingReader createBitPackingReader(int bitLength, InputStream in, long valueCount) { + switch (bitLength) { + case 0: + return new ZeroBitPackingReader(); + case 1: + return new OneBitPackingReader(in); + case 2: + return new TwoBitPackingReader(in); + case 3: + return new ThreeBitPackingReader(in, valueCount); + case 4: + return new FourBitPackingReader(in); + case 5: + return new FiveBitPackingReader(in, valueCount); + case 6: + return new SixBitPackingReader(in, valueCount); + case 7: + return new SevenBitPackingReader(in, valueCount); + case 8: + return new EightBitPackingReader(in); + default: + throw new UnsupportedOperationException("only support up to 8 for now"); + } + } +} + +abstract class BaseBitPackingWriter extends BitPackingWriter { + + void finish(int numberOfBits, int buffer, OutputStream out) throws IOException { + int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8); + buffer = buffer << padding; + int numberOfBytes = (numberOfBits + padding) / 8; + for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) { + out.write((buffer >>> i) & 0xFF); + } + } + + void finish(int numberOfBits, long buffer, OutputStream out) throws IOException { + int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8); + buffer = buffer << padding; + int numberOfBytes = (numberOfBits + padding) / 8; + for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) { + out.write((int)(buffer >>> i) & 0xFF); + } + } +} +abstract class BaseBitPackingReader extends BitPackingReader { + + int alignToBytes(int bitsCount) { + return BytesUtils.paddedByteCountFromBits(bitsCount); + } + +} + +class ZeroBitPackingWriter extends BitPackingWriter { + + @Override + public void write(int val) throws IOException { + } + + @Override + public void finish() { + } + +} +class ZeroBitPackingReader extends BitPackingReader { + + @Override + public int read() throws IOException { + return 0; + } + +} + +class OneBitPackingWriter extends BitPackingWriter { + + private OutputStream out; + + private int buffer = 0; + private int count = 0; + + public OneBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 1; + buffer |= val; + ++ count; + if (count == 8) { + out.write(buffer); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + while (count != 0) { + write(0); + } + // check this does not impede perf + out = null; + } + +} +class OneBitPackingReader extends BitPackingReader { + + private final InputStream in; + + private int buffer = 0; + private int count = 0; + + public OneBitPackingReader(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + if (count == 0) { + buffer = in.read(); + count = 8; + } + int result = (buffer >> (count - 1)) & 1; + -- count; + return result; + } + +} + +class TwoBitPackingWriter extends BitPackingWriter { + + private OutputStream out; + + private int buffer = 0; + private int count = 0; + + public TwoBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 2; + buffer |= val; + ++ count; + if (count == 4) { + out.write(buffer); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + while (count != 0) { + write(0); + } + // check this does not impede perf + out = null; + } + +} +class TwoBitPackingReader extends BitPackingReader { + + private final InputStream in; + + private int buffer = 0; + private int count = 0; + + public TwoBitPackingReader(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + if (count == 0) { + buffer = in.read(); + count = 4; + } + int result = (buffer >> ((count - 1) * 2)) & 3; + -- count; + return result; + } + +} + +class ThreeBitPackingWriter extends BaseBitPackingWriter { + + private OutputStream out; + + private int buffer = 0; + private int count = 0; + + public ThreeBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 3; + buffer |= val; + ++ count; + if (count == 8) { + out.write((buffer >>> 16) & 0xFF); + out.write((buffer >>> 8) & 0xFF); + out.write((buffer >>> 0) & 0xFF); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + if (count != 0) { + int numberOfBits = count * 3; + finish(numberOfBits, buffer, out); + buffer = 0; + count = 0; + } + // check this does not impede perf + out = null; + } + +} +class ThreeBitPackingReader extends BaseBitPackingReader { + + private final InputStream in; + private final long valueCount; + + private int buffer = 0; + private int count = 0; + + private long totalRead = 0; + + public ThreeBitPackingReader(InputStream in, long valueCount) { + this.in = in; + this.valueCount = valueCount; + } + + @Override + public int read() throws IOException { + if (count == 0) { + if (valueCount - totalRead < 8) { + buffer = 0; + int bitsToRead = 3 * (int)(valueCount - totalRead); + int bytesToRead = alignToBytes(bitsToRead); + for (int i = 3 - 1 ; i >= 3 - bytesToRead ; i--) { + buffer |= in.read() << (i * 8); + } + count = 8; + totalRead = valueCount; + } else { + buffer = (in.read() << 16) + (in.read() << 8) + in.read(); + count = 8; + totalRead += 8; + } + } + int result = (buffer >> ((count - 1) * 3)) & 7; + -- count; + return result; + } + +} + +class FourBitPackingWriter extends BitPackingWriter { + + private OutputStream out; + + private int buffer = 0; + private int count = 0; + + public FourBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 4; + buffer |= val; + ++ count; + if (count == 2) { + out.write(buffer); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + while (count != 0) { + // downside: this aligns on whatever the buffer size is. + write(0); + } + // check this does not impede perf + out = null; + } + +} +class FourBitPackingReader extends BitPackingReader { + + private final InputStream in; + + private int buffer = 0; + private int count = 0; + + public FourBitPackingReader(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + if (count == 0) { + buffer = in.read(); + count = 2; + } + int result = (buffer >> ((count - 1) * 4)) & 15; + -- count; + return result; + } + +} + +class FiveBitPackingWriter extends BaseBitPackingWriter { + + private OutputStream out; + + private long buffer = 0; + private int count = 0; + + public FiveBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 5; + buffer |= val; + ++ count; + if (count == 8) { + out.write((int)(buffer >>> 32) & 0xFF); + out.write((int)(buffer >>> 24) & 0xFF); + out.write((int)(buffer >>> 16) & 0xFF); + out.write((int)(buffer >>> 8) & 0xFF); + out.write((int)(buffer >>> 0) & 0xFF); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + if (count != 0) { + int numberOfBits = count * 5; + finish(numberOfBits, buffer, out); + buffer = 0; + count = 0; + } + // check this does not impede perf + out = null; + } + +} +class FiveBitPackingReader extends BaseBitPackingReader { + + private final InputStream in; + private final long valueCount; + + private long buffer = 0; + private int count = 0; + private long totalRead = 0; + + + public FiveBitPackingReader(InputStream in, long valueCount) { + this.in = in; + this.valueCount = valueCount; + } + + @Override + public int read() throws IOException { + if (count == 0) { + if (valueCount - totalRead < 8) { + buffer = 0; + int bitsToRead = 5 * (int)(valueCount - totalRead); + int bytesToRead = alignToBytes(bitsToRead); + for (int i = 5 - 1; i >= 5 - bytesToRead ; i--) { + buffer |= (((long)in.read()) & 255) << (i * 8); + } + count = 8; + totalRead = valueCount; + } else { + buffer = + ((((long)in.read()) & 255) << 32) + + ((((long)in.read()) & 255) << 24) + + (in.read() << 16) + + (in.read() << 8) + + in.read(); + count = 8; + totalRead += 8; + } + } + int result = (((int)(buffer >> ((count - 1) * 5))) & 31); + -- count; + return result; + } + +} + +class SixBitPackingWriter extends BaseBitPackingWriter { + + private OutputStream out; + + private int buffer = 0; + private int count = 0; + + public SixBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 6; + buffer |= val; + ++ count; + if (count == 4) { + out.write((buffer >>> 16) & 0xFF); + out.write((buffer >>> 8) & 0xFF); + out.write((buffer >>> 0) & 0xFF); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + if (count != 0) { + int numberOfBits = count * 6; + finish(numberOfBits, buffer, out); + buffer = 0; + count = 0; + } + // check this does not impede perf + out = null; + } + +} +class SixBitPackingReader extends BaseBitPackingReader { + + private final InputStream in; + private final long valueCount; + + private int buffer = 0; + private int count = 0; + + private long totalRead = 0; + + + public SixBitPackingReader(InputStream in, long valueCount) { + this.in = in; + this.valueCount = valueCount; + } + + @Override + public int read() throws IOException { + if (count == 0) { + if (valueCount - totalRead < 4) { + buffer = 0; + int bitsToRead = 6 * (int)(valueCount - totalRead); + int bytesToRead = alignToBytes(bitsToRead); + for (int i = 3 - 1; i >= 3 - bytesToRead ; i--) { + buffer |= in.read() << (i * 8); + } + count = 4; + totalRead = valueCount; + } else { + buffer = (in.read() << 16) + (in.read() << 8) + in.read(); + count = 4; + totalRead += 4; + } + } + int result = (buffer >> ((count - 1) * 6)) & 63; + -- count; + return result; + } + +} + +class SevenBitPackingWriter extends BaseBitPackingWriter { + + private OutputStream out; + + private long buffer = 0; + private int count = 0; + + public SevenBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + buffer = buffer << 7; + buffer |= val; + ++ count; + if (count == 8) { + out.write((int)(buffer >>> 48) & 0xFF); + out.write((int)(buffer >>> 40) & 0xFF); + out.write((int)(buffer >>> 32) & 0xFF); + out.write((int)(buffer >>> 24) & 0xFF); + out.write((int)(buffer >>> 16) & 0xFF); + out.write((int)(buffer >>> 8) & 0xFF); + out.write((int)(buffer >>> 0) & 0xFF); + buffer = 0; + count = 0; + } + } + + @Override + public void finish() throws IOException { + if (count != 0) { + int numberOfBits = count * 7; + finish(numberOfBits, buffer, out); + buffer = 0; + count = 0; + } + // check this does not impede perf + out = null; + } + +} +class SevenBitPackingReader extends BaseBitPackingReader { + + private final InputStream in; + private final long valueCount; + + private long buffer = 0; + private int count = 0; + private long totalRead = 0; + + + public SevenBitPackingReader(InputStream in, long valueCount) { + this.in = in; + this.valueCount = valueCount; + } + + @Override + public int read() throws IOException { + if (count == 0) { + if (valueCount - totalRead < 8) { + buffer = 0; + int bitsToRead = 7 * (int)(valueCount - totalRead); + int bytesToRead = alignToBytes(bitsToRead); + for (int i = 7 - 1; i >= 7 - bytesToRead ; i--) { + buffer |= (((long)in.read()) & 255) << (i * 8); + } + count = 8; + totalRead = valueCount; + } else { + buffer = + ((((long)in.read()) & 255) << 48) + + ((((long)in.read()) & 255) << 40) + + ((((long)in.read()) & 255) << 32) + + ((((long)in.read()) & 255) << 24) + + (in.read() << 16) + + (in.read() << 8) + + in.read(); + count = 8; + totalRead += 8; + } + } + int result = (((int)(buffer >> ((count - 1) * 7))) & 127); + -- count; + return result; + } + +} + +class EightBitPackingWriter extends BitPackingWriter { + + private OutputStream out; + + public EightBitPackingWriter(OutputStream out) { + this.out = out; + } + + @Override + public void write(int val) throws IOException { + out.write(val); + } + + @Override + public void finish() throws IOException { + // check this does not impede perf + out = null; + } + +} +class EightBitPackingReader extends BitPackingReader { + + private final InputStream in; + + public EightBitPackingReader(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + return in.read(); + } + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java new file mode 100644 index 0000000..448c0be --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +import static org.apache.parquet.Log.DEBUG; +import static org.apache.parquet.bytes.BytesInput.concat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.parquet.Log; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; + +/** + * Uses the generated Byte based bit packing to write ints into a BytesInput + * + * @author Julien Le Dem + * + */ +public class ByteBasedBitPackingEncoder { + private static final Log LOG = Log.getLog(ByteBasedBitPackingEncoder.class); + + private static final int VALUES_WRITTEN_AT_A_TIME = 8; + + private final int bitWidth; + private final BytePacker packer; + private final int[] input = new int[VALUES_WRITTEN_AT_A_TIME]; + private final int slabSize; + private int inputSize; + private byte[] packed; + private int packedPosition; + private final List slabs = new ArrayList(); + private int totalValues; + + /** + * @param bitWidth the number of bits used to encode an int + */ + public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) { + this.bitWidth = bitWidth; + this.inputSize = 0; + // must be a multiple of bitWidth + this.slabSize = bitWidth * 64 * 1024; + initPackedSlab(); + this.packer = packer.newBytePacker(bitWidth); + } + + /** + * writes an int using the requested number of bits. + * accepts only value < 2^bitWidth + * @param value the value to write + * @throws IOException + */ + public void writeInt(int value) throws IOException { + input[inputSize] = value; + ++ inputSize; + if (inputSize == VALUES_WRITTEN_AT_A_TIME) { + pack(); + if (packedPosition == slabSize) { + slabs.add(BytesInput.from(packed)); + initPackedSlab(); + } + } + } + + private void pack() { + packer.pack8Values(input, 0, packed, packedPosition); + packedPosition += bitWidth; + totalValues += inputSize; + inputSize = 0; + } + + private void initPackedSlab() { + packed = new byte[slabSize]; + packedPosition = 0; + } + + /** + * @return the bytes representing the packed values + * @throws IOException + */ + public BytesInput toBytes() throws IOException { + int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); + + if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes"); + if (inputSize > 0) { + for (int i = inputSize; i < input.length; i++) { + input[i] = 0; + } + pack(); + } + return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); + } + + /** + * @return size of the data as it would be written + */ + public long getBufferSize() { + return BytesUtils.paddedByteCountFromBits(totalValues * bitWidth); + } + + /** + * @return total memory allocated + */ + public long getAllocatedSize() { + return (slabs.size() * slabSize) + packed.length + input.length * 4; + } + + public String memUsageString(String prefix) { + return String.format("%s ByteBitPacking %d slabs, %d bytes", prefix, slabs.size(), getAllocatedSize()); + } + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java new file mode 100644 index 0000000..b9a37ad --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +/** + * Packs and unpacks into bytes + * + * packing unpacking treats: + * - n values at a time (with n % 8 == 0) + * - bitWidth * (n/8) bytes at a time. + * + * @author Julien Le Dem + * + */ +public abstract class BytePacker { + + private final int bitWidth; + + BytePacker(int bitWidth) { + this.bitWidth = bitWidth; + } + + /** + * @return the width in bits used for encoding, also how many bytes are packed/unpacked at a time by pack8Values/unpack8Values + */ + public final int getBitWidth() { + return bitWidth; + } + + /** + * pack 8 values from input at inPos into bitWidth bytes in output at outPos. + * nextPosition: inPos += 8; outPos += getBitWidth() + * @param input the input values + * @param inPos where to read from in input + * @param output the output bytes + * @param outPos where to write to in output + */ + public abstract void pack8Values(final int[] input, final int inPos, final byte[] output, final int outPos); + + /** + * pack 32 values from input at inPos into bitWidth * 4 bytes in output at outPos. + * nextPosition: inPos += 32; outPos += getBitWidth() * 4 + * @param input the input values + * @param inPos where to read from in input + * @param output the output bytes + * @param outPos where to write to in output + */ + public abstract void pack32Values(int[] input, int inPos, byte[] output, int outPos); + + /** + * unpack bitWidth bytes from input at inPos into 8 values in output at outPos. + * nextPosition: inPos += getBitWidth(); outPos += 8 + * @param input the input bytes + * @param inPos where to read from in input + * @param output the output values + * @param outPos where to write to in output + */ + public abstract void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos); + + /** + * unpack bitWidth * 4 bytes from input at inPos into 32 values in output at outPos. + * nextPosition: inPos += getBitWidth() * 4; outPos += 32 + * @param input the input bytes + * @param inPos where to read from in input + * @param output the output values + * @param outPos where to write to in output + */ + public abstract void unpack32Values(byte[] input, int inPos, int[] output, int outPos); + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java new file mode 100644 index 0000000..fd75461 --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +public interface BytePackerFactory { + + BytePacker newBytePacker(int width); + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java new file mode 100644 index 0000000..1d8d616 --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +/** + * Packs and unpacks into ints + * + * packing unpacking treats: + * - 32 values at a time + * - bitWidth ints at a time. + * + * @author Julien Le Dem + * + */ +public abstract class IntPacker { + + private final int bitWidth; + + IntPacker(int bitWidth) { + this.bitWidth = bitWidth; + } + + /** + * @return the width in bits used for encoding, also how many ints are packed/unpacked at a time + */ + public final int getBitWidth() { + return bitWidth; + } + + /** + * pack 32 values from input at inPos into bitWidth ints in output at outPos. + * nextPosition: inPos += 32; outPos += getBitWidth() + * @param input the input values + * @param inPos where to read from in input + * @param output the output ints + * @param outPos where to write to in output + */ + public abstract void pack32Values(int[] input, int inPos, int[] output, int outPos); + + /** + * unpack bitWidth ints from input at inPos into 32 values in output at outPos. + * nextPosition: inPos += getBitWidth(); outPos += 32 + * @param input the input int + * @param inPos where to read from in input + * @param output the output values + * @param outPos where to write to in output + */ + public abstract void unpack32Values(int[] input, int inPos, int[] output, int outPos); + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java new file mode 100644 index 0000000..a9586bd --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +public interface IntPackerFactory { + + IntPacker newIntPacker(int width); + +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java new file mode 100644 index 0000000..ed14edf --- /dev/null +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.bitpacking; + +/** + * Factory for packing implementations + * + * @author Julien Le Dem + * + */ +public enum Packer { + + /** + * packers who fill the Least Significant Bit First + * int and byte packer have the same result on Big Endian architectures + */ + BIG_ENDIAN { + @Override + public IntPacker newIntPacker(int width) { + return beIntPackerFactory.newIntPacker(width); + } + @Override + public BytePacker newBytePacker(int width) { + return beBytePackerFactory.newBytePacker(width); + } + }, + + /** + * packers who fill the Most Significant Bit first + * int and byte packer have the same result on Little Endian architectures + */ + LITTLE_ENDIAN { + @Override + public IntPacker newIntPacker(int width) { + return leIntPackerFactory.newIntPacker(width); + } + @Override + public BytePacker newBytePacker(int width) { + return leBytePackerFactory.newBytePacker(width); + } + }; + + private static IntPackerFactory getIntPackerFactory(String name) { + return (IntPackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory"); + } + + private static BytePackerFactory getBytePackerFactory(String name) { + return (BytePackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory"); + } + + private static Object getStaticField(String className, String fieldName) { + try { + return Class.forName(className).getField(fieldName).get(null); + } catch (IllegalArgumentException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + static BytePackerFactory beBytePackerFactory = getBytePackerFactory("ByteBitPackingBE"); + static IntPackerFactory beIntPackerFactory = getIntPackerFactory("LemireBitPackingBE"); + static BytePackerFactory leBytePackerFactory = getBytePackerFactory("ByteBitPackingLE"); + static IntPackerFactory leIntPackerFactory = getIntPackerFactory("LemireBitPackingLE"); + + /** + * @param width the width in bits of the packed values + * @return an int based packer + */ + public abstract IntPacker newIntPacker(int width); + + /** + * @param width the width in bits of the packed values + * @return a byte based packer + */ + public abstract BytePacker newBytePacker(int width); +} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java b/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java deleted file mode 100644 index e0d25eb..0000000 --- a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package parquet.bytes; - -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.List; - -import parquet.Log; - - -/** - * A source of bytes capable of writing itself to an output. - * A BytesInput should be consumed right away. - * It is not a container. - * For example if it is referring to a stream, - * subsequent BytesInput reads from the stream will be incorrect - * if the previous has not been consumed. - * - * @author Julien Le Dem - * - */ -abstract public class BytesInput { - private static final Log LOG = Log.getLog(BytesInput.class); - private static final boolean DEBUG = false;//Log.DEBUG; - private static final EmptyBytesInput EMPTY_BYTES_INPUT = new EmptyBytesInput(); - - /** - * logically concatenate the provided inputs - * @param inputs the inputs to concatenate - * @return a concatenated input - */ - public static BytesInput concat(BytesInput... inputs) { - return new SequenceBytesIn(Arrays.asList(inputs)); - } - - /** - * logically concatenate the provided inputs - * @param inputs the inputs to concatenate - * @return a concatenated input - */ - public static BytesInput concat(List inputs) { - return new SequenceBytesIn(inputs); - } - - /** - * @param in - * @param bytes number of bytes to read - * @return a BytesInput that will read that number of bytes from the stream - */ - public static BytesInput from(InputStream in, int bytes) { - return new StreamBytesInput(in, bytes); - } - - /** - * - * @param in - * @return a Bytes input that will write the given bytes - */ - public static BytesInput from(byte[] in) { - if (DEBUG) LOG.debug("BytesInput from array of " + in.length + " bytes"); - return new ByteArrayBytesInput(in, 0 , in.length); - } - - public static BytesInput from(byte[] in, int offset, int length) { - if (DEBUG) LOG.debug("BytesInput from array of " + length + " bytes"); - return new ByteArrayBytesInput(in, offset, length); - } - - /** - * @param intValue the int to write - * @return a BytesInput that will write 4 bytes in little endian - */ - public static BytesInput fromInt(int intValue) { - return new IntBytesInput(intValue); - } - - /** - * @param intValue the int to write - * @return a BytesInput that will write var int - */ - public static BytesInput fromUnsignedVarInt(int intValue) { - return new UnsignedVarIntBytesInput(intValue); - } - - /** - * - * @param intValue the int to write - */ - public static BytesInput fromZigZagVarInt(int intValue) { - int zigZag = (intValue << 1) ^ (intValue >> 31); - return new UnsignedVarIntBytesInput(zigZag); - } - - /** - * @param arrayOut - * @return a BytesInput that will write the content of the buffer - */ - public static BytesInput from(CapacityByteArrayOutputStream arrayOut) { - return new CapacityBAOSBytesInput(arrayOut); - } - - /** - * @param arrayOut - * @return a BytesInput that will write the content of the buffer - */ - public static BytesInput from(ByteArrayOutputStream baos) { - return new BAOSBytesInput(baos); - } - - /** - * @return an empty bytes input - */ - public static BytesInput empty() { - return EMPTY_BYTES_INPUT; - } - - /** - * copies the input into a new byte array - * @param bytesInput - * @return - * @throws IOException - */ - public static BytesInput copy(BytesInput bytesInput) throws IOException { - return from(bytesInput.toByteArray()); - } - - /** - * writes the bytes into a stream - * @param out - * @throws IOException - */ - abstract public void writeAllTo(OutputStream out) throws IOException; - - /** - * - * @return a new byte array materializing the contents of this input - * @throws IOException - */ - public byte[] toByteArray() throws IOException { - BAOS baos = new BAOS((int)size()); - this.writeAllTo(baos); - if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes"); - return baos.getBuf(); - } - - /** - * - * @return the size in bytes that would be written - */ - abstract public long size(); - - private static final class BAOS extends ByteArrayOutputStream { - private BAOS(int size) { - super(size); - } - - public byte[] getBuf() { - return this.buf; - } - } - - private static class StreamBytesInput extends BytesInput { - private static final Log LOG = Log.getLog(BytesInput.StreamBytesInput.class); - private final InputStream in; - private final int byteCount; - - private StreamBytesInput(InputStream in, int byteCount) { - super(); - this.in = in; - this.byteCount = byteCount; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - if (DEBUG) LOG.debug("write All "+ byteCount + " bytes"); - // TODO: more efficient - out.write(this.toByteArray()); - } - - public byte[] toByteArray() throws IOException { - if (DEBUG) LOG.debug("read all "+ byteCount + " bytes"); - byte[] buf = new byte[byteCount]; - new DataInputStream(in).readFully(buf); - return buf; - } - - @Override - public long size() { - return byteCount; - } - - } - - private static class SequenceBytesIn extends BytesInput { - private static final Log LOG = Log.getLog(BytesInput.SequenceBytesIn.class); - - private final List inputs; - private final long size; - - private SequenceBytesIn(List inputs) { - this.inputs = inputs; - long total = 0; - for (BytesInput input : inputs) { - total += input.size(); - } - this.size = total; - } - - @SuppressWarnings("unused") - @Override - public void writeAllTo(OutputStream out) throws IOException { - for (BytesInput input : inputs) { - if (DEBUG) LOG.debug("write " + input.size() + " bytes to out"); - if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("{"); - input.writeAllTo(out); - if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("}"); - } - } - - @Override - public long size() { - return size; - } - - } - - private static class IntBytesInput extends BytesInput { - - private final int intValue; - - public IntBytesInput(int intValue) { - this.intValue = intValue; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - BytesUtils.writeIntLittleEndian(out, intValue); - } - - @Override - public long size() { - return 4; - } - - } - - private static class UnsignedVarIntBytesInput extends BytesInput { - - private final int intValue; - - public UnsignedVarIntBytesInput(int intValue) { - this.intValue = intValue; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - BytesUtils.writeUnsignedVarInt(intValue, out); - } - - @Override - public long size() { - int s = 5 - ((Integer.numberOfLeadingZeros(intValue) + 3) / 7); - return s == 0 ? 1 : s; - } - } - - private static class EmptyBytesInput extends BytesInput { - - @Override - public void writeAllTo(OutputStream out) throws IOException { - } - - @Override - public long size() { - return 0; - } - - } - - private static class CapacityBAOSBytesInput extends BytesInput { - - private final CapacityByteArrayOutputStream arrayOut; - - private CapacityBAOSBytesInput(CapacityByteArrayOutputStream arrayOut) { - this.arrayOut = arrayOut; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - arrayOut.writeTo(out); - } - - @Override - public long size() { - return arrayOut.size(); - } - - } - - private static class BAOSBytesInput extends BytesInput { - - private final ByteArrayOutputStream arrayOut; - - private BAOSBytesInput(ByteArrayOutputStream arrayOut) { - this.arrayOut = arrayOut; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - arrayOut.writeTo(out); - } - - @Override - public long size() { - return arrayOut.size(); - } - - } - - private static class ByteArrayBytesInput extends BytesInput { - - private final byte[] in; - private final int offset; - private final int length; - - private ByteArrayBytesInput(byte[] in, int offset, int length) { - this.in = in; - this.offset = offset; - this.length = length; - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - out.write(in, offset, length); - } - - @Override - public long size() { - return length; - } - - } - -} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java deleted file mode 100644 index d307471..0000000 --- a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package parquet.bytes; - -import static java.lang.Math.max; -import static java.lang.Math.pow; -import static java.lang.String.format; -import static java.lang.System.arraycopy; -import static parquet.Preconditions.checkArgument; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import parquet.Log; - -/** - * Similar to a {@link ByteArrayOutputStream}, but uses a different strategy for growing that does not involve copying. - * Where ByteArrayOutputStream is backed by a single array that "grows" by copying into a new larger array, this output - * stream grows by allocating a new array (slab) and adding it to a list of previous arrays. - * - * Each new slab is allocated to be the same size as all the previous slabs combined, so these allocations become - * exponentially less frequent, just like ByteArrayOutputStream, with one difference. This output stream accepts a - * max capacity hint, which is a hint describing the max amount of data that will be written to this stream. As the - * total size of this stream nears this max, this stream starts to grow linearly instead of exponentially. - * So new slabs are allocated to be 1/5th of the max capacity hint, - * instead of equal to the total previous size of all slabs. This is useful because it prevents allocating roughly - * twice the needed space when a new slab is added just before the stream is done being used. - * - * When reusing this stream it will adjust the initial slab size based on the previous data size, aiming for fewer - * allocations, with the assumption that a similar amount of data will be written to this stream on re-use. - * See ({@link CapacityByteArrayOutputStream#reset()}). - * - * @author Julien Le Dem - * - */ -public class CapacityByteArrayOutputStream extends OutputStream { - private static final Log LOG = Log.getLog(CapacityByteArrayOutputStream.class); - private static final byte[] EMPTY_SLAB = new byte[0]; - - private int initialSlabSize; - private final int maxCapacityHint; - private final List slabs = new ArrayList(); - - private byte[] currentSlab; - private int currentSlabIndex; - private int bytesAllocated = 0; - private int bytesUsed = 0; - - /** - * Return an initial slab size such that a CapacityByteArrayOutputStream constructed with it - * will end up allocating targetNumSlabs in order to reach targetCapacity. This aims to be - * a balance between the overhead of creating new slabs and wasting memory by eagerly making - * initial slabs too big. - * - * Note that targetCapacity here need not match maxCapacityHint in the constructor of - * CapacityByteArrayOutputStream, though often that would make sense. - * - * @param minSlabSize no matter what we shouldn't make slabs any smaller than this - * @param targetCapacity after we've allocated targetNumSlabs how much capacity should we have? - * @param targetNumSlabs how many slabs should it take to reach targetCapacity? - */ - public static int initialSlabSizeHeuristic(int minSlabSize, int targetCapacity, int targetNumSlabs) { - // initialSlabSize = (targetCapacity / (2^targetNumSlabs)) means we double targetNumSlabs times - // before reaching the targetCapacity - // eg for page size of 1MB we start at 1024 bytes. - // we also don't want to start too small, so we also apply a minimum. - return max(minSlabSize, ((int) (targetCapacity / pow(2, targetNumSlabs)))); - } - - /** - * Construct a CapacityByteArrayOutputStream configured such that its initial slab size is - * determined by {@link #initialSlabSizeHeuristic}, with targetCapacity == maxCapacityHint - */ - public static CapacityByteArrayOutputStream withTargetNumSlabs( - int minSlabSize, int maxCapacityHint, int targetNumSlabs) { - - return new CapacityByteArrayOutputStream( - initialSlabSizeHeuristic(minSlabSize, maxCapacityHint, targetNumSlabs), - maxCapacityHint); - } - - /** - * Defaults maxCapacityHint to 1MB - * @param initialSlabSize - * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int)} - */ - @Deprecated - public CapacityByteArrayOutputStream(int initialSlabSize) { - this(initialSlabSize, 1024 * 1024); - } - - /** - * @param initialSlabSize the size to make the first slab - * @param maxCapacityHint a hint (not guarantee) of the max amount of data written to this stream - */ - public CapacityByteArrayOutputStream(int initialSlabSize, int maxCapacityHint) { - checkArgument(initialSlabSize > 0, "initialSlabSize must be > 0"); - checkArgument(maxCapacityHint > 0, "maxCapacityHint must be > 0"); - checkArgument(maxCapacityHint >= initialSlabSize, String.format("maxCapacityHint can't be less than initialSlabSize %d %d", initialSlabSize, maxCapacityHint)); - this.initialSlabSize = initialSlabSize; - this.maxCapacityHint = maxCapacityHint; - reset(); - } - - /** - * the new slab is guaranteed to be at least minimumSize - * @param minimumSize the size of the data we want to copy in the new slab - */ - private void addSlab(int minimumSize) { - int nextSlabSize; - - if (bytesUsed == 0) { - nextSlabSize = initialSlabSize; - } else if (bytesUsed > maxCapacityHint / 5) { - // to avoid an overhead of up to twice the needed size, we get linear when approaching target page size - nextSlabSize = maxCapacityHint / 5; - } else { - // double the size every time - nextSlabSize = bytesUsed; - } - - if (nextSlabSize < minimumSize) { - if (Log.DEBUG) LOG.debug(format("slab size %,d too small for value of size %,d. Bumping up slab size", nextSlabSize, minimumSize)); - nextSlabSize = minimumSize; - } - - if (Log.DEBUG) LOG.debug(format("used %d slabs, adding new slab of size %d", slabs.size(), nextSlabSize)); - - this.currentSlab = new byte[nextSlabSize]; - this.slabs.add(currentSlab); - this.bytesAllocated += nextSlabSize; - this.currentSlabIndex = 0; - } - - @Override - public void write(int b) { - if (currentSlabIndex == currentSlab.length) { - addSlab(1); - } - currentSlab[currentSlabIndex] = (byte) b; - currentSlabIndex += 1; - bytesUsed += 1; - } - - @Override - public void write(byte b[], int off, int len) { - if ((off < 0) || (off > b.length) || (len < 0) || - ((off + len) - b.length > 0)) { - throw new IndexOutOfBoundsException( - String.format("Given byte array of size %d, with requested length(%d) and offset(%d)", b.length, len, off)); - } - if (currentSlabIndex + len >= currentSlab.length) { - final int length1 = currentSlab.length - currentSlabIndex; - arraycopy(b, off, currentSlab, currentSlabIndex, length1); - final int length2 = len - length1; - addSlab(length2); - arraycopy(b, off + length1, currentSlab, currentSlabIndex, length2); - currentSlabIndex = length2; - } else { - arraycopy(b, off, currentSlab, currentSlabIndex, len); - currentSlabIndex += len; - } - bytesUsed += len; - } - - /** - * Writes the complete contents of this buffer to the specified output stream argument. the output - * stream's write method out.write(slab, 0, slab.length)) will be called once per slab. - * - * @param out the output stream to which to write the data. - * @exception IOException if an I/O error occurs. - */ - public void writeTo(OutputStream out) throws IOException { - for (int i = 0; i < slabs.size() - 1; i++) { - final byte[] slab = slabs.get(i); - out.write(slab); - } - out.write(currentSlab, 0, currentSlabIndex); - } - - /** - * @return The total size in bytes of data written to this stream. - */ - public long size() { - return bytesUsed; - } - - /** - * - * @return The total size in bytes currently allocated for this stream. - */ - public int getCapacity() { - return bytesAllocated; - } - - /** - * When re-using an instance with reset, it will adjust slab size based on previous data size. - * The intent is to reuse the same instance for the same type of data (for example, the same column). - * The assumption is that the size in the buffer will be consistent. - */ - public void reset() { - // readjust slab size. - // 7 = 2^3 - 1 so that doubling the initial size 3 times will get to the same size - this.initialSlabSize = max(bytesUsed / 7, initialSlabSize); - if (Log.DEBUG) LOG.debug(String.format("initial slab of size %d", initialSlabSize)); - this.slabs.clear(); - this.bytesAllocated = 0; - this.bytesUsed = 0; - this.currentSlab = EMPTY_SLAB; - this.currentSlabIndex = 0; - } - - /** - * @return the index of the last value written to this stream, which - * can be passed to {@link #setByte(long, byte)} in order to change it - */ - public long getCurrentIndex() { - checkArgument(bytesUsed > 0, "This is an empty stream"); - return bytesUsed - 1; - } - - /** - * Replace the byte stored at position index in this stream with value - * - * @param index which byte to replace - * @param value the value to replace it with - */ - public void setByte(long index, byte value) { - checkArgument(index < bytesUsed, "Index: " + index + " is >= the current size of: " + bytesUsed); - - long seen = 0; - for (int i = 0; i < slabs.size(); i++) { - byte[] slab = slabs.get(i); - if (index < seen + slab.length) { - // ok found index - slab[(int)(index-seen)] = value; - break; - } - seen += slab.length; - } - } - - /** - * @param prefix a prefix to be used for every new line in the string - * @return a text representation of the memory usage of this structure - */ - public String memUsageString(String prefix) { - return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), getCapacity()); - } - - /** - * @return the total number of allocated slabs - */ - int getSlabCount() { - return slabs.size(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java b/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java deleted file mode 100644 index 6a8437b..0000000 --- a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package parquet.bytes; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import static java.lang.String.format; - -public class ConcatenatingByteArrayCollector extends BytesInput { - private final List slabs = new ArrayList(); - private long size = 0; - - public void collect(BytesInput bytesInput) throws IOException { - byte[] bytes = bytesInput.toByteArray(); - slabs.add(bytes); - size += bytes.length; - } - - public void reset() { - size = 0; - slabs.clear(); - } - - @Override - public void writeAllTo(OutputStream out) throws IOException { - for (byte[] slab : slabs) { - out.write(slab); - } - } - - @Override - public long size() { - return size; - } - - /** - * @param prefix a prefix to be used for every new line in the string - * @return a text representation of the memory usage of this structure - */ - public String memUsageString(String prefix) { - return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), size); - } - -} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java deleted file mode 100644 index 6ea9325..0000000 --- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package parquet.bytes; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; - -/** - * Based on DataInputStream but little endian and without the String/char methods - * - * @author Julien Le Dem - * - */ -public final class LittleEndianDataInputStream extends InputStream { - - private final InputStream in; - - /** - * Creates a LittleEndianDataInputStream that uses the specified - * underlying InputStream. - * - * @param in the specified input stream - */ - public LittleEndianDataInputStream(InputStream in) { - this.in = in; - } - - /** - * See the general contract of the readFully - * method of DataInput. - *

- * Bytes - * for this operation are read from the contained - * input stream. - * - * @param b the buffer into which the data is read. - * @exception EOFException if this input stream reaches the end before - * reading all the bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final void readFully(byte b[]) throws IOException { - readFully(b, 0, b.length); - } - - /** - * See the general contract of the readFully - * method of DataInput. - *

- * Bytes - * for this operation are read from the contained - * input stream. - * - * @param b the buffer into which the data is read. - * @param off the start offset of the data. - * @param len the number of bytes to read. - * @exception EOFException if this input stream reaches the end before - * reading all the bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final void readFully(byte b[], int off, int len) throws IOException { - if (len < 0) - throw new IndexOutOfBoundsException(); - int n = 0; - while (n < len) { - int count = in.read(b, off + n, len - n); - if (count < 0) - throw new EOFException(); - n += count; - } - } - - /** - * See the general contract of the skipBytes - * method of DataInput. - *

- * Bytes for this operation are read from the contained - * input stream. - * - * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. - * @exception IOException if the contained input stream does not support - * seek, or the stream has been closed and - * the contained input stream does not support - * reading after close, or another I/O error occurs. - */ - public final int skipBytes(int n) throws IOException { - int total = 0; - int cur = 0; - - while ((total 0)) { - total += cur; - } - - return total; - } - - /** - * @return - * @throws IOException - * @see java.io.InputStream#read() - */ - public int read() throws IOException { - return in.read(); - } - - /** - * @return - * @see java.lang.Object#hashCode() - */ - public int hashCode() { - return in.hashCode(); - } - - /** - * @param b - * @return - * @throws IOException - * @see java.io.InputStream#read(byte[]) - */ - public int read(byte[] b) throws IOException { - return in.read(b); - } - - /** - * @param obj - * @return - * @see java.lang.Object#equals(java.lang.Object) - */ - public boolean equals(Object obj) { - return in.equals(obj); - } - - /** - * @param b - * @param off - * @param len - * @return - * @throws IOException - * @see java.io.InputStream#read(byte[], int, int) - */ - public int read(byte[] b, int off, int len) throws IOException { - return in.read(b, off, len); - } - - /** - * @param n - * @return - * @throws IOException - * @see java.io.InputStream#skip(long) - */ - public long skip(long n) throws IOException { - return in.skip(n); - } - - /** - * @return - * @throws IOException - * @see java.io.InputStream#available() - */ - public int available() throws IOException { - return in.available(); - } - - /** - * @throws IOException - * @see java.io.InputStream#close() - */ - public void close() throws IOException { - in.close(); - } - - /** - * @param readlimit - * @see java.io.InputStream#mark(int) - */ - public void mark(int readlimit) { - in.mark(readlimit); - } - - /** - * @throws IOException - * @see java.io.InputStream#reset() - */ - public void reset() throws IOException { - in.reset(); - } - - /** - * @return - * @see java.io.InputStream#markSupported() - */ - public boolean markSupported() { - return in.markSupported(); - } - - /** - * See the general contract of the readBoolean - * method of DataInput. - *

- * Bytes for this operation are read from the contained - * input stream. - * - * @return the boolean value read. - * @exception EOFException if this input stream has reached the end. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final boolean readBoolean() throws IOException { - int ch = in.read(); - if (ch < 0) - throw new EOFException(); - return (ch != 0); - } - - /** - * See the general contract of the readByte - * method of DataInput. - *

- * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next byte of this input stream as a signed 8-bit - * byte. - * @exception EOFException if this input stream has reached the end. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final byte readByte() throws IOException { - int ch = in.read(); - if (ch < 0) - throw new EOFException(); - return (byte)(ch); - } - - /** - * See the general contract of the readUnsignedByte - * method of DataInput. - *

- * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next byte of this input stream, interpreted as an - * unsigned 8-bit number. - * @exception EOFException if this input stream has reached the end. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final int readUnsignedByte() throws IOException { - int ch = in.read(); - if (ch < 0) - throw new EOFException(); - return ch; - } - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next two bytes of this input stream, interpreted as a - * signed 16-bit number. - * @exception EOFException if this input stream reaches the end before - * reading two bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final short readShort() throws IOException { - int ch2 = in.read(); - int ch1 = in.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (short)((ch1 << 8) + (ch2 << 0)); - } - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next two bytes of this input stream, interpreted as an - * unsigned 16-bit integer. - * @exception EOFException if this input stream reaches the end before - * reading two bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final int readUnsignedShort() throws IOException { - int ch2 = in.read(); - int ch1 = in.read(); - if ((ch1 | ch2) < 0) - throw new EOFException(); - return (ch1 << 8) + (ch2 << 0); - } - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next four bytes of this input stream, interpreted as an - * int. - * @exception EOFException if this input stream reaches the end before - * reading four bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final int readInt() throws IOException { - // TODO: has this been benchmarked against two alternate implementations? - // 1) Integer.reverseBytes(in.readInt()) - // 2) keep a member byte[4], wrapped by an IntBuffer with appropriate endianness set, - // and call IntBuffer.get() - // Both seem like they might be faster. - int ch4 = in.read(); - int ch3 = in.read(); - int ch2 = in.read(); - int ch1 = in.read(); - if ((ch1 | ch2 | ch3 | ch4) < 0) - throw new EOFException(); - return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0)); - } - - private byte readBuffer[] = new byte[8]; - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next eight bytes of this input stream, interpreted as a - * long. - * @exception EOFException if this input stream reaches the end before - * reading eight bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.io.FilterInputStream#in - */ - public final long readLong() throws IOException { - // TODO: see perf question above in readInt - readFully(readBuffer, 0, 8); - return (((long)readBuffer[7] << 56) + - ((long)(readBuffer[6] & 255) << 48) + - ((long)(readBuffer[5] & 255) << 40) + - ((long)(readBuffer[4] & 255) << 32) + - ((long)(readBuffer[3] & 255) << 24) + - ((readBuffer[2] & 255) << 16) + - ((readBuffer[1] & 255) << 8) + - ((readBuffer[0] & 255) << 0)); - } - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next four bytes of this input stream, interpreted as a - * float. - * @exception EOFException if this input stream reaches the end before - * reading four bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.lang.Float#intBitsToFloat(int) - */ - public final float readFloat() throws IOException { - return Float.intBitsToFloat(readInt()); - } - - /** - * Bytes - * for this operation are read from the contained - * input stream. - * - * @return the next eight bytes of this input stream, interpreted as a - * double. - * @exception EOFException if this input stream reaches the end before - * reading eight bytes. - * @exception IOException the stream has been closed and the contained - * input stream does not support reading after close, or - * another I/O error occurs. - * @see java.lang.Double#longBitsToDouble(long) - */ - public final double readDouble() throws IOException { - return Double.longBitsToDouble(readLong()); - } - -} http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java ---------------------------------------------------------------------- diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java deleted file mode 100644 index 2be7299..0000000 --- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package parquet.bytes; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * Based on DataOutputStream but in little endian and without the String/char methods - * - * @author Julien Le Dem - * - */ -public class LittleEndianDataOutputStream extends OutputStream { - - private final OutputStream out; - - /** - * Creates a new data output stream to write data to the specified - * underlying output stream. The counter written is - * set to zero. - * - * @param out the underlying output stream, to be saved for later - * use. - * @see java.io.FilterOutputStream#out - */ - public LittleEndianDataOutputStream(OutputStream out) { - this.out = out; - } - - /** - * Writes the specified byte (the low eight bits of the argument - * b) to the underlying output stream. If no exception - * is thrown, the counter written is incremented by - * 1. - *

- * Implements the write method of OutputStream. - * - * @param b the byte to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public void write(int b) throws IOException { - out.write(b); - } - - /** - * Writes len bytes from the specified byte array - * starting at offset off to the underlying output stream. - * If no exception is thrown, the counter written is - * incremented by len. - * - * @param b the data. - * @param off the start offset in the data. - * @param len the number of bytes to write. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public void write(byte b[], int off, int len) throws IOException { - out.write(b, off, len); - } - - /** - * Flushes this data output stream. This forces any buffered output - * bytes to be written out to the stream. - *

- * The flush method of DataOutputStream - * calls the flush method of its underlying output stream. - * - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - * @see java.io.OutputStream#flush() - */ - public void flush() throws IOException { - out.flush(); - } - - /** - * Writes a boolean to the underlying output stream as - * a 1-byte value. The value true is written out as the - * value (byte)1; the value false is - * written out as the value (byte)0. If no exception is - * thrown, the counter written is incremented by - * 1. - * - * @param v a boolean value to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public final void writeBoolean(boolean v) throws IOException { - out.write(v ? 1 : 0); - } - - /** - * Writes out a byte to the underlying output stream as - * a 1-byte value. If no exception is thrown, the counter - * written is incremented by 1. - * - * @param v a byte value to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public final void writeByte(int v) throws IOException { - out.write(v); - } - - /** - * Writes a short to the underlying output stream as two - * bytes, low byte first. If no exception is thrown, the counter - * written is incremented by 2. - * - * @param v a short to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public final void writeShort(int v) throws IOException { - out.write((v >>> 0) & 0xFF); - out.write((v >>> 8) & 0xFF); - } - - /** - * Writes an int to the underlying output stream as four - * bytes, low byte first. If no exception is thrown, the counter - * written is incremented by 4. - * - * @param v an int to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public final void writeInt(int v) throws IOException { - // TODO: see note in LittleEndianDataInputStream: maybe faster - // to use Integer.reverseBytes() and then writeInt, or a ByteBuffer - // approach - out.write((v >>> 0) & 0xFF); - out.write((v >>> 8) & 0xFF); - out.write((v >>> 16) & 0xFF); - out.write((v >>> 24) & 0xFF); - } - - private byte writeBuffer[] = new byte[8]; - - /** - * Writes a long to the underlying output stream as eight - * bytes, low byte first. In no exception is thrown, the counter - * written is incremented by 8. - * - * @param v a long to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - */ - public final void writeLong(long v) throws IOException { - writeBuffer[7] = (byte)(v >>> 56); - writeBuffer[6] = (byte)(v >>> 48); - writeBuffer[5] = (byte)(v >>> 40); - writeBuffer[4] = (byte)(v >>> 32); - writeBuffer[3] = (byte)(v >>> 24); - writeBuffer[2] = (byte)(v >>> 16); - writeBuffer[1] = (byte)(v >>> 8); - writeBuffer[0] = (byte)(v >>> 0); - out.write(writeBuffer, 0, 8); - } - - /** - * Converts the float argument to an int using the - * floatToIntBits method in class Float, - * and then writes that int value to the underlying - * output stream as a 4-byte quantity, low byte first. If no - * exception is thrown, the counter written is - * incremented by 4. - * - * @param v a float value to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - * @see java.lang.Float#floatToIntBits(float) - */ - public final void writeFloat(float v) throws IOException { - writeInt(Float.floatToIntBits(v)); - } - - /** - * Converts the double argument to a long using the - * doubleToLongBits method in class Double, - * and then writes that long value to the underlying - * output stream as an 8-byte quantity, low byte first. If no - * exception is thrown, the counter written is - * incremented by 8. - * - * @param v a double value to be written. - * @exception IOException if an I/O error occurs. - * @see java.io.FilterOutputStream#out - * @see java.lang.Double#doubleToLongBits(double) - */ - public final void writeDouble(double v) throws IOException { - writeLong(Double.doubleToLongBits(v)); - } - -}