parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [18/51] [partial] parquet-mr git commit: PARQUET-23: Rename to org.apache.parquet.
Date Mon, 27 Apr 2015 23:12:15 GMT
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
new file mode 100644
index 0000000..c0acd11
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BitPacking.java
@@ -0,0 +1,725 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
+
+// TODO: rework the whole thing. It does not need to use streams at all
+/**
+ * provides the correct implementation of a bitpacking based on the width in bits
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class BitPacking {
+
+  /**
+   * to writes ints to a stream packed to only the needed bits.
+   * there is no guarantee of corecteness if ints larger than the max size are written
+   *
+   * @author Julien Le Dem
+   *
+   */
+  abstract public static class BitPackingWriter {
+    /**
+     * will write the bits to the underlying stream aligned on the buffer size
+     * @param val the value to encode
+     * @throws IOException
+     */
+    abstract public void write(int val) throws IOException;
+
+    /**
+     * will flush the buffer to the underlying stream (and pad with 0s)
+     * @throws IOException
+     */
+    abstract public void finish() throws IOException;
+  }
+
+  /**
+   * to read back what has been written with the corresponding  writer
+   *
+   * @author Julien Le Dem
+   *
+   */
+  abstract public static class BitPackingReader {
+
+    /**
+     *
+     * @return and int decoded from the underlying stream
+     * @throws IOException
+     */
+    abstract public int read() throws IOException;
+  }
+
+  private BitPacking() {
+  }
+
+  /**
+   * @param bitLength the width in bits of the integers to write
+   * @param out the stream to write the bytes to
+   * @return the correct implementation for the width
+   */
+  public static BitPackingWriter getBitPackingWriter(int bitLength, OutputStream out) {
+    switch (bitLength) {
+    case 0:
+      return new ZeroBitPackingWriter();
+    case 1:
+      return new OneBitPackingWriter(out);
+    case 2:
+      return new TwoBitPackingWriter(out);
+    case 3:
+      return new ThreeBitPackingWriter(out);
+    case 4:
+      return new FourBitPackingWriter(out);
+    case 5:
+      return new FiveBitPackingWriter(out);
+    case 6:
+      return new SixBitPackingWriter(out);
+    case 7:
+      return new SevenBitPackingWriter(out);
+    case 8:
+      return new EightBitPackingWriter(out);
+    default:
+      throw new UnsupportedOperationException("only support up to 8 for now");
+    }
+  }
+
+  /**
+   *
+   * @param bitLength the width in bits of the integers to read
+   * @param inthe stream to read the bytes from
+   * @return the correct implementation for the width
+   */
+  public static BitPackingReader createBitPackingReader(int bitLength, InputStream in, long valueCount) {
+    switch (bitLength) {
+    case 0:
+      return new ZeroBitPackingReader();
+    case 1:
+      return new OneBitPackingReader(in);
+    case 2:
+      return new TwoBitPackingReader(in);
+    case 3:
+      return new ThreeBitPackingReader(in, valueCount);
+    case 4:
+      return new FourBitPackingReader(in);
+    case 5:
+      return new FiveBitPackingReader(in, valueCount);
+    case 6:
+      return new SixBitPackingReader(in, valueCount);
+    case 7:
+      return new SevenBitPackingReader(in, valueCount);
+    case 8:
+      return new EightBitPackingReader(in);
+    default:
+      throw new UnsupportedOperationException("only support up to 8 for now");
+    }
+  }
+}
+
+abstract class BaseBitPackingWriter extends BitPackingWriter {
+
+  void finish(int numberOfBits, int buffer, OutputStream out) throws IOException {
+    int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
+    buffer = buffer << padding;
+    int numberOfBytes = (numberOfBits + padding) / 8;
+    for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
+      out.write((buffer >>> i) & 0xFF);
+    }
+  }
+
+  void finish(int numberOfBits, long buffer, OutputStream out) throws IOException {
+    int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
+    buffer = buffer << padding;
+    int numberOfBytes = (numberOfBits + padding) / 8;
+    for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
+      out.write((int)(buffer >>> i) & 0xFF);
+    }
+  }
+}
+abstract class BaseBitPackingReader extends BitPackingReader {
+
+  int alignToBytes(int bitsCount) {
+    return BytesUtils.paddedByteCountFromBits(bitsCount);
+  }
+
+}
+
+class ZeroBitPackingWriter extends BitPackingWriter {
+
+  @Override
+  public void write(int val) throws IOException {
+  }
+
+  @Override
+  public void finish() {
+  }
+
+}
+class ZeroBitPackingReader extends BitPackingReader {
+
+  @Override
+  public int read() throws IOException {
+    return 0;
+  }
+
+}
+
+class OneBitPackingWriter extends BitPackingWriter {
+
+  private OutputStream out;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public OneBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 1;
+    buffer |= val;
+    ++ count;
+    if (count == 8) {
+      out.write(buffer);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    while (count != 0) {
+      write(0);
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class OneBitPackingReader extends BitPackingReader {
+
+  private final InputStream in;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public OneBitPackingReader(InputStream in) {
+    this.in = in;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      buffer = in.read();
+      count = 8;
+    }
+    int result = (buffer >> (count - 1)) & 1;
+    -- count;
+    return result;
+  }
+
+}
+
+class TwoBitPackingWriter extends BitPackingWriter {
+
+  private OutputStream out;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public TwoBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 2;
+    buffer |= val;
+    ++ count;
+    if (count == 4) {
+      out.write(buffer);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    while (count != 0) {
+      write(0);
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class TwoBitPackingReader extends BitPackingReader {
+
+  private final InputStream in;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public TwoBitPackingReader(InputStream in) {
+    this.in = in;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      buffer = in.read();
+      count = 4;
+    }
+    int result = (buffer >> ((count - 1) * 2)) & 3;
+    -- count;
+    return result;
+  }
+
+}
+
+class ThreeBitPackingWriter extends BaseBitPackingWriter {
+
+  private OutputStream out;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public ThreeBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 3;
+    buffer |= val;
+    ++ count;
+    if (count == 8) {
+      out.write((buffer >>> 16) & 0xFF);
+      out.write((buffer >>>  8) & 0xFF);
+      out.write((buffer >>>  0) & 0xFF);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    if (count != 0) {
+      int numberOfBits = count * 3;
+      finish(numberOfBits, buffer, out);
+      buffer = 0;
+      count = 0;
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class ThreeBitPackingReader extends BaseBitPackingReader {
+
+  private final InputStream in;
+  private final long valueCount;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  private long totalRead = 0;
+
+  public ThreeBitPackingReader(InputStream in, long valueCount) {
+    this.in = in;
+    this.valueCount = valueCount;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      if (valueCount - totalRead < 8) {
+        buffer = 0;
+        int bitsToRead = 3 * (int)(valueCount - totalRead);
+        int bytesToRead = alignToBytes(bitsToRead);
+        for (int i = 3 - 1 ; i >= 3 - bytesToRead ; i--) {
+          buffer |= in.read() << (i * 8);
+        }
+        count = 8;
+        totalRead = valueCount;
+      } else {
+        buffer = (in.read() << 16) + (in.read() << 8) + in.read();
+        count = 8;
+        totalRead += 8;
+      }
+    }
+    int result = (buffer >> ((count - 1) * 3)) & 7;
+    -- count;
+    return result;
+  }
+
+}
+
+class FourBitPackingWriter extends BitPackingWriter {
+
+  private OutputStream out;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public FourBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 4;
+    buffer |= val;
+    ++ count;
+    if (count == 2) {
+      out.write(buffer);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    while (count != 0) {
+      // downside: this aligns on whatever the buffer size is.
+      write(0);
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class FourBitPackingReader extends BitPackingReader {
+
+  private final InputStream in;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public FourBitPackingReader(InputStream in) {
+    this.in = in;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      buffer = in.read();
+      count = 2;
+    }
+    int result = (buffer >> ((count - 1) * 4)) & 15;
+    -- count;
+    return result;
+  }
+
+}
+
+class FiveBitPackingWriter extends BaseBitPackingWriter {
+
+  private OutputStream out;
+
+  private long buffer = 0;
+  private int count = 0;
+
+  public FiveBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 5;
+    buffer |= val;
+    ++ count;
+    if (count == 8) {
+      out.write((int)(buffer >>> 32) & 0xFF);
+      out.write((int)(buffer >>> 24) & 0xFF);
+      out.write((int)(buffer >>> 16) & 0xFF);
+      out.write((int)(buffer >>>  8) & 0xFF);
+      out.write((int)(buffer >>>  0) & 0xFF);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    if (count != 0) {
+      int numberOfBits = count * 5;
+      finish(numberOfBits, buffer, out);
+      buffer = 0;
+      count = 0;
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class FiveBitPackingReader extends BaseBitPackingReader {
+
+  private final InputStream in;
+  private final long valueCount;
+
+  private long buffer = 0;
+  private int count = 0;
+  private long totalRead = 0;
+
+
+  public FiveBitPackingReader(InputStream in, long valueCount) {
+    this.in = in;
+    this.valueCount = valueCount;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      if (valueCount - totalRead < 8) {
+        buffer = 0;
+        int bitsToRead = 5 * (int)(valueCount - totalRead);
+        int bytesToRead = alignToBytes(bitsToRead);
+        for (int i = 5 - 1; i >= 5 - bytesToRead ; i--) {
+          buffer |= (((long)in.read()) & 255) << (i * 8);
+        }
+        count = 8;
+        totalRead = valueCount;
+      } else {
+        buffer =
+            ((((long)in.read()) & 255) << 32)
+            + ((((long)in.read()) & 255) << 24)
+            + (in.read() << 16)
+            + (in.read() << 8)
+            + in.read();
+        count = 8;
+        totalRead += 8;
+      }
+    }
+    int result = (((int)(buffer >> ((count - 1) * 5))) & 31);
+    -- count;
+    return result;
+  }
+
+}
+
+class SixBitPackingWriter extends BaseBitPackingWriter {
+
+  private OutputStream out;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  public SixBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 6;
+    buffer |= val;
+    ++ count;
+    if (count == 4) {
+      out.write((buffer >>> 16) & 0xFF);
+      out.write((buffer >>>  8) & 0xFF);
+      out.write((buffer >>>  0) & 0xFF);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    if (count != 0) {
+      int numberOfBits = count * 6;
+      finish(numberOfBits, buffer, out);
+      buffer = 0;
+      count = 0;
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class SixBitPackingReader extends BaseBitPackingReader {
+
+  private final InputStream in;
+  private final long valueCount;
+
+  private int buffer = 0;
+  private int count = 0;
+
+  private long totalRead = 0;
+
+
+  public SixBitPackingReader(InputStream in, long valueCount) {
+    this.in = in;
+    this.valueCount = valueCount;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      if (valueCount - totalRead < 4) {
+        buffer = 0;
+        int bitsToRead = 6 * (int)(valueCount - totalRead);
+        int bytesToRead = alignToBytes(bitsToRead);
+        for (int i = 3 - 1; i >= 3 - bytesToRead ; i--) {
+          buffer |= in.read() << (i * 8);
+        }
+        count = 4;
+        totalRead = valueCount;
+      } else {
+        buffer = (in.read() << 16) + (in.read() << 8) + in.read();
+        count = 4;
+        totalRead += 4;
+      }
+    }
+    int result = (buffer >> ((count - 1) * 6)) & 63;
+    -- count;
+    return result;
+  }
+
+}
+
+class SevenBitPackingWriter extends BaseBitPackingWriter {
+
+  private OutputStream out;
+
+  private long buffer = 0;
+  private int count = 0;
+
+  public SevenBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    buffer = buffer << 7;
+    buffer |= val;
+    ++ count;
+    if (count == 8) {
+      out.write((int)(buffer >>> 48) & 0xFF);
+      out.write((int)(buffer >>> 40) & 0xFF);
+      out.write((int)(buffer >>> 32) & 0xFF);
+      out.write((int)(buffer >>> 24) & 0xFF);
+      out.write((int)(buffer >>> 16) & 0xFF);
+      out.write((int)(buffer >>>  8) & 0xFF);
+      out.write((int)(buffer >>>  0) & 0xFF);
+      buffer = 0;
+      count = 0;
+    }
+  }
+
+  @Override
+  public void finish() throws IOException {
+    if (count != 0) {
+      int numberOfBits = count * 7;
+      finish(numberOfBits, buffer, out);
+      buffer = 0;
+      count = 0;
+    }
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class SevenBitPackingReader extends BaseBitPackingReader {
+
+  private final InputStream in;
+  private final long valueCount;
+
+  private long buffer = 0;
+  private int count = 0;
+  private long totalRead = 0;
+
+
+  public SevenBitPackingReader(InputStream in, long valueCount) {
+    this.in = in;
+    this.valueCount = valueCount;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (count == 0) {
+      if (valueCount - totalRead  < 8) {
+        buffer = 0;
+        int bitsToRead = 7 * (int)(valueCount - totalRead);
+        int bytesToRead = alignToBytes(bitsToRead);
+        for (int i = 7 - 1; i >= 7 - bytesToRead ; i--) {
+          buffer |= (((long)in.read()) & 255) << (i * 8);
+        }
+        count = 8;
+        totalRead = valueCount;
+      } else {
+        buffer =
+            ((((long)in.read()) & 255) << 48)
+            + ((((long)in.read()) & 255) << 40)
+            + ((((long)in.read()) & 255) << 32)
+            + ((((long)in.read()) & 255) << 24)
+            + (in.read() << 16)
+            + (in.read() << 8)
+            + in.read();
+        count = 8;
+        totalRead += 8;
+      }
+    }
+    int result = (((int)(buffer >> ((count - 1) * 7))) & 127);
+    -- count;
+    return result;
+  }
+
+}
+
+class EightBitPackingWriter extends BitPackingWriter {
+
+  private OutputStream out;
+
+  public EightBitPackingWriter(OutputStream out) {
+    this.out = out;
+  }
+
+  @Override
+  public void write(int val) throws IOException {
+    out.write(val);
+  }
+
+  @Override
+  public void finish() throws IOException {
+    // check this does not impede perf
+    out = null;
+  }
+
+}
+class EightBitPackingReader extends BitPackingReader {
+
+  private final InputStream in;
+
+  public EightBitPackingReader(InputStream in) {
+    this.in = in;
+  }
+
+  @Override
+  public int read() throws IOException {
+    return in.read();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
new file mode 100644
index 0000000..448c0be
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
@@ -0,0 +1,130 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import static org.apache.parquet.Log.DEBUG;
+import static org.apache.parquet.bytes.BytesInput.concat;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.parquet.Log;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+
+/**
+ * Uses the generated Byte based bit packing to write ints into a BytesInput
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class ByteBasedBitPackingEncoder {
+  private static final Log LOG = Log.getLog(ByteBasedBitPackingEncoder.class);
+
+  private static final int VALUES_WRITTEN_AT_A_TIME = 8;
+
+  private final int bitWidth;
+  private final BytePacker packer;
+  private final int[] input = new int[VALUES_WRITTEN_AT_A_TIME];
+  private final int slabSize;
+  private int inputSize;
+  private byte[] packed;
+  private int packedPosition;
+  private final List<BytesInput> slabs = new ArrayList<BytesInput>();
+  private int totalValues;
+
+  /**
+   * @param bitWidth the number of bits used to encode an int
+   */
+  public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) {
+    this.bitWidth = bitWidth;
+    this.inputSize = 0;
+    // must be a multiple of bitWidth
+    this.slabSize = bitWidth * 64 * 1024;
+    initPackedSlab();
+    this.packer = packer.newBytePacker(bitWidth);
+  }
+
+  /**
+   * writes an int using the requested number of bits.
+   * accepts only value < 2^bitWidth
+   * @param value the value to write
+   * @throws IOException
+   */
+  public void writeInt(int value) throws IOException {
+    input[inputSize] = value;
+    ++ inputSize;
+    if (inputSize == VALUES_WRITTEN_AT_A_TIME) {
+      pack();
+      if (packedPosition == slabSize) {
+        slabs.add(BytesInput.from(packed));
+        initPackedSlab();
+      }
+    }
+  }
+
+  private void pack() {
+    packer.pack8Values(input, 0, packed, packedPosition);
+    packedPosition += bitWidth;
+    totalValues += inputSize;
+    inputSize = 0;
+  }
+
+  private void initPackedSlab() {
+    packed = new byte[slabSize];
+    packedPosition = 0;
+  }
+
+  /**
+   * @return the bytes representing the packed values
+   * @throws IOException
+   */
+  public BytesInput toBytes() throws IOException {
+    int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth);
+
+    if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes");
+    if (inputSize > 0) {
+      for (int i = inputSize; i < input.length; i++) {
+        input[i] = 0;
+      }
+      pack();
+    }
+    return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength));
+  }
+
+  /**
+   * @return size of the data as it would be written
+   */
+  public long getBufferSize() {
+    return BytesUtils.paddedByteCountFromBits(totalValues * bitWidth);
+  }
+
+  /**
+   * @return total memory allocated
+   */
+  public long getAllocatedSize() {
+    return (slabs.size() * slabSize) + packed.length + input.length * 4;
+  }
+
+  public String memUsageString(String prefix) {
+    return String.format("%s ByteBitPacking %d slabs, %d bytes", prefix, slabs.size(), getAllocatedSize());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
new file mode 100644
index 0000000..b9a37ad
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePacker.java
@@ -0,0 +1,86 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Packs and unpacks into bytes
+ *
+ * packing unpacking treats:
+ *  - n values at a time (with n % 8 == 0)
+ *  - bitWidth * (n/8) bytes at a time.
+ *
+ * @author Julien Le Dem
+ *
+ */
+public abstract class BytePacker {
+
+  private final int bitWidth;
+
+  BytePacker(int bitWidth) {
+    this.bitWidth = bitWidth;
+  }
+
+  /**
+   * @return the width in bits used for encoding, also how many bytes are packed/unpacked at a time by pack8Values/unpack8Values
+   */
+  public final int getBitWidth() {
+    return bitWidth;
+  }
+
+  /**
+   * pack 8 values from input at inPos into bitWidth bytes in output at outPos.
+   * nextPosition: inPos += 8; outPos += getBitWidth()
+   * @param input the input values
+   * @param inPos where to read from in input
+   * @param output the output bytes
+   * @param outPos where to write to in output
+   */
+  public abstract void pack8Values(final int[] input, final int inPos, final byte[] output, final int outPos);
+
+  /**
+   * pack 32 values from input at inPos into bitWidth * 4 bytes in output at outPos.
+   * nextPosition: inPos += 32; outPos += getBitWidth() * 4
+   * @param input the input values
+   * @param inPos where to read from in input
+   * @param output the output bytes
+   * @param outPos where to write to in output
+   */
+  public abstract void pack32Values(int[] input, int inPos, byte[] output, int outPos);
+
+  /**
+   * unpack bitWidth bytes from input at inPos into 8 values in output at outPos.
+   * nextPosition: inPos += getBitWidth(); outPos += 8
+   * @param input the input bytes
+   * @param inPos where to read from in input
+   * @param output the output values
+   * @param outPos where to write to in output
+   */
+  public abstract void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos);
+
+  /**
+   * unpack bitWidth * 4 bytes from input at inPos into 32 values in output at outPos.
+   * nextPosition: inPos += getBitWidth() * 4; outPos += 32
+   * @param input the input bytes
+   * @param inPos where to read from in input
+   * @param output the output values
+   * @param outPos where to write to in output
+   */
+  public abstract void unpack32Values(byte[] input, int inPos, int[] output, int outPos);
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
new file mode 100644
index 0000000..fd75461
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/BytePackerFactory.java
@@ -0,0 +1,25 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+public interface BytePackerFactory {
+
+  BytePacker newBytePacker(int width);
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
new file mode 100644
index 0000000..1d8d616
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPacker.java
@@ -0,0 +1,66 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Packs and unpacks into ints
+ *
+ * packing unpacking treats:
+ *  - 32 values at a time
+ *  - bitWidth ints at a time.
+ *
+ * @author Julien Le Dem
+ *
+ */
+public abstract class IntPacker {
+
+  private final int bitWidth;
+
+  IntPacker(int bitWidth) {
+    this.bitWidth = bitWidth;
+  }
+
+  /**
+   * @return the width in bits used for encoding, also how many ints are packed/unpacked at a time
+   */
+  public final int getBitWidth() {
+    return bitWidth;
+  }
+
+  /**
+   * pack 32 values from input at inPos into bitWidth ints in output at outPos.
+   * nextPosition: inPos += 32; outPos += getBitWidth()
+   * @param input the input values
+   * @param inPos where to read from in input
+   * @param output the output ints
+   * @param outPos where to write to in output
+   */
+  public abstract void pack32Values(int[] input, int inPos, int[] output, int outPos);
+
+  /**
+   * unpack bitWidth ints from input at inPos into 32 values in output at outPos.
+   * nextPosition: inPos += getBitWidth(); outPos += 32
+   * @param input the input int
+   * @param inPos where to read from in input
+   * @param output the output values
+   * @param outPos where to write to in output
+   */
+  public abstract void unpack32Values(int[] input, int inPos, int[] output, int outPos);
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
new file mode 100644
index 0000000..a9586bd
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/IntPackerFactory.java
@@ -0,0 +1,25 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+public interface IntPackerFactory {
+
+  IntPacker newIntPacker(int width);
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
new file mode 100644
index 0000000..ed14edf
--- /dev/null
+++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/Packer.java
@@ -0,0 +1,99 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+/**
+ * Factory for packing implementations
+ *
+ * @author Julien Le Dem
+ *
+ */
+public enum Packer {
+
+  /**
+   * packers who fill the Least Significant Bit First
+   * int and byte packer have the same result on Big Endian architectures
+   */
+  BIG_ENDIAN {
+    @Override
+    public IntPacker newIntPacker(int width) {
+      return beIntPackerFactory.newIntPacker(width);
+    }
+    @Override
+    public BytePacker newBytePacker(int width) {
+      return beBytePackerFactory.newBytePacker(width);
+    }
+  },
+
+  /**
+   * packers who fill the Most Significant Bit first
+   * int and byte packer have the same result on Little Endian architectures
+   */
+  LITTLE_ENDIAN {
+    @Override
+    public IntPacker newIntPacker(int width) {
+      return leIntPackerFactory.newIntPacker(width);
+    }
+    @Override
+    public BytePacker newBytePacker(int width) {
+      return leBytePackerFactory.newBytePacker(width);
+    }
+  };
+
+  private static IntPackerFactory getIntPackerFactory(String name) {
+    return (IntPackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory");
+  }
+
+  private static BytePackerFactory getBytePackerFactory(String name) {
+    return (BytePackerFactory)getStaticField("org.apache.parquet.column.values.bitpacking." + name, "factory");
+  }
+
+  private static Object getStaticField(String className, String fieldName) {
+    try {
+      return Class.forName(className).getField(fieldName).get(null);
+    } catch (IllegalArgumentException e) {
+      throw new RuntimeException(e);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e);
+    } catch (NoSuchFieldException e) {
+      throw new RuntimeException(e);
+    } catch (SecurityException e) {
+      throw new RuntimeException(e);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  static BytePackerFactory beBytePackerFactory = getBytePackerFactory("ByteBitPackingBE");
+  static IntPackerFactory beIntPackerFactory = getIntPackerFactory("LemireBitPackingBE");
+  static BytePackerFactory leBytePackerFactory = getBytePackerFactory("ByteBitPackingLE");
+  static IntPackerFactory leIntPackerFactory = getIntPackerFactory("LemireBitPackingLE");
+
+  /**
+   * @param width the width in bits of the packed values
+   * @return an int based packer
+   */
+  public abstract IntPacker newIntPacker(int width);
+
+  /**
+   * @param width the width in bits of the packed values
+   * @return a byte based packer
+   */
+  public abstract BytePacker newBytePacker(int width);
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java b/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
deleted file mode 100644
index e0d25eb..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/BytesInput.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.List;
-
-import parquet.Log;
-
-
-/**
- * A source of bytes capable of writing itself to an output.
- * A BytesInput should be consumed right away.
- * It is not a container.
- * For example if it is referring to a stream,
- * subsequent BytesInput reads from the stream will be incorrect
- * if the previous has not been consumed.
- *
- * @author Julien Le Dem
- *
- */
-abstract public class BytesInput {
-  private static final Log LOG = Log.getLog(BytesInput.class);
-  private static final boolean DEBUG = false;//Log.DEBUG;
-  private static final EmptyBytesInput EMPTY_BYTES_INPUT = new EmptyBytesInput();
-
-  /**
-   * logically concatenate the provided inputs
-   * @param inputs the inputs to concatenate
-   * @return a concatenated input
-   */
-  public static BytesInput concat(BytesInput... inputs) {
-    return new SequenceBytesIn(Arrays.asList(inputs));
-  }
-
-  /**
-   * logically concatenate the provided inputs
-   * @param inputs the inputs to concatenate
-   * @return a concatenated input
-   */
-  public static BytesInput concat(List<BytesInput> inputs) {
-    return new SequenceBytesIn(inputs);
-  }
-
-  /**
-   * @param in
-   * @param bytes number of bytes to read
-   * @return a BytesInput that will read that number of bytes from the stream
-   */
-  public static BytesInput from(InputStream in, int bytes) {
-    return new StreamBytesInput(in, bytes);
-  }
-
-  /**
-   *
-   * @param in
-   * @return a Bytes input that will write the given bytes
-   */
-  public static BytesInput from(byte[] in) {
-    if (DEBUG) LOG.debug("BytesInput from array of " + in.length + " bytes");
-    return new ByteArrayBytesInput(in, 0 , in.length);
-  }
-
-  public static BytesInput from(byte[] in, int offset, int length) {
-    if (DEBUG) LOG.debug("BytesInput from array of " + length + " bytes");
-    return new ByteArrayBytesInput(in, offset, length);
-  }
-
-  /**
-   * @param intValue the int to write
-   * @return a BytesInput that will write 4 bytes in little endian
-   */
-  public static BytesInput fromInt(int intValue) {
-    return new IntBytesInput(intValue);
-  }
-
-  /**
-   * @param intValue the int to write
-   * @return a BytesInput that will write var int
-   */
-  public static BytesInput fromUnsignedVarInt(int intValue) {
-    return new UnsignedVarIntBytesInput(intValue);
-  }
-
-  /**
-   *
-   * @param intValue the int to write
-   */
-  public static BytesInput fromZigZagVarInt(int intValue) {
-    int zigZag = (intValue << 1) ^ (intValue >> 31);
-    return new UnsignedVarIntBytesInput(zigZag);
-  }
-
-  /**
-   * @param arrayOut
-   * @return a BytesInput that will write the content of the buffer
-   */
-  public static BytesInput from(CapacityByteArrayOutputStream arrayOut) {
-    return new CapacityBAOSBytesInput(arrayOut);
-  }
-
-  /**
-   * @param arrayOut
-   * @return a BytesInput that will write the content of the buffer
-   */
-  public static BytesInput from(ByteArrayOutputStream baos) {
-    return new BAOSBytesInput(baos);
-  }
-
-  /**
-   * @return an empty bytes input
-   */
-  public static BytesInput empty() {
-    return EMPTY_BYTES_INPUT;
-  }
-
-  /**
-   * copies the input into a new byte array
-   * @param bytesInput
-   * @return
-   * @throws IOException
-   */
-  public static BytesInput copy(BytesInput bytesInput) throws IOException {
-    return from(bytesInput.toByteArray());
-  }
-
-  /**
-   * writes the bytes into a stream
-   * @param out
-   * @throws IOException
-   */
-  abstract public void writeAllTo(OutputStream out) throws IOException;
-
-  /**
-   *
-   * @return a new byte array materializing the contents of this input
-   * @throws IOException
-   */
-  public byte[] toByteArray() throws IOException {
-    BAOS baos = new BAOS((int)size());
-    this.writeAllTo(baos);
-    if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes");
-    return baos.getBuf();
-  }
-
-  /**
-   *
-   * @return the size in bytes that would be written
-   */
-  abstract public long size();
-
-  private static final class BAOS extends ByteArrayOutputStream {
-    private BAOS(int size) {
-      super(size);
-    }
-
-    public byte[] getBuf() {
-      return this.buf;
-    }
-  }
-
-  private static class StreamBytesInput extends BytesInput {
-    private static final Log LOG = Log.getLog(BytesInput.StreamBytesInput.class);
-    private final InputStream in;
-    private final int byteCount;
-
-    private StreamBytesInput(InputStream in, int byteCount) {
-      super();
-      this.in = in;
-      this.byteCount = byteCount;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      if (DEBUG) LOG.debug("write All "+ byteCount + " bytes");
-      // TODO: more efficient
-      out.write(this.toByteArray());
-    }
-
-    public byte[] toByteArray() throws IOException {
-      if (DEBUG) LOG.debug("read all "+ byteCount + " bytes");
-      byte[] buf = new byte[byteCount];
-      new DataInputStream(in).readFully(buf);
-      return buf;
-    }
-
-    @Override
-    public long size() {
-      return byteCount;
-    }
-
-  }
-
-  private static class SequenceBytesIn extends BytesInput {
-    private static final Log LOG = Log.getLog(BytesInput.SequenceBytesIn.class);
-
-    private final List<BytesInput> inputs;
-    private final long size;
-
-    private SequenceBytesIn(List<BytesInput> inputs) {
-      this.inputs = inputs;
-      long total = 0;
-      for (BytesInput input : inputs) {
-        total += input.size();
-      }
-      this.size = total;
-    }
-
-    @SuppressWarnings("unused")
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      for (BytesInput input : inputs) {
-        if (DEBUG) LOG.debug("write " + input.size() + " bytes to out");
-        if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("{");
-        input.writeAllTo(out);
-        if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("}");
-      }
-    }
-
-    @Override
-    public long size() {
-      return size;
-    }
-
-  }
-
-  private static class IntBytesInput extends BytesInput {
-
-    private final int intValue;
-
-    public IntBytesInput(int intValue) {
-      this.intValue = intValue;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      BytesUtils.writeIntLittleEndian(out, intValue);
-    }
-
-    @Override
-    public long size() {
-      return 4;
-    }
-
-  }
-
-  private static class UnsignedVarIntBytesInput extends BytesInput {
-
-    private final int intValue;
-
-    public UnsignedVarIntBytesInput(int intValue) {
-      this.intValue = intValue;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      BytesUtils.writeUnsignedVarInt(intValue, out);
-    }
-
-    @Override
-    public long size() {
-      int s = 5 - ((Integer.numberOfLeadingZeros(intValue) + 3) / 7);
-      return s == 0 ? 1 : s;
-    }
-  }
-
-  private static class EmptyBytesInput extends BytesInput {
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-    }
-
-    @Override
-    public long size() {
-      return 0;
-    }
-
-  }
-
-  private static class CapacityBAOSBytesInput extends BytesInput {
-
-    private final CapacityByteArrayOutputStream arrayOut;
-
-    private CapacityBAOSBytesInput(CapacityByteArrayOutputStream arrayOut) {
-      this.arrayOut = arrayOut;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      arrayOut.writeTo(out);
-    }
-
-    @Override
-    public long size() {
-      return arrayOut.size();
-    }
-
-  }
-
-  private static class BAOSBytesInput extends BytesInput {
-
-    private final ByteArrayOutputStream arrayOut;
-
-    private BAOSBytesInput(ByteArrayOutputStream arrayOut) {
-      this.arrayOut = arrayOut;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      arrayOut.writeTo(out);
-    }
-
-    @Override
-    public long size() {
-      return arrayOut.size();
-    }
-
-  }
-
-  private static class ByteArrayBytesInput extends BytesInput {
-
-    private final byte[] in;
-    private final int offset;
-    private final int length;
-
-    private ByteArrayBytesInput(byte[] in, int offset, int length) {
-      this.in = in;
-      this.offset = offset;
-      this.length = length;
-    }
-
-    @Override
-    public void writeAllTo(OutputStream out) throws IOException {
-      out.write(in, offset, length);
-    }
-
-    @Override
-    public long size() {
-      return length;
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
deleted file mode 100644
index d307471..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/CapacityByteArrayOutputStream.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import static java.lang.Math.max;
-import static java.lang.Math.pow;
-import static java.lang.String.format;
-import static java.lang.System.arraycopy;
-import static parquet.Preconditions.checkArgument;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import parquet.Log;
-
-/**
- * Similar to a {@link ByteArrayOutputStream}, but uses a different strategy for growing that does not involve copying.
- * Where ByteArrayOutputStream is backed by a single array that "grows" by copying into a new larger array, this output
- * stream grows by allocating a new array (slab) and adding it to a list of previous arrays.
- *
- * Each new slab is allocated to be the same size as all the previous slabs combined, so these allocations become
- * exponentially less frequent, just like ByteArrayOutputStream, with one difference. This output stream accepts a
- * max capacity hint, which is a hint describing the max amount of data that will be written to this stream. As the
- * total size of this stream nears this max, this stream starts to grow linearly instead of exponentially.
- * So new slabs are allocated to be 1/5th of the max capacity hint,
- * instead of equal to the total previous size of all slabs. This is useful because it prevents allocating roughly
- * twice the needed space when a new slab is added just before the stream is done being used.
- *
- * When reusing this stream it will adjust the initial slab size based on the previous data size, aiming for fewer
- * allocations, with the assumption that a similar amount of data will be written to this stream on re-use.
- * See ({@link CapacityByteArrayOutputStream#reset()}).
- *
- * @author Julien Le Dem
- *
- */
-public class CapacityByteArrayOutputStream extends OutputStream {
-  private static final Log LOG = Log.getLog(CapacityByteArrayOutputStream.class);
-  private static final byte[] EMPTY_SLAB = new byte[0];
-
-  private int initialSlabSize;
-  private final int maxCapacityHint;
-  private final List<byte[]> slabs = new ArrayList<byte[]>();
-
-  private byte[] currentSlab;
-  private int currentSlabIndex;
-  private int bytesAllocated = 0;
-  private int bytesUsed = 0;
-
-  /**
-   * Return an initial slab size such that a CapacityByteArrayOutputStream constructed with it
-   * will end up allocating targetNumSlabs in order to reach targetCapacity. This aims to be
-   * a balance between the overhead of creating new slabs and wasting memory by eagerly making
-   * initial slabs too big.
-   *
-   * Note that targetCapacity here need not match maxCapacityHint in the constructor of
-   * CapacityByteArrayOutputStream, though often that would make sense.
-   *
-   * @param minSlabSize no matter what we shouldn't make slabs any smaller than this
-   * @param targetCapacity after we've allocated targetNumSlabs how much capacity should we have?
-   * @param targetNumSlabs how many slabs should it take to reach targetCapacity?
-   */
-  public static int initialSlabSizeHeuristic(int minSlabSize, int targetCapacity, int targetNumSlabs) {
-    // initialSlabSize = (targetCapacity / (2^targetNumSlabs)) means we double targetNumSlabs times
-    // before reaching the targetCapacity
-    // eg for page size of 1MB we start at 1024 bytes.
-    // we also don't want to start too small, so we also apply a minimum.
-    return max(minSlabSize, ((int) (targetCapacity / pow(2, targetNumSlabs))));
-  }
-
-  /**
-   * Construct a CapacityByteArrayOutputStream configured such that its initial slab size is
-   * determined by {@link #initialSlabSizeHeuristic}, with targetCapacity == maxCapacityHint
-   */
-  public static CapacityByteArrayOutputStream withTargetNumSlabs(
-      int minSlabSize, int maxCapacityHint, int targetNumSlabs) {
-
-    return new CapacityByteArrayOutputStream(
-        initialSlabSizeHeuristic(minSlabSize, maxCapacityHint, targetNumSlabs),
-        maxCapacityHint);
-  }
-
-  /**
-   * Defaults maxCapacityHint to 1MB
-   * @param initialSlabSize
-   * @deprecated use {@link CapacityByteArrayOutputStream#CapacityByteArrayOutputStream(int, int)}
-   */
-  @Deprecated
-  public CapacityByteArrayOutputStream(int initialSlabSize) {
-    this(initialSlabSize, 1024 * 1024);
-  }
-
-  /**
-   * @param initialSlabSize the size to make the first slab
-   * @param maxCapacityHint a hint (not guarantee) of the max amount of data written to this stream
-   */
-  public CapacityByteArrayOutputStream(int initialSlabSize, int maxCapacityHint) {
-    checkArgument(initialSlabSize > 0, "initialSlabSize must be > 0");
-    checkArgument(maxCapacityHint > 0, "maxCapacityHint must be > 0");
-    checkArgument(maxCapacityHint >= initialSlabSize, String.format("maxCapacityHint can't be less than initialSlabSize %d %d", initialSlabSize, maxCapacityHint));
-    this.initialSlabSize = initialSlabSize;
-    this.maxCapacityHint = maxCapacityHint;
-    reset();
-  }
-
-  /**
-   * the new slab is guaranteed to be at least minimumSize
-   * @param minimumSize the size of the data we want to copy in the new slab
-   */
-  private void addSlab(int minimumSize) {
-    int nextSlabSize;
-
-    if (bytesUsed == 0) {
-      nextSlabSize = initialSlabSize;
-    } else if (bytesUsed > maxCapacityHint / 5) {
-      // to avoid an overhead of up to twice the needed size, we get linear when approaching target page size
-      nextSlabSize = maxCapacityHint / 5;
-    } else {
-      // double the size every time
-      nextSlabSize = bytesUsed;
-    }
-
-    if (nextSlabSize < minimumSize) {
-      if (Log.DEBUG) LOG.debug(format("slab size %,d too small for value of size %,d. Bumping up slab size", nextSlabSize, minimumSize));
-      nextSlabSize = minimumSize;
-    }
-
-    if (Log.DEBUG) LOG.debug(format("used %d slabs, adding new slab of size %d", slabs.size(), nextSlabSize));
-
-    this.currentSlab = new byte[nextSlabSize];
-    this.slabs.add(currentSlab);
-    this.bytesAllocated += nextSlabSize;
-    this.currentSlabIndex = 0;
-  }
-
-  @Override
-  public void write(int b) {
-    if (currentSlabIndex == currentSlab.length) {
-      addSlab(1);
-    }
-    currentSlab[currentSlabIndex] = (byte) b;
-    currentSlabIndex += 1;
-    bytesUsed += 1;
-  }
-
-  @Override
-  public void write(byte b[], int off, int len) {
-    if ((off < 0) || (off > b.length) || (len < 0) ||
-        ((off + len) - b.length > 0)) {
-      throw new IndexOutOfBoundsException(
-          String.format("Given byte array of size %d, with requested length(%d) and offset(%d)", b.length, len, off));
-    }
-    if (currentSlabIndex + len >= currentSlab.length) {
-      final int length1 = currentSlab.length - currentSlabIndex;
-      arraycopy(b, off, currentSlab, currentSlabIndex, length1);
-      final int length2 = len - length1;
-      addSlab(length2);
-      arraycopy(b, off + length1, currentSlab, currentSlabIndex, length2);
-      currentSlabIndex = length2;
-    } else {
-      arraycopy(b, off, currentSlab, currentSlabIndex, len);
-      currentSlabIndex += len;
-    }
-    bytesUsed += len;
-  }
-
-  /**
-   * Writes the complete contents of this buffer to the specified output stream argument. the output
-   * stream's write method <code>out.write(slab, 0, slab.length)</code>) will be called once per slab.
-   *
-   * @param      out   the output stream to which to write the data.
-   * @exception  IOException  if an I/O error occurs.
-   */
-  public void writeTo(OutputStream out) throws IOException {
-    for (int i = 0; i < slabs.size() - 1; i++) {
-      final byte[] slab = slabs.get(i);
-      out.write(slab);
-    }
-    out.write(currentSlab, 0, currentSlabIndex);
-  }
-
-  /**
-   * @return The total size in bytes of data written to this stream.
-   */
-  public long size() {
-    return bytesUsed;
-  }
-
-  /**
-   *
-   * @return The total size in bytes currently allocated for this stream.
-   */
-  public int getCapacity() {
-    return bytesAllocated;
-  }
-
-  /**
-   * When re-using an instance with reset, it will adjust slab size based on previous data size.
-   * The intent is to reuse the same instance for the same type of data (for example, the same column).
-   * The assumption is that the size in the buffer will be consistent.
-   */
-  public void reset() {
-    // readjust slab size.
-    // 7 = 2^3 - 1 so that doubling the initial size 3 times will get to the same size
-    this.initialSlabSize = max(bytesUsed / 7, initialSlabSize);
-    if (Log.DEBUG) LOG.debug(String.format("initial slab of size %d", initialSlabSize));
-    this.slabs.clear();
-    this.bytesAllocated = 0;
-    this.bytesUsed = 0;
-    this.currentSlab = EMPTY_SLAB;
-    this.currentSlabIndex = 0;
-  }
-
-  /**
-   * @return the index of the last value written to this stream, which
-   * can be passed to {@link #setByte(long, byte)} in order to change it
-   */
-  public long getCurrentIndex() {
-    checkArgument(bytesUsed > 0, "This is an empty stream");
-    return bytesUsed - 1;
-  }
-
-  /**
-   * Replace the byte stored at position index in this stream with value
-   *
-   * @param index which byte to replace
-   * @param value the value to replace it with
-   */
-  public void setByte(long index, byte value) {
-    checkArgument(index < bytesUsed, "Index: " + index + " is >= the current size of: " + bytesUsed);
-
-    long seen = 0;
-    for (int i = 0; i < slabs.size(); i++) {
-      byte[] slab = slabs.get(i);
-      if (index < seen + slab.length) {
-        // ok found index
-        slab[(int)(index-seen)] = value;
-        break;
-      }
-      seen += slab.length;
-    }
-  }
-
-  /**
-   * @param prefix  a prefix to be used for every new line in the string
-   * @return a text representation of the memory usage of this structure
-   */
-  public String memUsageString(String prefix) {
-    return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), getCapacity());
-  }
-
-  /**
-   * @return the total number of allocated slabs
-   */
-  int getSlabCount() {
-    return slabs.size();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java b/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
deleted file mode 100644
index 6a8437b..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/ConcatenatingByteArrayCollector.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.lang.String.format;
-
-public class ConcatenatingByteArrayCollector extends BytesInput {
-  private final List<byte[]> slabs = new ArrayList<byte[]>();
-  private long size = 0;
-
-  public void collect(BytesInput bytesInput) throws IOException {
-    byte[] bytes = bytesInput.toByteArray();
-    slabs.add(bytes);
-    size += bytes.length;
-  }
-
-  public void reset() {
-    size = 0;
-    slabs.clear();
-  }
-
-  @Override
-  public void writeAllTo(OutputStream out) throws IOException {
-    for (byte[] slab : slabs) {
-      out.write(slab);
-    }
-  }
-
-  @Override
-  public long size() {
-    return size;
-  }
-
-  /**
-   * @param prefix  a prefix to be used for every new line in the string
-   * @return a text representation of the memory usage of this structure
-   */
-  public String memUsageString(String prefix) {
-    return format("%s %s %d slabs, %,d bytes", prefix, getClass().getSimpleName(), slabs.size(), size);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
deleted file mode 100644
index 6ea9325..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataInputStream.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Based on DataInputStream but little endian and without the String/char methods
- *
- * @author Julien Le Dem
- *
- */
-public final class LittleEndianDataInputStream extends InputStream {
-
-  private final InputStream in;
-
-  /**
-   * Creates a LittleEndianDataInputStream that uses the specified
-   * underlying InputStream.
-   *
-   * @param  in   the specified input stream
-   */
-  public LittleEndianDataInputStream(InputStream in) {
-    this.in = in;
-  }
-
-  /**
-   * See the general contract of the <code>readFully</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @param      b   the buffer into which the data is read.
-   * @exception  EOFException  if this input stream reaches the end before
-   *             reading all the bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final void readFully(byte b[]) throws IOException {
-    readFully(b, 0, b.length);
-  }
-
-  /**
-   * See the general contract of the <code>readFully</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @param      b     the buffer into which the data is read.
-   * @param      off   the start offset of the data.
-   * @param      len   the number of bytes to read.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading all the bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final void readFully(byte b[], int off, int len) throws IOException {
-    if (len < 0)
-      throw new IndexOutOfBoundsException();
-    int n = 0;
-    while (n < len) {
-      int count = in.read(b, off + n, len - n);
-      if (count < 0)
-        throw new EOFException();
-      n += count;
-    }
-  }
-
-  /**
-   * See the general contract of the <code>skipBytes</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes for this operation are read from the contained
-   * input stream.
-   *
-   * @param      n   the number of bytes to be skipped.
-   * @return     the actual number of bytes skipped.
-   * @exception  IOException  if the contained input stream does not support
-   *             seek, or the stream has been closed and
-   *             the contained input stream does not support
-   *             reading after close, or another I/O error occurs.
-   */
-  public final int skipBytes(int n) throws IOException {
-    int total = 0;
-    int cur = 0;
-
-    while ((total<n) && ((cur = (int) in.skip(n-total)) > 0)) {
-      total += cur;
-    }
-
-    return total;
-  }
-
-  /**
-   * @return
-   * @throws IOException
-   * @see java.io.InputStream#read()
-   */
-  public int read() throws IOException {
-    return in.read();
-  }
-
-  /**
-   * @return
-   * @see java.lang.Object#hashCode()
-   */
-  public int hashCode() {
-    return in.hashCode();
-  }
-
-  /**
-   * @param b
-   * @return
-   * @throws IOException
-   * @see java.io.InputStream#read(byte[])
-   */
-  public int read(byte[] b) throws IOException {
-    return in.read(b);
-  }
-
-  /**
-   * @param obj
-   * @return
-   * @see java.lang.Object#equals(java.lang.Object)
-   */
-  public boolean equals(Object obj) {
-    return in.equals(obj);
-  }
-
-  /**
-   * @param b
-   * @param off
-   * @param len
-   * @return
-   * @throws IOException
-   * @see java.io.InputStream#read(byte[], int, int)
-   */
-  public int read(byte[] b, int off, int len) throws IOException {
-    return in.read(b, off, len);
-  }
-
-  /**
-   * @param n
-   * @return
-   * @throws IOException
-   * @see java.io.InputStream#skip(long)
-   */
-  public long skip(long n) throws IOException {
-    return in.skip(n);
-  }
-
-  /**
-   * @return
-   * @throws IOException
-   * @see java.io.InputStream#available()
-   */
-  public int available() throws IOException {
-    return in.available();
-  }
-
-  /**
-   * @throws IOException
-   * @see java.io.InputStream#close()
-   */
-  public void close() throws IOException {
-    in.close();
-  }
-
-  /**
-   * @param readlimit
-   * @see java.io.InputStream#mark(int)
-   */
-  public void mark(int readlimit) {
-    in.mark(readlimit);
-  }
-
-  /**
-   * @throws IOException
-   * @see java.io.InputStream#reset()
-   */
-  public void reset() throws IOException {
-    in.reset();
-  }
-
-  /**
-   * @return
-   * @see java.io.InputStream#markSupported()
-   */
-  public boolean markSupported() {
-    return in.markSupported();
-  }
-
-  /**
-   * See the general contract of the <code>readBoolean</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the <code>boolean</code> value read.
-   * @exception  EOFException  if this input stream has reached the end.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final boolean readBoolean() throws IOException {
-    int ch = in.read();
-    if (ch < 0)
-      throw new EOFException();
-    return (ch != 0);
-  }
-
-  /**
-   * See the general contract of the <code>readByte</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next byte of this input stream as a signed 8-bit
-   *             <code>byte</code>.
-   * @exception  EOFException  if this input stream has reached the end.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final byte readByte() throws IOException {
-    int ch = in.read();
-    if (ch < 0)
-      throw new EOFException();
-    return (byte)(ch);
-  }
-
-  /**
-   * See the general contract of the <code>readUnsignedByte</code>
-   * method of <code>DataInput</code>.
-   * <p>
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next byte of this input stream, interpreted as an
-   *             unsigned 8-bit number.
-   * @exception  EOFException  if this input stream has reached the end.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see         java.io.FilterInputStream#in
-   */
-  public final int readUnsignedByte() throws IOException {
-    int ch = in.read();
-    if (ch < 0)
-      throw new EOFException();
-    return ch;
-  }
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next two bytes of this input stream, interpreted as a
-   *             signed 16-bit number.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading two bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final short readShort() throws IOException {
-    int ch2 = in.read();
-    int ch1 = in.read();
-    if ((ch1 | ch2) < 0)
-      throw new EOFException();
-    return (short)((ch1 << 8) + (ch2 << 0));
-  }
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next two bytes of this input stream, interpreted as an
-   *             unsigned 16-bit integer.
-   * @exception  EOFException  if this input stream reaches the end before
-   *             reading two bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final int readUnsignedShort() throws IOException {
-    int ch2 = in.read();
-    int ch1 = in.read();
-    if ((ch1 | ch2) < 0)
-      throw new EOFException();
-    return (ch1 << 8) + (ch2 << 0);
-  }
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next four bytes of this input stream, interpreted as an
-   *             <code>int</code>.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading four bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final int readInt() throws IOException {
-    // TODO: has this been benchmarked against two alternate implementations?
-    // 1) Integer.reverseBytes(in.readInt())
-    // 2) keep a member byte[4], wrapped by an IntBuffer with appropriate endianness set,
-    //    and call IntBuffer.get()
-    // Both seem like they might be faster.
-    int ch4 = in.read();
-    int ch3 = in.read();
-    int ch2 = in.read();
-    int ch1 = in.read();
-    if ((ch1 | ch2 | ch3 | ch4) < 0)
-      throw new EOFException();
-    return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0));
-  }
-
-  private byte readBuffer[] = new byte[8];
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next eight bytes of this input stream, interpreted as a
-   *             <code>long</code>.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading eight bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.io.FilterInputStream#in
-   */
-  public final long readLong() throws IOException {
-    // TODO: see perf question above in readInt
-    readFully(readBuffer, 0, 8);
-    return (((long)readBuffer[7] << 56) +
-        ((long)(readBuffer[6] & 255) << 48) +
-        ((long)(readBuffer[5] & 255) << 40) +
-        ((long)(readBuffer[4] & 255) << 32) +
-        ((long)(readBuffer[3] & 255) << 24) +
-        ((readBuffer[2] & 255) << 16) +
-        ((readBuffer[1] & 255) <<  8) +
-        ((readBuffer[0] & 255) <<  0));
-  }
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next four bytes of this input stream, interpreted as a
-   *             <code>float</code>.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading four bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.lang.Float#intBitsToFloat(int)
-   */
-  public final float readFloat() throws IOException {
-    return Float.intBitsToFloat(readInt());
-  }
-
-  /**
-   * Bytes
-   * for this operation are read from the contained
-   * input stream.
-   *
-   * @return     the next eight bytes of this input stream, interpreted as a
-   *             <code>double</code>.
-   * @exception  EOFException  if this input stream reaches the end before
-   *               reading eight bytes.
-   * @exception  IOException   the stream has been closed and the contained
-   *             input stream does not support reading after close, or
-   *             another I/O error occurs.
-   * @see        java.lang.Double#longBitsToDouble(long)
-   */
-  public final double readDouble() throws IOException {
-    return Double.longBitsToDouble(readLong());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java b/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
deleted file mode 100644
index 2be7299..0000000
--- a/parquet-encoding/src/main/java/parquet/bytes/LittleEndianDataOutputStream.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Based on DataOutputStream but in little endian and without the String/char methods
- *
- * @author Julien Le Dem
- *
- */
-public class LittleEndianDataOutputStream extends OutputStream {
-
-  private final OutputStream out;
-
-  /**
-   * Creates a new data output stream to write data to the specified
-   * underlying output stream. The counter <code>written</code> is
-   * set to zero.
-   *
-   * @param   out   the underlying output stream, to be saved for later
-   *                use.
-   * @see     java.io.FilterOutputStream#out
-   */
-  public LittleEndianDataOutputStream(OutputStream out) {
-    this.out = out;
-  }
-
-  /**
-   * Writes the specified byte (the low eight bits of the argument
-   * <code>b</code>) to the underlying output stream. If no exception
-   * is thrown, the counter <code>written</code> is incremented by
-   * <code>1</code>.
-   * <p>
-   * Implements the <code>write</code> method of <code>OutputStream</code>.
-   *
-   * @param      b   the <code>byte</code> to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public void write(int b) throws IOException {
-    out.write(b);
-  }
-
-  /**
-   * Writes <code>len</code> bytes from the specified byte array
-   * starting at offset <code>off</code> to the underlying output stream.
-   * If no exception is thrown, the counter <code>written</code> is
-   * incremented by <code>len</code>.
-   *
-   * @param      b     the data.
-   * @param      off   the start offset in the data.
-   * @param      len   the number of bytes to write.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public void write(byte b[], int off, int len) throws IOException {
-    out.write(b, off, len);
-  }
-
-  /**
-   * Flushes this data output stream. This forces any buffered output
-   * bytes to be written out to the stream.
-   * <p>
-   * The <code>flush</code> method of <code>DataOutputStream</code>
-   * calls the <code>flush</code> method of its underlying output stream.
-   *
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   * @see        java.io.OutputStream#flush()
-   */
-  public void flush() throws IOException {
-    out.flush();
-  }
-
-  /**
-   * Writes a <code>boolean</code> to the underlying output stream as
-   * a 1-byte value. The value <code>true</code> is written out as the
-   * value <code>(byte)1</code>; the value <code>false</code> is
-   * written out as the value <code>(byte)0</code>. If no exception is
-   * thrown, the counter <code>written</code> is incremented by
-   * <code>1</code>.
-   *
-   * @param      v   a <code>boolean</code> value to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public final void writeBoolean(boolean v) throws IOException {
-    out.write(v ? 1 : 0);
-  }
-
-  /**
-   * Writes out a <code>byte</code> to the underlying output stream as
-   * a 1-byte value. If no exception is thrown, the counter
-   * <code>written</code> is incremented by <code>1</code>.
-   *
-   * @param      v   a <code>byte</code> value to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public final void writeByte(int v) throws IOException {
-    out.write(v);
-  }
-
-  /**
-   * Writes a <code>short</code> to the underlying output stream as two
-   * bytes, low byte first. If no exception is thrown, the counter
-   * <code>written</code> is incremented by <code>2</code>.
-   *
-   * @param      v   a <code>short</code> to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public final void writeShort(int v) throws IOException {
-    out.write((v >>> 0) & 0xFF);
-    out.write((v >>> 8) & 0xFF);
-  }
-
-  /**
-   * Writes an <code>int</code> to the underlying output stream as four
-   * bytes, low byte first. If no exception is thrown, the counter
-   * <code>written</code> is incremented by <code>4</code>.
-   *
-   * @param      v   an <code>int</code> to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public final void writeInt(int v) throws IOException {
-    // TODO: see note in LittleEndianDataInputStream: maybe faster
-    // to use Integer.reverseBytes() and then writeInt, or a ByteBuffer
-    // approach
-    out.write((v >>>  0) & 0xFF);
-    out.write((v >>>  8) & 0xFF);
-    out.write((v >>> 16) & 0xFF);
-    out.write((v >>> 24) & 0xFF);
-  }
-
-  private byte writeBuffer[] = new byte[8];
-
-  /**
-   * Writes a <code>long</code> to the underlying output stream as eight
-   * bytes, low byte first. In no exception is thrown, the counter
-   * <code>written</code> is incremented by <code>8</code>.
-   *
-   * @param      v   a <code>long</code> to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   */
-  public final void writeLong(long v) throws IOException {
-    writeBuffer[7] = (byte)(v >>> 56);
-    writeBuffer[6] = (byte)(v >>> 48);
-    writeBuffer[5] = (byte)(v >>> 40);
-    writeBuffer[4] = (byte)(v >>> 32);
-    writeBuffer[3] = (byte)(v >>> 24);
-    writeBuffer[2] = (byte)(v >>> 16);
-    writeBuffer[1] = (byte)(v >>>  8);
-    writeBuffer[0] = (byte)(v >>>  0);
-    out.write(writeBuffer, 0, 8);
-  }
-
-  /**
-   * Converts the float argument to an <code>int</code> using the
-   * <code>floatToIntBits</code> method in class <code>Float</code>,
-   * and then writes that <code>int</code> value to the underlying
-   * output stream as a 4-byte quantity, low byte first. If no
-   * exception is thrown, the counter <code>written</code> is
-   * incremented by <code>4</code>.
-   *
-   * @param      v   a <code>float</code> value to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   * @see        java.lang.Float#floatToIntBits(float)
-   */
-  public final void writeFloat(float v) throws IOException {
-    writeInt(Float.floatToIntBits(v));
-  }
-
-  /**
-   * Converts the double argument to a <code>long</code> using the
-   * <code>doubleToLongBits</code> method in class <code>Double</code>,
-   * and then writes that <code>long</code> value to the underlying
-   * output stream as an 8-byte quantity, low byte first. If no
-   * exception is thrown, the counter <code>written</code> is
-   * incremented by <code>8</code>.
-   *
-   * @param      v   a <code>double</code> value to be written.
-   * @exception  IOException  if an I/O error occurs.
-   * @see        java.io.FilterOutputStream#out
-   * @see        java.lang.Double#doubleToLongBits(double)
-   */
-  public final void writeDouble(double v) throws IOException {
-    writeLong(Double.doubleToLongBits(v));
-  }
-
-}


Mime
View raw message