parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [17/51] [partial] parquet-mr git commit: PARQUET-23: Rename to org.apache.parquet.
Date Mon, 27 Apr 2015 23:12:14 GMT
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BitPacking.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BitPacking.java
deleted file mode 100644
index 54518b2..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BitPacking.java
+++ /dev/null
@@ -1,725 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import parquet.bytes.BytesUtils;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-// TODO: rework the whole thing. It does not need to use streams at all
-/**
- * provides the correct implementation of a bitpacking based on the width in bits
- *
- * @author Julien Le Dem
- *
- */
-public class BitPacking {
-
-  /**
-   * to writes ints to a stream packed to only the needed bits.
-   * there is no guarantee of corecteness if ints larger than the max size are written
-   *
-   * @author Julien Le Dem
-   *
-   */
-  abstract public static class BitPackingWriter {
-    /**
-     * will write the bits to the underlying stream aligned on the buffer size
-     * @param val the value to encode
-     * @throws IOException
-     */
-    abstract public void write(int val) throws IOException;
-
-    /**
-     * will flush the buffer to the underlying stream (and pad with 0s)
-     * @throws IOException
-     */
-    abstract public void finish() throws IOException;
-  }
-
-  /**
-   * to read back what has been written with the corresponding  writer
-   *
-   * @author Julien Le Dem
-   *
-   */
-  abstract public static class BitPackingReader {
-
-    /**
-     *
-     * @return and int decoded from the underlying stream
-     * @throws IOException
-     */
-    abstract public int read() throws IOException;
-  }
-
-  private BitPacking() {
-  }
-
-  /**
-   * @param bitLength the width in bits of the integers to write
-   * @param out the stream to write the bytes to
-   * @return the correct implementation for the width
-   */
-  public static BitPackingWriter getBitPackingWriter(int bitLength, OutputStream out) {
-    switch (bitLength) {
-    case 0:
-      return new ZeroBitPackingWriter();
-    case 1:
-      return new OneBitPackingWriter(out);
-    case 2:
-      return new TwoBitPackingWriter(out);
-    case 3:
-      return new ThreeBitPackingWriter(out);
-    case 4:
-      return new FourBitPackingWriter(out);
-    case 5:
-      return new FiveBitPackingWriter(out);
-    case 6:
-      return new SixBitPackingWriter(out);
-    case 7:
-      return new SevenBitPackingWriter(out);
-    case 8:
-      return new EightBitPackingWriter(out);
-    default:
-      throw new UnsupportedOperationException("only support up to 8 for now");
-    }
-  }
-
-  /**
-   *
-   * @param bitLength the width in bits of the integers to read
-   * @param inthe stream to read the bytes from
-   * @return the correct implementation for the width
-   */
-  public static BitPackingReader createBitPackingReader(int bitLength, InputStream in, long valueCount) {
-    switch (bitLength) {
-    case 0:
-      return new ZeroBitPackingReader();
-    case 1:
-      return new OneBitPackingReader(in);
-    case 2:
-      return new TwoBitPackingReader(in);
-    case 3:
-      return new ThreeBitPackingReader(in, valueCount);
-    case 4:
-      return new FourBitPackingReader(in);
-    case 5:
-      return new FiveBitPackingReader(in, valueCount);
-    case 6:
-      return new SixBitPackingReader(in, valueCount);
-    case 7:
-      return new SevenBitPackingReader(in, valueCount);
-    case 8:
-      return new EightBitPackingReader(in);
-    default:
-      throw new UnsupportedOperationException("only support up to 8 for now");
-    }
-  }
-}
-
-abstract class BaseBitPackingWriter extends BitPackingWriter {
-
-  void finish(int numberOfBits, int buffer, OutputStream out) throws IOException {
-    int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
-    buffer = buffer << padding;
-    int numberOfBytes = (numberOfBits + padding) / 8;
-    for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
-      out.write((buffer >>> i) & 0xFF);
-    }
-  }
-
-  void finish(int numberOfBits, long buffer, OutputStream out) throws IOException {
-    int padding = numberOfBits % 8 == 0 ? 0 : 8 - (numberOfBits % 8);
-    buffer = buffer << padding;
-    int numberOfBytes = (numberOfBits + padding) / 8;
-    for (int i = (numberOfBytes - 1) * 8; i >= 0 ; i -= 8) {
-      out.write((int)(buffer >>> i) & 0xFF);
-    }
-  }
-}
-abstract class BaseBitPackingReader extends BitPackingReader {
-
-  int alignToBytes(int bitsCount) {
-    return BytesUtils.paddedByteCountFromBits(bitsCount);
-  }
-
-}
-
-class ZeroBitPackingWriter extends BitPackingWriter {
-
-  @Override
-  public void write(int val) throws IOException {
-  }
-
-  @Override
-  public void finish() {
-  }
-
-}
-class ZeroBitPackingReader extends BitPackingReader {
-
-  @Override
-  public int read() throws IOException {
-    return 0;
-  }
-
-}
-
-class OneBitPackingWriter extends BitPackingWriter {
-
-  private OutputStream out;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public OneBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 1;
-    buffer |= val;
-    ++ count;
-    if (count == 8) {
-      out.write(buffer);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    while (count != 0) {
-      write(0);
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class OneBitPackingReader extends BitPackingReader {
-
-  private final InputStream in;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public OneBitPackingReader(InputStream in) {
-    this.in = in;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      buffer = in.read();
-      count = 8;
-    }
-    int result = (buffer >> (count - 1)) & 1;
-    -- count;
-    return result;
-  }
-
-}
-
-class TwoBitPackingWriter extends BitPackingWriter {
-
-  private OutputStream out;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public TwoBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 2;
-    buffer |= val;
-    ++ count;
-    if (count == 4) {
-      out.write(buffer);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    while (count != 0) {
-      write(0);
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class TwoBitPackingReader extends BitPackingReader {
-
-  private final InputStream in;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public TwoBitPackingReader(InputStream in) {
-    this.in = in;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      buffer = in.read();
-      count = 4;
-    }
-    int result = (buffer >> ((count - 1) * 2)) & 3;
-    -- count;
-    return result;
-  }
-
-}
-
-class ThreeBitPackingWriter extends BaseBitPackingWriter {
-
-  private OutputStream out;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public ThreeBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 3;
-    buffer |= val;
-    ++ count;
-    if (count == 8) {
-      out.write((buffer >>> 16) & 0xFF);
-      out.write((buffer >>>  8) & 0xFF);
-      out.write((buffer >>>  0) & 0xFF);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    if (count != 0) {
-      int numberOfBits = count * 3;
-      finish(numberOfBits, buffer, out);
-      buffer = 0;
-      count = 0;
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class ThreeBitPackingReader extends BaseBitPackingReader {
-
-  private final InputStream in;
-  private final long valueCount;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  private long totalRead = 0;
-
-  public ThreeBitPackingReader(InputStream in, long valueCount) {
-    this.in = in;
-    this.valueCount = valueCount;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      if (valueCount - totalRead < 8) {
-        buffer = 0;
-        int bitsToRead = 3 * (int)(valueCount - totalRead);
-        int bytesToRead = alignToBytes(bitsToRead);
-        for (int i = 3 - 1 ; i >= 3 - bytesToRead ; i--) {
-          buffer |= in.read() << (i * 8);
-        }
-        count = 8;
-        totalRead = valueCount;
-      } else {
-        buffer = (in.read() << 16) + (in.read() << 8) + in.read();
-        count = 8;
-        totalRead += 8;
-      }
-    }
-    int result = (buffer >> ((count - 1) * 3)) & 7;
-    -- count;
-    return result;
-  }
-
-}
-
-class FourBitPackingWriter extends BitPackingWriter {
-
-  private OutputStream out;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public FourBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 4;
-    buffer |= val;
-    ++ count;
-    if (count == 2) {
-      out.write(buffer);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    while (count != 0) {
-      // downside: this aligns on whatever the buffer size is.
-      write(0);
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class FourBitPackingReader extends BitPackingReader {
-
-  private final InputStream in;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public FourBitPackingReader(InputStream in) {
-    this.in = in;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      buffer = in.read();
-      count = 2;
-    }
-    int result = (buffer >> ((count - 1) * 4)) & 15;
-    -- count;
-    return result;
-  }
-
-}
-
-class FiveBitPackingWriter extends BaseBitPackingWriter {
-
-  private OutputStream out;
-
-  private long buffer = 0;
-  private int count = 0;
-
-  public FiveBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 5;
-    buffer |= val;
-    ++ count;
-    if (count == 8) {
-      out.write((int)(buffer >>> 32) & 0xFF);
-      out.write((int)(buffer >>> 24) & 0xFF);
-      out.write((int)(buffer >>> 16) & 0xFF);
-      out.write((int)(buffer >>>  8) & 0xFF);
-      out.write((int)(buffer >>>  0) & 0xFF);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    if (count != 0) {
-      int numberOfBits = count * 5;
-      finish(numberOfBits, buffer, out);
-      buffer = 0;
-      count = 0;
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class FiveBitPackingReader extends BaseBitPackingReader {
-
-  private final InputStream in;
-  private final long valueCount;
-
-  private long buffer = 0;
-  private int count = 0;
-  private long totalRead = 0;
-
-
-  public FiveBitPackingReader(InputStream in, long valueCount) {
-    this.in = in;
-    this.valueCount = valueCount;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      if (valueCount - totalRead < 8) {
-        buffer = 0;
-        int bitsToRead = 5 * (int)(valueCount - totalRead);
-        int bytesToRead = alignToBytes(bitsToRead);
-        for (int i = 5 - 1; i >= 5 - bytesToRead ; i--) {
-          buffer |= (((long)in.read()) & 255) << (i * 8);
-        }
-        count = 8;
-        totalRead = valueCount;
-      } else {
-        buffer =
-            ((((long)in.read()) & 255) << 32)
-            + ((((long)in.read()) & 255) << 24)
-            + (in.read() << 16)
-            + (in.read() << 8)
-            + in.read();
-        count = 8;
-        totalRead += 8;
-      }
-    }
-    int result = (((int)(buffer >> ((count - 1) * 5))) & 31);
-    -- count;
-    return result;
-  }
-
-}
-
-class SixBitPackingWriter extends BaseBitPackingWriter {
-
-  private OutputStream out;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  public SixBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 6;
-    buffer |= val;
-    ++ count;
-    if (count == 4) {
-      out.write((buffer >>> 16) & 0xFF);
-      out.write((buffer >>>  8) & 0xFF);
-      out.write((buffer >>>  0) & 0xFF);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    if (count != 0) {
-      int numberOfBits = count * 6;
-      finish(numberOfBits, buffer, out);
-      buffer = 0;
-      count = 0;
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class SixBitPackingReader extends BaseBitPackingReader {
-
-  private final InputStream in;
-  private final long valueCount;
-
-  private int buffer = 0;
-  private int count = 0;
-
-  private long totalRead = 0;
-
-
-  public SixBitPackingReader(InputStream in, long valueCount) {
-    this.in = in;
-    this.valueCount = valueCount;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      if (valueCount - totalRead < 4) {
-        buffer = 0;
-        int bitsToRead = 6 * (int)(valueCount - totalRead);
-        int bytesToRead = alignToBytes(bitsToRead);
-        for (int i = 3 - 1; i >= 3 - bytesToRead ; i--) {
-          buffer |= in.read() << (i * 8);
-        }
-        count = 4;
-        totalRead = valueCount;
-      } else {
-        buffer = (in.read() << 16) + (in.read() << 8) + in.read();
-        count = 4;
-        totalRead += 4;
-      }
-    }
-    int result = (buffer >> ((count - 1) * 6)) & 63;
-    -- count;
-    return result;
-  }
-
-}
-
-class SevenBitPackingWriter extends BaseBitPackingWriter {
-
-  private OutputStream out;
-
-  private long buffer = 0;
-  private int count = 0;
-
-  public SevenBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    buffer = buffer << 7;
-    buffer |= val;
-    ++ count;
-    if (count == 8) {
-      out.write((int)(buffer >>> 48) & 0xFF);
-      out.write((int)(buffer >>> 40) & 0xFF);
-      out.write((int)(buffer >>> 32) & 0xFF);
-      out.write((int)(buffer >>> 24) & 0xFF);
-      out.write((int)(buffer >>> 16) & 0xFF);
-      out.write((int)(buffer >>>  8) & 0xFF);
-      out.write((int)(buffer >>>  0) & 0xFF);
-      buffer = 0;
-      count = 0;
-    }
-  }
-
-  @Override
-  public void finish() throws IOException {
-    if (count != 0) {
-      int numberOfBits = count * 7;
-      finish(numberOfBits, buffer, out);
-      buffer = 0;
-      count = 0;
-    }
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class SevenBitPackingReader extends BaseBitPackingReader {
-
-  private final InputStream in;
-  private final long valueCount;
-
-  private long buffer = 0;
-  private int count = 0;
-  private long totalRead = 0;
-
-
-  public SevenBitPackingReader(InputStream in, long valueCount) {
-    this.in = in;
-    this.valueCount = valueCount;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (count == 0) {
-      if (valueCount - totalRead  < 8) {
-        buffer = 0;
-        int bitsToRead = 7 * (int)(valueCount - totalRead);
-        int bytesToRead = alignToBytes(bitsToRead);
-        for (int i = 7 - 1; i >= 7 - bytesToRead ; i--) {
-          buffer |= (((long)in.read()) & 255) << (i * 8);
-        }
-        count = 8;
-        totalRead = valueCount;
-      } else {
-        buffer =
-            ((((long)in.read()) & 255) << 48)
-            + ((((long)in.read()) & 255) << 40)
-            + ((((long)in.read()) & 255) << 32)
-            + ((((long)in.read()) & 255) << 24)
-            + (in.read() << 16)
-            + (in.read() << 8)
-            + in.read();
-        count = 8;
-        totalRead += 8;
-      }
-    }
-    int result = (((int)(buffer >> ((count - 1) * 7))) & 127);
-    -- count;
-    return result;
-  }
-
-}
-
-class EightBitPackingWriter extends BitPackingWriter {
-
-  private OutputStream out;
-
-  public EightBitPackingWriter(OutputStream out) {
-    this.out = out;
-  }
-
-  @Override
-  public void write(int val) throws IOException {
-    out.write(val);
-  }
-
-  @Override
-  public void finish() throws IOException {
-    // check this does not impede perf
-    out = null;
-  }
-
-}
-class EightBitPackingReader extends BitPackingReader {
-
-  private final InputStream in;
-
-  public EightBitPackingReader(InputStream in) {
-    this.in = in;
-  }
-
-  @Override
-  public int read() throws IOException {
-    return in.read();
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
deleted file mode 100644
index 3aaab0d..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import static parquet.Log.DEBUG;
-import static parquet.bytes.BytesInput.concat;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import parquet.Log;
-import parquet.bytes.BytesInput;
-import parquet.bytes.BytesUtils;
-
-/**
- * Uses the generated Byte based bit packing to write ints into a BytesInput
- *
- * @author Julien Le Dem
- *
- */
-public class ByteBasedBitPackingEncoder {
-  private static final Log LOG = Log.getLog(ByteBasedBitPackingEncoder.class);
-
-  private static final int VALUES_WRITTEN_AT_A_TIME = 8;
-
-  private final int bitWidth;
-  private final BytePacker packer;
-  private final int[] input = new int[VALUES_WRITTEN_AT_A_TIME];
-  private final int slabSize;
-  private int inputSize;
-  private byte[] packed;
-  private int packedPosition;
-  private final List<BytesInput> slabs = new ArrayList<BytesInput>();
-  private int totalValues;
-
-  /**
-   * @param bitWidth the number of bits used to encode an int
-   */
-  public ByteBasedBitPackingEncoder(int bitWidth, Packer packer) {
-    this.bitWidth = bitWidth;
-    this.inputSize = 0;
-    // must be a multiple of bitWidth
-    this.slabSize = bitWidth * 64 * 1024;
-    initPackedSlab();
-    this.packer = packer.newBytePacker(bitWidth);
-  }
-
-  /**
-   * writes an int using the requested number of bits.
-   * accepts only value < 2^bitWidth
-   * @param value the value to write
-   * @throws IOException
-   */
-  public void writeInt(int value) throws IOException {
-    input[inputSize] = value;
-    ++ inputSize;
-    if (inputSize == VALUES_WRITTEN_AT_A_TIME) {
-      pack();
-      if (packedPosition == slabSize) {
-        slabs.add(BytesInput.from(packed));
-        initPackedSlab();
-      }
-    }
-  }
-
-  private void pack() {
-    packer.pack8Values(input, 0, packed, packedPosition);
-    packedPosition += bitWidth;
-    totalValues += inputSize;
-    inputSize = 0;
-  }
-
-  private void initPackedSlab() {
-    packed = new byte[slabSize];
-    packedPosition = 0;
-  }
-
-  /**
-   * @return the bytes representing the packed values
-   * @throws IOException
-   */
-  public BytesInput toBytes() throws IOException {
-    int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth);
-
-    if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes");
-    if (inputSize > 0) {
-      for (int i = inputSize; i < input.length; i++) {
-        input[i] = 0;
-      }
-      pack();
-    }
-    return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength));
-  }
-
-  /**
-   * @return size of the data as it would be written
-   */
-  public long getBufferSize() {
-    return BytesUtils.paddedByteCountFromBits(totalValues * bitWidth);
-  }
-
-  /**
-   * @return total memory allocated
-   */
-  public long getAllocatedSize() {
-    return (slabs.size() * slabSize) + packed.length + input.length * 4;
-  }
-
-  public String memUsageString(String prefix) {
-    return String.format("%s ByteBitPacking %d slabs, %d bytes", prefix, slabs.size(), getAllocatedSize());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePacker.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePacker.java
deleted file mode 100644
index 0033683..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePacker.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-/**
- * Packs and unpacks into bytes
- *
- * packing unpacking treats:
- *  - n values at a time (with n % 8 == 0)
- *  - bitWidth * (n/8) bytes at a time.
- *
- * @author Julien Le Dem
- *
- */
-public abstract class BytePacker {
-
-  private final int bitWidth;
-
-  BytePacker(int bitWidth) {
-    this.bitWidth = bitWidth;
-  }
-
-  /**
-   * @return the width in bits used for encoding, also how many bytes are packed/unpacked at a time by pack8Values/unpack8Values
-   */
-  public final int getBitWidth() {
-    return bitWidth;
-  }
-
-  /**
-   * pack 8 values from input at inPos into bitWidth bytes in output at outPos.
-   * nextPosition: inPos += 8; outPos += getBitWidth()
-   * @param input the input values
-   * @param inPos where to read from in input
-   * @param output the output bytes
-   * @param outPos where to write to in output
-   */
-  public abstract void pack8Values(final int[] input, final int inPos, final byte[] output, final int outPos);
-
-  /**
-   * pack 32 values from input at inPos into bitWidth * 4 bytes in output at outPos.
-   * nextPosition: inPos += 32; outPos += getBitWidth() * 4
-   * @param input the input values
-   * @param inPos where to read from in input
-   * @param output the output bytes
-   * @param outPos where to write to in output
-   */
-  public abstract void pack32Values(int[] input, int inPos, byte[] output, int outPos);
-
-  /**
-   * unpack bitWidth bytes from input at inPos into 8 values in output at outPos.
-   * nextPosition: inPos += getBitWidth(); outPos += 8
-   * @param input the input bytes
-   * @param inPos where to read from in input
-   * @param output the output values
-   * @param outPos where to write to in output
-   */
-  public abstract void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos);
-
-  /**
-   * unpack bitWidth * 4 bytes from input at inPos into 32 values in output at outPos.
-   * nextPosition: inPos += getBitWidth() * 4; outPos += 32
-   * @param input the input bytes
-   * @param inPos where to read from in input
-   * @param output the output values
-   * @param outPos where to write to in output
-   */
-  public abstract void unpack32Values(byte[] input, int inPos, int[] output, int outPos);
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePackerFactory.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePackerFactory.java
deleted file mode 100644
index 2ad4c12..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/BytePackerFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-public interface BytePackerFactory {
-
-  BytePacker newBytePacker(int width);
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPacker.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPacker.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPacker.java
deleted file mode 100644
index 2dbc20d..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPacker.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-/**
- * Packs and unpacks into ints
- *
- * packing unpacking treats:
- *  - 32 values at a time
- *  - bitWidth ints at a time.
- *
- * @author Julien Le Dem
- *
- */
-public abstract class IntPacker {
-
-  private final int bitWidth;
-
-  IntPacker(int bitWidth) {
-    this.bitWidth = bitWidth;
-  }
-
-  /**
-   * @return the width in bits used for encoding, also how many ints are packed/unpacked at a time
-   */
-  public final int getBitWidth() {
-    return bitWidth;
-  }
-
-  /**
-   * pack 32 values from input at inPos into bitWidth ints in output at outPos.
-   * nextPosition: inPos += 32; outPos += getBitWidth()
-   * @param input the input values
-   * @param inPos where to read from in input
-   * @param output the output ints
-   * @param outPos where to write to in output
-   */
-  public abstract void pack32Values(int[] input, int inPos, int[] output, int outPos);
-
-  /**
-   * unpack bitWidth ints from input at inPos into 32 values in output at outPos.
-   * nextPosition: inPos += getBitWidth(); outPos += 32
-   * @param input the input int
-   * @param inPos where to read from in input
-   * @param output the output values
-   * @param outPos where to write to in output
-   */
-  public abstract void unpack32Values(int[] input, int inPos, int[] output, int outPos);
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPackerFactory.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPackerFactory.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPackerFactory.java
deleted file mode 100644
index 39e756e..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/IntPackerFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-public interface IntPackerFactory {
-
-  IntPacker newIntPacker(int width);
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/main/java/parquet/column/values/bitpacking/Packer.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/Packer.java b/parquet-encoding/src/main/java/parquet/column/values/bitpacking/Packer.java
deleted file mode 100644
index 0566d66..0000000
--- a/parquet-encoding/src/main/java/parquet/column/values/bitpacking/Packer.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-/**
- * Factory for packing implementations
- *
- * @author Julien Le Dem
- *
- */
-public enum Packer {
-
-  /**
-   * packers who fill the Least Significant Bit First
-   * int and byte packer have the same result on Big Endian architectures
-   */
-  BIG_ENDIAN {
-    @Override
-    public IntPacker newIntPacker(int width) {
-      return beIntPackerFactory.newIntPacker(width);
-    }
-    @Override
-    public BytePacker newBytePacker(int width) {
-      return beBytePackerFactory.newBytePacker(width);
-    }
-  },
-
-  /**
-   * packers who fill the Most Significant Bit first
-   * int and byte packer have the same result on Little Endian architectures
-   */
-  LITTLE_ENDIAN {
-    @Override
-    public IntPacker newIntPacker(int width) {
-      return leIntPackerFactory.newIntPacker(width);
-    }
-    @Override
-    public BytePacker newBytePacker(int width) {
-      return leBytePackerFactory.newBytePacker(width);
-    }
-  };
-
-  private static IntPackerFactory getIntPackerFactory(String name) {
-    return (IntPackerFactory)getStaticField("parquet.column.values.bitpacking." + name, "factory");
-  }
-
-  private static BytePackerFactory getBytePackerFactory(String name) {
-    return (BytePackerFactory)getStaticField("parquet.column.values.bitpacking." + name, "factory");
-  }
-
-  private static Object getStaticField(String className, String fieldName) {
-    try {
-      return Class.forName(className).getField(fieldName).get(null);
-    } catch (IllegalArgumentException e) {
-      throw new RuntimeException(e);
-    } catch (IllegalAccessException e) {
-      throw new RuntimeException(e);
-    } catch (NoSuchFieldException e) {
-      throw new RuntimeException(e);
-    } catch (SecurityException e) {
-      throw new RuntimeException(e);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  static BytePackerFactory beBytePackerFactory = getBytePackerFactory("ByteBitPackingBE");
-  static IntPackerFactory beIntPackerFactory = getIntPackerFactory("LemireBitPackingBE");
-  static BytePackerFactory leBytePackerFactory = getBytePackerFactory("ByteBitPackingLE");
-  static IntPackerFactory leIntPackerFactory = getIntPackerFactory("LemireBitPackingLE");
-
-  /**
-   * @param width the width in bits of the packed values
-   * @return an int based packer
-   */
-  public abstract IntPacker newIntPacker(int width);
-
-  /**
-   * @param width the width in bits of the packed values
-   * @return a byte based packer
-   */
-  public abstract BytePacker newBytePacker(int width);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/org/apache/parquet/bytes/TestCapacityByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/org/apache/parquet/bytes/TestCapacityByteArrayOutputStream.java b/parquet-encoding/src/test/java/org/apache/parquet/bytes/TestCapacityByteArrayOutputStream.java
new file mode 100644
index 0000000..b80fe40
--- /dev/null
+++ b/parquet-encoding/src/test/java/org/apache/parquet/bytes/TestCapacityByteArrayOutputStream.java
@@ -0,0 +1,242 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.bytes;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+public class TestCapacityByteArrayOutputStream {
+
+  @Test
+  public void testWrite() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    final int expectedSize = 54;
+    for (int i = 0; i < expectedSize; i++) {
+      capacityByteArrayOutputStream.write(i);
+      assertEquals(i + 1, capacityByteArrayOutputStream.size());
+    }
+    validate(capacityByteArrayOutputStream, expectedSize);
+  }
+
+  @Test
+  public void testWriteArray() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    int v = 23;
+    writeArraysOf3(capacityByteArrayOutputStream, v);
+    validate(capacityByteArrayOutputStream, v * 3);
+  }
+
+  @Test
+  public void testWriteArrayAndInt() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    for (int i = 0; i < 23; i++) {
+      byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1)};
+      capacityByteArrayOutputStream.write(toWrite);
+      capacityByteArrayOutputStream.write((byte)(i * 3 + 2));
+      assertEquals((i + 1) * 3, capacityByteArrayOutputStream.size());
+    }
+    validate(capacityByteArrayOutputStream, 23 * 3);
+
+  }
+
+  protected CapacityByteArrayOutputStream newCapacityBAOS(int initialSize) {
+    return new CapacityByteArrayOutputStream(10, 1000000);
+  }
+
+  @Test
+  public void testReset() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    for (int i = 0; i < 54; i++) {
+      capacityByteArrayOutputStream.write(i);
+      assertEquals(i + 1, capacityByteArrayOutputStream.size());
+    }
+    capacityByteArrayOutputStream.reset();
+    for (int i = 0; i < 54; i++) {
+      capacityByteArrayOutputStream.write(54 + i);
+      assertEquals(i + 1, capacityByteArrayOutputStream.size());
+    }
+    final byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
+    assertEquals(54, byteArray.length);
+    for (int i = 0; i < 54; i++) {
+      assertEquals(i + " in " + Arrays.toString(byteArray) ,54 + i, byteArray[i]);
+    }
+  }
+
+  @Test
+  public void testWriteArrayBiggerThanSlab() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    int v = 23;
+    writeArraysOf3(capacityByteArrayOutputStream, v);
+    int n = v * 3;
+    byte[] toWrite = { // bigger than 2 slabs of size of 10
+        (byte)n, (byte)(n + 1), (byte)(n + 2), (byte)(n + 3), (byte)(n + 4), (byte)(n + 5),
+        (byte)(n + 6), (byte)(n + 7), (byte)(n + 8), (byte)(n + 9), (byte)(n + 10),
+        (byte)(n + 11), (byte)(n + 12), (byte)(n + 13), (byte)(n + 14), (byte)(n + 15),
+        (byte)(n + 16), (byte)(n + 17), (byte)(n + 18), (byte)(n + 19), (byte)(n + 20)};
+    capacityByteArrayOutputStream.write(toWrite);
+    n = n + toWrite.length;
+    assertEquals(n, capacityByteArrayOutputStream.size());
+    validate(capacityByteArrayOutputStream, n);
+    capacityByteArrayOutputStream.reset();
+    // check it works after reset too
+    capacityByteArrayOutputStream.write(toWrite);
+    assertEquals(toWrite.length, capacityByteArrayOutputStream.size());
+    byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
+    assertEquals(toWrite.length, byteArray.length);
+    for (int i = 0; i < toWrite.length; i++) {
+      assertEquals(toWrite[i], byteArray[i]);
+    }
+  }
+
+  @Test
+  public void testWriteArrayManySlabs() throws Throwable {
+    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
+    int it = 500;
+    int v = 23;
+    for (int j = 0; j < it; j++) {
+      for (int i = 0; i < v; i++) {
+        byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1), (byte)(i * 3 + 2)};
+        capacityByteArrayOutputStream.write(toWrite);
+        assertEquals((i + 1) * 3 + v * 3 * j, capacityByteArrayOutputStream.size());
+      }
+    }
+    byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
+    assertEquals(v * 3 * it, byteArray.length);
+    for (int i = 0; i < v * 3 * it; i++) {
+      assertEquals(i % (v * 3), byteArray[i]);
+    }
+    // verifying we have not created 500 * 23 / 10 slabs
+    assertTrue("slab count: " + capacityByteArrayOutputStream.getSlabCount(),capacityByteArrayOutputStream.getSlabCount() <= 20);
+    capacityByteArrayOutputStream.reset();
+    writeArraysOf3(capacityByteArrayOutputStream, v);
+    validate(capacityByteArrayOutputStream, v * 3);
+    // verifying we use less slabs now
+    assertTrue("slab count: " + capacityByteArrayOutputStream.getSlabCount(),capacityByteArrayOutputStream.getSlabCount() <= 2);
+  }
+
+  @Test
+  public void testReplaceByte() throws Throwable {
+    // test replace the first value
+    {
+      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
+      cbaos.write(10);
+      assertEquals(0, cbaos.getCurrentIndex());
+      cbaos.setByte(0, (byte) 7);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      cbaos.writeTo(baos);
+      assertEquals(7, baos.toByteArray()[0]);
+    }
+
+    // test replace value in the first slab
+    {
+      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
+      cbaos.write(10);
+      cbaos.write(13);
+      cbaos.write(15);
+      cbaos.write(17);
+      assertEquals(3, cbaos.getCurrentIndex());
+      cbaos.write(19);
+      cbaos.setByte(3, (byte) 7);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      cbaos.writeTo(baos);
+      assertArrayEquals(new byte[]{10, 13, 15, 7, 19}, baos.toByteArray());
+    }
+
+    // test replace in *not* the first slab
+    {
+      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
+
+      // advance part way through the 3rd slab
+      for (int i = 0; i < 12; i++) {
+        cbaos.write(100 + i);
+      }
+      assertEquals(11, cbaos.getCurrentIndex());
+
+      cbaos.setByte(6, (byte) 7);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      cbaos.writeTo(baos);
+      assertArrayEquals(
+        new byte[]{100, 101, 102, 103, 104, 105, 7, 107, 108, 109, 110, 111},
+        baos.toByteArray());
+    }
+
+    // test replace last value of a slab
+    {
+      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
+
+      // advance part way through the 3rd slab
+      for (int i = 0; i < 12; i++) {
+        cbaos.write(100 + i);
+      }
+      assertEquals(11, cbaos.getCurrentIndex());
+
+      cbaos.setByte(9, (byte) 7);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      cbaos.writeTo(baos);
+      assertArrayEquals(
+        new byte[]{100, 101, 102, 103, 104, 105, 106, 107, 108, 7, 110, 111},
+        baos.toByteArray());
+    }
+
+    // test replace last value
+    {
+      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
+
+      // advance part way through the 3rd slab
+      for (int i = 0; i < 12; i++) {
+        cbaos.write(100 + i);
+      }
+      assertEquals(11, cbaos.getCurrentIndex());
+
+      cbaos.setByte(11, (byte) 7);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      cbaos.writeTo(baos);
+      assertArrayEquals(
+        new byte[]{100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 7},
+        baos.toByteArray());
+    }
+
+  }
+
+  private void writeArraysOf3(CapacityByteArrayOutputStream capacityByteArrayOutputStream, int n)
+      throws IOException {
+    for (int i = 0; i < n; i++) {
+      byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1), (byte)(i * 3 + 2)};
+      capacityByteArrayOutputStream.write(toWrite);
+      assertEquals((i + 1) * 3, capacityByteArrayOutputStream.size());
+    }
+  }
+
+  private void validate(
+      CapacityByteArrayOutputStream capacityByteArrayOutputStream,
+      final int expectedSize) throws IOException {
+    final byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
+    assertEquals(expectedSize, byteArray.length);
+    for (int i = 0; i < expectedSize; i++) {
+      assertEquals(i, byteArray[i]);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPacking.java b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPacking.java
new file mode 100644
index 0000000..64679e5
--- /dev/null
+++ b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPacking.java
@@ -0,0 +1,218 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.parquet.Log;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
+
+public class TestBitPacking {
+  private static final Log LOG = Log.getLog(TestBitPacking.class);
+
+  @Test
+  public void testZero() throws IOException {
+    int bitLength = 0;
+    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    String expected = "";
+    validateEncodeDecode(bitLength, vals, expected);
+  }
+
+  @Test
+  public void testOne_0() throws IOException {
+    int[] vals = {0};
+    String expected = "00000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_1() throws IOException {
+    int[] vals = {1};
+    String expected = "10000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_0_0() throws IOException {
+    int[] vals = {0, 0};
+    String expected = "00000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_1_1() throws IOException {
+    int[] vals = {1, 1};
+    String expected = "11000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_9_1s() throws IOException {
+    int[] vals = {1, 1, 1, 1, 1, 1, 1, 1, 1};
+    String expected = "11111111 10000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_9_0s() throws IOException {
+    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0};
+    String expected = "00000000 00000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_7_0s_1_1() throws IOException {
+    int[] vals = {0, 0, 0, 0, 0, 0, 0, 1};
+    String expected = "00000001";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne_9_0s_1_1() throws IOException {
+    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    String expected = "00000000 01000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testOne() throws IOException {
+    int[] vals = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1};
+    String expected = "01001110 01000000";
+    validateEncodeDecode(1, vals, expected);
+  }
+
+  @Test
+  public void testTwo() throws IOException {
+    int[] vals = {0, 1, 2, 3, 3, 3, 2, 1, 1, 0, 0, 0, 1};
+    String expected = "00011011 11111001 01000000 01000000";
+    validateEncodeDecode(2, vals, expected);
+  }
+
+  @Test
+  public void testThree() throws IOException {
+    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 1};
+    String expected =
+        "00000101 00111001 01110111 " +
+        "00100000";
+    validateEncodeDecode(3, vals, expected);
+  }
+
+  @Test
+  public void testFour() throws IOException {
+    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1};
+    String expected = "00000001 00100011 01000101 01100111 10001001 10101011 11001101 11101111 00010000";
+    validateEncodeDecode(4, vals, expected);
+  }
+
+  @Test
+  public void testFive() throws IOException {
+    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 1};
+    String expected =
+        "00000000 01000100 00110010 00010100 11000111 " +
+        "01000010 01010100 10110110 00110101 11001111 " +
+        "10000100 01100101 00111010 01010110 11010111 " +
+        "11000110 01110101 10111110 01110111 11011111 " +
+        "00001000";
+    validateEncodeDecode(5, vals, expected);
+  }
+
+  @Test
+  public void testSix() throws IOException {
+    int[] vals = { 0, 28, 34, 35, 63, 1};
+    // 000000, 011100, 100010, 100011, 111111, 000001
+    String expected =
+        "00000001 11001000 10100011 " +
+        "11111100 00010000";
+    validateEncodeDecode(6, vals, expected);
+  }
+
+  @Test
+  public void testSeven() throws IOException {
+    int[] vals = { 0, 28, 34, 35, 63, 1, 125, 1, 1};
+    // 0000000, 0011100, 0100010, 0100011, 0111111, 0000001, 1111101, 0000001, 0000001
+    String expected =
+        "00000000 01110001 00010010 00110111 11100000 01111110 10000001 " +
+        "00000010";
+    validateEncodeDecode(7, vals, expected);
+  }
+
+  private void validateEncodeDecode(int bitLength, int[] vals, String expected)
+      throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    BitPackingWriter w = BitPacking.getBitPackingWriter(bitLength, baos);
+    for (int i : vals) {
+      w.write(i);
+    }
+    w.finish();
+    byte[] bytes = baos.toByteArray();
+    LOG.debug("vals ("+bitLength+"): " + toString(vals));
+    LOG.debug("bytes: " + toString(bytes));
+    Assert.assertEquals(expected, toString(bytes));
+    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+    BitPackingReader r = BitPacking.createBitPackingReader(bitLength, bais, vals.length);
+    int[] result = new int[vals.length];
+    for (int i = 0; i < result.length; i++) {
+      result[i] = r.read();
+    }
+    LOG.debug("result: " + toString(result));
+    assertArrayEquals(vals, result);
+  }
+
+  public static String toString(int[] vals) {
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (int i : vals) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(" ");
+      }
+      sb.append(i);
+    }
+    return sb.toString();
+  }
+
+  public static String toString(byte[] bytes) {
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (byte b : bytes) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(" ");
+      }
+      int i = b < 0 ? 256 + b : b;
+      String binaryString = Integer.toBinaryString(i);
+      for (int j = binaryString.length(); j<8; ++j) {
+        sb.append("0");
+      }
+      sb.append(binaryString);
+    }
+    return sb.toString();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
new file mode 100644
index 0000000..293b961
--- /dev/null
+++ b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
@@ -0,0 +1,40 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import org.junit.Test;
+
+public class TestByteBasedBitPackingEncoder {
+
+  @Test
+  public void testSlabBoundary() {
+    for (int i = 0; i < 32; i++) {
+      final ByteBasedBitPackingEncoder encoder = new ByteBasedBitPackingEncoder(i, Packer.BIG_ENDIAN);
+      // make sure to write more than a slab
+      for (int j = 0; j < 64 * 1024 * 32 + 10; j++) {
+        try {
+          encoder.writeInt(j);
+        } catch (Exception e) {
+          throw new RuntimeException(i + ": error writing " + j, e);
+        }
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking.java b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking.java
new file mode 100644
index 0000000..a5ce37e
--- /dev/null
+++ b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestByteBitPacking.java
@@ -0,0 +1,150 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.parquet.Log;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
+
+public class TestByteBitPacking {
+  private static final Log LOG = Log.getLog(TestByteBitPacking.class);
+
+  @Test
+  public void testPackUnPack() {
+    LOG.debug("");
+    LOG.debug("testPackUnPack");
+    for (int i = 1; i < 32; i++) {
+      LOG.debug("Width: " + i);
+      int[] unpacked = new int[32];
+      int[] values = generateValues(i);
+      packUnpack(Packer.BIG_ENDIAN.newBytePacker(i), values, unpacked);
+      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
+      Assert.assertArrayEquals("width "+i, values, unpacked);
+    }
+  }
+
+  private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
+    byte[] packed = new byte[packer.getBitWidth() * 4];
+    packer.pack32Values(values, 0, packed, 0);
+    LOG.debug("packed: " + TestBitPacking.toString(packed));
+    packer.unpack32Values(packed, 0, unpacked, 0);
+  }
+
+  private int[] generateValues(int bitWidth) {
+    int[] values = new int[32];
+    for (int j = 0; j < values.length; j++) {
+      values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
+    }
+    LOG.debug("Input:  " + TestBitPacking.toString(values));
+    return values;
+  }
+
+  @Test
+  public void testPackUnPackAgainstHandWritten() throws IOException {
+    LOG.debug("");
+    LOG.debug("testPackUnPackAgainstHandWritten");
+    for (int i = 1; i < 8; i++) {
+      LOG.debug("Width: " + i);
+      byte[] packed = new byte[i * 4];
+      int[] unpacked = new int[32];
+      int[] values = generateValues(i);
+
+      // pack generated
+      final BytePacker packer = Packer.BIG_ENDIAN.newBytePacker(i);
+      packer.pack32Values(values, 0, packed, 0);
+
+      LOG.debug("Generated: " + TestBitPacking.toString(packed));
+
+      // pack manual
+      final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
+      final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
+      for (int j = 0; j < values.length; j++) {
+        writer.write(values[j]);
+      }
+      final byte[] packedManualAsBytes = manualOut.toByteArray();
+      LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
+
+      // unpack manual
+      final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packed), 32);
+      for (int j = 0; j < unpacked.length; j++) {
+        unpacked[j] = reader.read();
+      }
+
+      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
+      Assert.assertArrayEquals("width " + i, values, unpacked);
+    }
+  }
+
+  @Test
+  public void testPackUnPackAgainstLemire() throws IOException {
+    for (Packer pack: Packer.values()) {
+      LOG.debug("");
+      LOG.debug("testPackUnPackAgainstLemire " + pack.name());
+      for (int i = 1; i < 32; i++) {
+        LOG.debug("Width: " + i);
+        int[] packed = new int[i];
+        int[] unpacked = new int[32];
+        int[] values = generateValues(i);
+
+        // pack lemire
+        final IntPacker packer = pack.newIntPacker(i);
+        packer.pack32Values(values, 0, packed, 0);
+        // convert to bytes
+        final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
+        for (int v : packed) {
+          switch(pack) {
+          case LITTLE_ENDIAN:
+            lemireOut.write((v >>>  0) & 0xFF);
+            lemireOut.write((v >>>  8) & 0xFF);
+            lemireOut.write((v >>> 16) & 0xFF);
+            lemireOut.write((v >>> 24) & 0xFF);
+            break;
+          case BIG_ENDIAN:
+            lemireOut.write((v >>> 24) & 0xFF);
+            lemireOut.write((v >>> 16) & 0xFF);
+            lemireOut.write((v >>>  8) & 0xFF);
+            lemireOut.write((v >>>  0) & 0xFF);
+            break;
+          }
+        }
+        final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
+        LOG.debug("Lemire out: " + TestBitPacking.toString(packedByLemireAsBytes));
+
+        // pack manual
+        final BytePacker bytePacker = pack.newBytePacker(i);
+        byte[] packedGenerated = new byte[i * 4];
+        bytePacker.pack32Values(values, 0, packedGenerated, 0);
+        LOG.debug("Gener. out: " + TestBitPacking.toString(packedGenerated));
+        Assert.assertEquals(pack.name() + " width " + i, TestBitPacking.toString(packedByLemireAsBytes), TestBitPacking.toString(packedGenerated));
+
+        bytePacker.unpack32Values(packedByLemireAsBytes, 0, unpacked, 0);
+        LOG.debug("Output: " + TestBitPacking.toString(unpacked));
+
+        Assert.assertArrayEquals("width " + i, values, unpacked);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestLemireBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestLemireBitPacking.java b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestLemireBitPacking.java
new file mode 100644
index 0000000..e0c97e0
--- /dev/null
+++ b/parquet-encoding/src/test/java/org/apache/parquet/column/values/bitpacking/TestLemireBitPacking.java
@@ -0,0 +1,123 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.values.bitpacking;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.parquet.Log;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader;
+import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
+
+public class TestLemireBitPacking {
+  private static final Log LOG = Log.getLog(TestLemireBitPacking.class);
+
+  @Test
+  public void testPackUnPack() {
+    for (Packer packer : Packer.values()) {
+      LOG.debug("");
+      LOG.debug("testPackUnPack");
+      for (int i = 1; i < 32; i++) {
+        LOG.debug("Width: " + i);
+        int[] values = generateValues(i);
+        int[] unpacked = new int[32];
+        {
+          packUnpack(packer.newIntPacker(i), values, unpacked);
+          LOG.debug("int based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
+          Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
+        }
+        {
+          packUnpack(packer.newBytePacker(i), values, unpacked);
+          LOG.debug("byte based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
+          Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
+        }
+      }
+    }
+  }
+
+  private void packUnpack(IntPacker packer, int[] values, int[] unpacked) {
+    int[] packed = new int[packer.getBitWidth()];
+    packer.pack32Values(values, 0, packed, 0);
+    packer.unpack32Values(packed, 0, unpacked, 0);
+  }
+
+  private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
+    byte[] packed = new byte[packer.getBitWidth() * 4];
+    packer.pack32Values(values, 0, packed, 0);
+    packer.unpack32Values(packed, 0, unpacked, 0);
+  }
+
+  private int[] generateValues(int bitWidth) {
+    int[] values = new int[32];
+    for (int j = 0; j < values.length; j++) {
+      values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
+    }
+    LOG.debug("Input:  " + TestBitPacking.toString(values));
+    return values;
+  }
+
+  @Test
+  public void testPackUnPackAgainstHandWritten() throws IOException {
+    LOG.debug("");
+    LOG.debug("testPackUnPackAgainstHandWritten");
+    for (int i = 1; i < 8; i++) {
+      LOG.debug("Width: " + i);
+      int[] packed = new int[i];
+      int[] unpacked = new int[32];
+      int[] values = generateValues(i);
+
+      // pack lemire
+      final IntPacker packer = Packer.BIG_ENDIAN.newIntPacker(i);
+      packer.pack32Values(values, 0, packed, 0);
+      // convert to ints
+      final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
+      for (int v : packed) {
+        lemireOut.write((v >>> 24) & 0xFF);
+        lemireOut.write((v >>> 16) & 0xFF);
+        lemireOut.write((v >>>  8) & 0xFF);
+        lemireOut.write((v >>>  0) & 0xFF);
+      }
+      final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
+      LOG.debug("Lemire: " + TestBitPacking.toString(packedByLemireAsBytes));
+
+      // pack manual
+      final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
+      final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
+      for (int j = 0; j < values.length; j++) {
+        writer.write(values[j]);
+      }
+      final byte[] packedManualAsBytes = manualOut.toByteArray();
+      LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
+
+      // unpack manual
+      final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packedByLemireAsBytes), 32);
+      for (int j = 0; j < unpacked.length; j++) {
+        unpacked[j] = reader.read();
+      }
+
+      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
+      Assert.assertArrayEquals("width " + i, values, unpacked);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/bytes/TestCapacityByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/bytes/TestCapacityByteArrayOutputStream.java b/parquet-encoding/src/test/java/parquet/bytes/TestCapacityByteArrayOutputStream.java
deleted file mode 100644
index 41006f1..0000000
--- a/parquet-encoding/src/test/java/parquet/bytes/TestCapacityByteArrayOutputStream.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.bytes;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.junit.Test;
-
-public class TestCapacityByteArrayOutputStream {
-
-  @Test
-  public void testWrite() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    final int expectedSize = 54;
-    for (int i = 0; i < expectedSize; i++) {
-      capacityByteArrayOutputStream.write(i);
-      assertEquals(i + 1, capacityByteArrayOutputStream.size());
-    }
-    validate(capacityByteArrayOutputStream, expectedSize);
-  }
-
-  @Test
-  public void testWriteArray() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    int v = 23;
-    writeArraysOf3(capacityByteArrayOutputStream, v);
-    validate(capacityByteArrayOutputStream, v * 3);
-  }
-
-  @Test
-  public void testWriteArrayAndInt() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    for (int i = 0; i < 23; i++) {
-      byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1)};
-      capacityByteArrayOutputStream.write(toWrite);
-      capacityByteArrayOutputStream.write((byte)(i * 3 + 2));
-      assertEquals((i + 1) * 3, capacityByteArrayOutputStream.size());
-    }
-    validate(capacityByteArrayOutputStream, 23 * 3);
-
-  }
-
-  protected CapacityByteArrayOutputStream newCapacityBAOS(int initialSize) {
-    return new CapacityByteArrayOutputStream(10, 1000000);
-  }
-
-  @Test
-  public void testReset() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    for (int i = 0; i < 54; i++) {
-      capacityByteArrayOutputStream.write(i);
-      assertEquals(i + 1, capacityByteArrayOutputStream.size());
-    }
-    capacityByteArrayOutputStream.reset();
-    for (int i = 0; i < 54; i++) {
-      capacityByteArrayOutputStream.write(54 + i);
-      assertEquals(i + 1, capacityByteArrayOutputStream.size());
-    }
-    final byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
-    assertEquals(54, byteArray.length);
-    for (int i = 0; i < 54; i++) {
-      assertEquals(i + " in " + Arrays.toString(byteArray) ,54 + i, byteArray[i]);
-    }
-  }
-
-  @Test
-  public void testWriteArrayBiggerThanSlab() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    int v = 23;
-    writeArraysOf3(capacityByteArrayOutputStream, v);
-    int n = v * 3;
-    byte[] toWrite = { // bigger than 2 slabs of size of 10
-        (byte)n, (byte)(n + 1), (byte)(n + 2), (byte)(n + 3), (byte)(n + 4), (byte)(n + 5),
-        (byte)(n + 6), (byte)(n + 7), (byte)(n + 8), (byte)(n + 9), (byte)(n + 10),
-        (byte)(n + 11), (byte)(n + 12), (byte)(n + 13), (byte)(n + 14), (byte)(n + 15),
-        (byte)(n + 16), (byte)(n + 17), (byte)(n + 18), (byte)(n + 19), (byte)(n + 20)};
-    capacityByteArrayOutputStream.write(toWrite);
-    n = n + toWrite.length;
-    assertEquals(n, capacityByteArrayOutputStream.size());
-    validate(capacityByteArrayOutputStream, n);
-    capacityByteArrayOutputStream.reset();
-    // check it works after reset too
-    capacityByteArrayOutputStream.write(toWrite);
-    assertEquals(toWrite.length, capacityByteArrayOutputStream.size());
-    byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
-    assertEquals(toWrite.length, byteArray.length);
-    for (int i = 0; i < toWrite.length; i++) {
-      assertEquals(toWrite[i], byteArray[i]);
-    }
-  }
-
-  @Test
-  public void testWriteArrayManySlabs() throws Throwable {
-    CapacityByteArrayOutputStream capacityByteArrayOutputStream = newCapacityBAOS(10);
-    int it = 500;
-    int v = 23;
-    for (int j = 0; j < it; j++) {
-      for (int i = 0; i < v; i++) {
-        byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1), (byte)(i * 3 + 2)};
-        capacityByteArrayOutputStream.write(toWrite);
-        assertEquals((i + 1) * 3 + v * 3 * j, capacityByteArrayOutputStream.size());
-      }
-    }
-    byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
-    assertEquals(v * 3 * it, byteArray.length);
-    for (int i = 0; i < v * 3 * it; i++) {
-      assertEquals(i % (v * 3), byteArray[i]);
-    }
-    // verifying we have not created 500 * 23 / 10 slabs
-    assertTrue("slab count: " + capacityByteArrayOutputStream.getSlabCount(),capacityByteArrayOutputStream.getSlabCount() <= 20);
-    capacityByteArrayOutputStream.reset();
-    writeArraysOf3(capacityByteArrayOutputStream, v);
-    validate(capacityByteArrayOutputStream, v * 3);
-    // verifying we use less slabs now
-    assertTrue("slab count: " + capacityByteArrayOutputStream.getSlabCount(),capacityByteArrayOutputStream.getSlabCount() <= 2);
-  }
-
-  @Test
-  public void testReplaceByte() throws Throwable {
-    // test replace the first value
-    {
-      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
-      cbaos.write(10);
-      assertEquals(0, cbaos.getCurrentIndex());
-      cbaos.setByte(0, (byte) 7);
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      cbaos.writeTo(baos);
-      assertEquals(7, baos.toByteArray()[0]);
-    }
-
-    // test replace value in the first slab
-    {
-      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
-      cbaos.write(10);
-      cbaos.write(13);
-      cbaos.write(15);
-      cbaos.write(17);
-      assertEquals(3, cbaos.getCurrentIndex());
-      cbaos.write(19);
-      cbaos.setByte(3, (byte) 7);
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      cbaos.writeTo(baos);
-      assertArrayEquals(new byte[]{10, 13, 15, 7, 19}, baos.toByteArray());
-    }
-
-    // test replace in *not* the first slab
-    {
-      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
-
-      // advance part way through the 3rd slab
-      for (int i = 0; i < 12; i++) {
-        cbaos.write(100 + i);
-      }
-      assertEquals(11, cbaos.getCurrentIndex());
-
-      cbaos.setByte(6, (byte) 7);
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      cbaos.writeTo(baos);
-      assertArrayEquals(
-        new byte[]{100, 101, 102, 103, 104, 105, 7, 107, 108, 109, 110, 111},
-        baos.toByteArray());
-    }
-
-    // test replace last value of a slab
-    {
-      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
-
-      // advance part way through the 3rd slab
-      for (int i = 0; i < 12; i++) {
-        cbaos.write(100 + i);
-      }
-      assertEquals(11, cbaos.getCurrentIndex());
-
-      cbaos.setByte(9, (byte) 7);
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      cbaos.writeTo(baos);
-      assertArrayEquals(
-        new byte[]{100, 101, 102, 103, 104, 105, 106, 107, 108, 7, 110, 111},
-        baos.toByteArray());
-    }
-
-    // test replace last value
-    {
-      CapacityByteArrayOutputStream cbaos = newCapacityBAOS(5);
-
-      // advance part way through the 3rd slab
-      for (int i = 0; i < 12; i++) {
-        cbaos.write(100 + i);
-      }
-      assertEquals(11, cbaos.getCurrentIndex());
-
-      cbaos.setByte(11, (byte) 7);
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      cbaos.writeTo(baos);
-      assertArrayEquals(
-        new byte[]{100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 7},
-        baos.toByteArray());
-    }
-
-  }
-
-  private void writeArraysOf3(CapacityByteArrayOutputStream capacityByteArrayOutputStream, int n)
-      throws IOException {
-    for (int i = 0; i < n; i++) {
-      byte[] toWrite = { (byte)(i * 3), (byte)(i * 3 + 1), (byte)(i * 3 + 2)};
-      capacityByteArrayOutputStream.write(toWrite);
-      assertEquals((i + 1) * 3, capacityByteArrayOutputStream.size());
-    }
-  }
-
-  private void validate(
-      CapacityByteArrayOutputStream capacityByteArrayOutputStream,
-      final int expectedSize) throws IOException {
-    final byte[] byteArray = BytesInput.from(capacityByteArrayOutputStream).toByteArray();
-    assertEquals(expectedSize, byteArray.length);
-    for (int i = 0; i < expectedSize; i++) {
-      assertEquals(i, byteArray[i]);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestBitPacking.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestBitPacking.java
deleted file mode 100644
index f8d7c3e..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestBitPacking.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import static org.junit.Assert.assertArrayEquals;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-public class TestBitPacking {
-  private static final Log LOG = Log.getLog(TestBitPacking.class);
-
-  @Test
-  public void testZero() throws IOException {
-    int bitLength = 0;
-    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-    String expected = "";
-    validateEncodeDecode(bitLength, vals, expected);
-  }
-
-  @Test
-  public void testOne_0() throws IOException {
-    int[] vals = {0};
-    String expected = "00000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_1() throws IOException {
-    int[] vals = {1};
-    String expected = "10000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_0_0() throws IOException {
-    int[] vals = {0, 0};
-    String expected = "00000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_1_1() throws IOException {
-    int[] vals = {1, 1};
-    String expected = "11000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_9_1s() throws IOException {
-    int[] vals = {1, 1, 1, 1, 1, 1, 1, 1, 1};
-    String expected = "11111111 10000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_9_0s() throws IOException {
-    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0};
-    String expected = "00000000 00000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_7_0s_1_1() throws IOException {
-    int[] vals = {0, 0, 0, 0, 0, 0, 0, 1};
-    String expected = "00000001";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne_9_0s_1_1() throws IOException {
-    int[] vals = {0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-    String expected = "00000000 01000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testOne() throws IOException {
-    int[] vals = {0, 1, 0, 0, 1, 1, 1, 0, 0, 1};
-    String expected = "01001110 01000000";
-    validateEncodeDecode(1, vals, expected);
-  }
-
-  @Test
-  public void testTwo() throws IOException {
-    int[] vals = {0, 1, 2, 3, 3, 3, 2, 1, 1, 0, 0, 0, 1};
-    String expected = "00011011 11111001 01000000 01000000";
-    validateEncodeDecode(2, vals, expected);
-  }
-
-  @Test
-  public void testThree() throws IOException {
-    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 1};
-    String expected =
-        "00000101 00111001 01110111 " +
-        "00100000";
-    validateEncodeDecode(3, vals, expected);
-  }
-
-  @Test
-  public void testFour() throws IOException {
-    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1};
-    String expected = "00000001 00100011 01000101 01100111 10001001 10101011 11001101 11101111 00010000";
-    validateEncodeDecode(4, vals, expected);
-  }
-
-  @Test
-  public void testFive() throws IOException {
-    int[] vals = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 1};
-    String expected =
-        "00000000 01000100 00110010 00010100 11000111 " +
-        "01000010 01010100 10110110 00110101 11001111 " +
-        "10000100 01100101 00111010 01010110 11010111 " +
-        "11000110 01110101 10111110 01110111 11011111 " +
-        "00001000";
-    validateEncodeDecode(5, vals, expected);
-  }
-
-  @Test
-  public void testSix() throws IOException {
-    int[] vals = { 0, 28, 34, 35, 63, 1};
-    // 000000, 011100, 100010, 100011, 111111, 000001
-    String expected =
-        "00000001 11001000 10100011 " +
-        "11111100 00010000";
-    validateEncodeDecode(6, vals, expected);
-  }
-
-  @Test
-  public void testSeven() throws IOException {
-    int[] vals = { 0, 28, 34, 35, 63, 1, 125, 1, 1};
-    // 0000000, 0011100, 0100010, 0100011, 0111111, 0000001, 1111101, 0000001, 0000001
-    String expected =
-        "00000000 01110001 00010010 00110111 11100000 01111110 10000001 " +
-        "00000010";
-    validateEncodeDecode(7, vals, expected);
-  }
-
-  private void validateEncodeDecode(int bitLength, int[] vals, String expected)
-      throws IOException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    BitPackingWriter w = BitPacking.getBitPackingWriter(bitLength, baos);
-    for (int i : vals) {
-      w.write(i);
-    }
-    w.finish();
-    byte[] bytes = baos.toByteArray();
-    LOG.debug("vals ("+bitLength+"): " + toString(vals));
-    LOG.debug("bytes: " + toString(bytes));
-    Assert.assertEquals(expected, toString(bytes));
-    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-    BitPackingReader r = BitPacking.createBitPackingReader(bitLength, bais, vals.length);
-    int[] result = new int[vals.length];
-    for (int i = 0; i < result.length; i++) {
-      result[i] = r.read();
-    }
-    LOG.debug("result: " + toString(result));
-    assertArrayEquals(vals, result);
-  }
-
-  public static String toString(int[] vals) {
-    StringBuilder sb = new StringBuilder();
-    boolean first = true;
-    for (int i : vals) {
-      if (first) {
-        first = false;
-      } else {
-        sb.append(" ");
-      }
-      sb.append(i);
-    }
-    return sb.toString();
-  }
-
-  public static String toString(byte[] bytes) {
-    StringBuilder sb = new StringBuilder();
-    boolean first = true;
-    for (byte b : bytes) {
-      if (first) {
-        first = false;
-      } else {
-        sb.append(" ");
-      }
-      int i = b < 0 ? 256 + b : b;
-      String binaryString = Integer.toBinaryString(i);
-      for (int j = binaryString.length(); j<8; ++j) {
-        sb.append("0");
-      }
-      sb.append(binaryString);
-    }
-    return sb.toString();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
deleted file mode 100644
index 820c5d2..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBasedBitPackingEncoder.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import org.junit.Test;
-
-public class TestByteBasedBitPackingEncoder {
-
-  @Test
-  public void testSlabBoundary() {
-    for (int i = 0; i < 32; i++) {
-      final ByteBasedBitPackingEncoder encoder = new ByteBasedBitPackingEncoder(i, Packer.BIG_ENDIAN);
-      // make sure to write more than a slab
-      for (int j = 0; j < 64 * 1024 * 32 + 10; j++) {
-        try {
-          encoder.writeInt(j);
-        } catch (Exception e) {
-          throw new RuntimeException(i + ": error writing " + j, e);
-        }
-      }
-    }
-  }
-
-}


Mime
View raw message