parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [16/51] [partial] parquet-mr git commit: PARQUET-23: Rename to org.apache.parquet.
Date Mon, 27 Apr 2015 23:12:13 GMT
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
deleted file mode 100644
index 6c1ffb7..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestByteBitPacking.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-public class TestByteBitPacking {
-  private static final Log LOG = Log.getLog(TestByteBitPacking.class);
-
-  @Test
-  public void testPackUnPack() {
-    LOG.debug("");
-    LOG.debug("testPackUnPack");
-    for (int i = 1; i < 32; i++) {
-      LOG.debug("Width: " + i);
-      int[] unpacked = new int[32];
-      int[] values = generateValues(i);
-      packUnpack(Packer.BIG_ENDIAN.newBytePacker(i), values, unpacked);
-      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
-      Assert.assertArrayEquals("width "+i, values, unpacked);
-    }
-  }
-
-  private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
-    byte[] packed = new byte[packer.getBitWidth() * 4];
-    packer.pack32Values(values, 0, packed, 0);
-    LOG.debug("packed: " + TestBitPacking.toString(packed));
-    packer.unpack32Values(packed, 0, unpacked, 0);
-  }
-
-  private int[] generateValues(int bitWidth) {
-    int[] values = new int[32];
-    for (int j = 0; j < values.length; j++) {
-      values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
-    }
-    LOG.debug("Input:  " + TestBitPacking.toString(values));
-    return values;
-  }
-
-  @Test
-  public void testPackUnPackAgainstHandWritten() throws IOException {
-    LOG.debug("");
-    LOG.debug("testPackUnPackAgainstHandWritten");
-    for (int i = 1; i < 8; i++) {
-      LOG.debug("Width: " + i);
-      byte[] packed = new byte[i * 4];
-      int[] unpacked = new int[32];
-      int[] values = generateValues(i);
-
-      // pack generated
-      final BytePacker packer = Packer.BIG_ENDIAN.newBytePacker(i);
-      packer.pack32Values(values, 0, packed, 0);
-
-      LOG.debug("Generated: " + TestBitPacking.toString(packed));
-
-      // pack manual
-      final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
-      final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
-      for (int j = 0; j < values.length; j++) {
-        writer.write(values[j]);
-      }
-      final byte[] packedManualAsBytes = manualOut.toByteArray();
-      LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
-
-      // unpack manual
-      final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packed), 32);
-      for (int j = 0; j < unpacked.length; j++) {
-        unpacked[j] = reader.read();
-      }
-
-      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
-      Assert.assertArrayEquals("width " + i, values, unpacked);
-    }
-  }
-
-  @Test
-  public void testPackUnPackAgainstLemire() throws IOException {
-    for (Packer pack: Packer.values()) {
-      LOG.debug("");
-      LOG.debug("testPackUnPackAgainstLemire " + pack.name());
-      for (int i = 1; i < 32; i++) {
-        LOG.debug("Width: " + i);
-        int[] packed = new int[i];
-        int[] unpacked = new int[32];
-        int[] values = generateValues(i);
-
-        // pack lemire
-        final IntPacker packer = pack.newIntPacker(i);
-        packer.pack32Values(values, 0, packed, 0);
-        // convert to bytes
-        final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
-        for (int v : packed) {
-          switch(pack) {
-          case LITTLE_ENDIAN:
-            lemireOut.write((v >>>  0) & 0xFF);
-            lemireOut.write((v >>>  8) & 0xFF);
-            lemireOut.write((v >>> 16) & 0xFF);
-            lemireOut.write((v >>> 24) & 0xFF);
-            break;
-          case BIG_ENDIAN:
-            lemireOut.write((v >>> 24) & 0xFF);
-            lemireOut.write((v >>> 16) & 0xFF);
-            lemireOut.write((v >>>  8) & 0xFF);
-            lemireOut.write((v >>>  0) & 0xFF);
-            break;
-          }
-        }
-        final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
-        LOG.debug("Lemire out: " + TestBitPacking.toString(packedByLemireAsBytes));
-
-        // pack manual
-        final BytePacker bytePacker = pack.newBytePacker(i);
-        byte[] packedGenerated = new byte[i * 4];
-        bytePacker.pack32Values(values, 0, packedGenerated, 0);
-        LOG.debug("Gener. out: " + TestBitPacking.toString(packedGenerated));
-        Assert.assertEquals(pack.name() + " width " + i, TestBitPacking.toString(packedByLemireAsBytes), TestBitPacking.toString(packedGenerated));
-
-        bytePacker.unpack32Values(packedByLemireAsBytes, 0, unpacked, 0);
-        LOG.debug("Output: " + TestBitPacking.toString(unpacked));
-
-        Assert.assertArrayEquals("width " + i, values, unpacked);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
----------------------------------------------------------------------
diff --git a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java b/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
deleted file mode 100644
index 0929865..0000000
--- a/parquet-encoding/src/test/java/parquet/column/values/bitpacking/TestLemireBitPacking.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.column.values.bitpacking;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.column.values.bitpacking.BitPacking.BitPackingReader;
-import parquet.column.values.bitpacking.BitPacking.BitPackingWriter;
-
-public class TestLemireBitPacking {
-  private static final Log LOG = Log.getLog(TestLemireBitPacking.class);
-
-  @Test
-  public void testPackUnPack() {
-    for (Packer packer : Packer.values()) {
-      LOG.debug("");
-      LOG.debug("testPackUnPack");
-      for (int i = 1; i < 32; i++) {
-        LOG.debug("Width: " + i);
-        int[] values = generateValues(i);
-        int[] unpacked = new int[32];
-        {
-          packUnpack(packer.newIntPacker(i), values, unpacked);
-          LOG.debug("int based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
-          Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
-        }
-        {
-          packUnpack(packer.newBytePacker(i), values, unpacked);
-          LOG.debug("byte based Output " + packer.name() + ": " + TestBitPacking.toString(unpacked));
-          Assert.assertArrayEquals(packer.name() + " width "+i, values, unpacked);
-        }
-      }
-    }
-  }
-
-  private void packUnpack(IntPacker packer, int[] values, int[] unpacked) {
-    int[] packed = new int[packer.getBitWidth()];
-    packer.pack32Values(values, 0, packed, 0);
-    packer.unpack32Values(packed, 0, unpacked, 0);
-  }
-
-  private void packUnpack(BytePacker packer, int[] values, int[] unpacked) {
-    byte[] packed = new byte[packer.getBitWidth() * 4];
-    packer.pack32Values(values, 0, packed, 0);
-    packer.unpack32Values(packed, 0, unpacked, 0);
-  }
-
-  private int[] generateValues(int bitWidth) {
-    int[] values = new int[32];
-    for (int j = 0; j < values.length; j++) {
-      values[j] = (int)(Math.random() * 100000) % (int)Math.pow(2, bitWidth);
-    }
-    LOG.debug("Input:  " + TestBitPacking.toString(values));
-    return values;
-  }
-
-  @Test
-  public void testPackUnPackAgainstHandWritten() throws IOException {
-    LOG.debug("");
-    LOG.debug("testPackUnPackAgainstHandWritten");
-    for (int i = 1; i < 8; i++) {
-      LOG.debug("Width: " + i);
-      int[] packed = new int[i];
-      int[] unpacked = new int[32];
-      int[] values = generateValues(i);
-
-      // pack lemire
-      final IntPacker packer = Packer.BIG_ENDIAN.newIntPacker(i);
-      packer.pack32Values(values, 0, packed, 0);
-      // convert to ints
-      final ByteArrayOutputStream lemireOut = new ByteArrayOutputStream();
-      for (int v : packed) {
-        lemireOut.write((v >>> 24) & 0xFF);
-        lemireOut.write((v >>> 16) & 0xFF);
-        lemireOut.write((v >>>  8) & 0xFF);
-        lemireOut.write((v >>>  0) & 0xFF);
-      }
-      final byte[] packedByLemireAsBytes = lemireOut.toByteArray();
-      LOG.debug("Lemire: " + TestBitPacking.toString(packedByLemireAsBytes));
-
-      // pack manual
-      final ByteArrayOutputStream manualOut = new ByteArrayOutputStream();
-      final BitPackingWriter writer = BitPacking.getBitPackingWriter(i, manualOut);
-      for (int j = 0; j < values.length; j++) {
-        writer.write(values[j]);
-      }
-      final byte[] packedManualAsBytes = manualOut.toByteArray();
-      LOG.debug("Manual: " + TestBitPacking.toString(packedManualAsBytes));
-
-      // unpack manual
-      final BitPackingReader reader = BitPacking.createBitPackingReader(i, new ByteArrayInputStream(packedByLemireAsBytes), 32);
-      for (int j = 0; j < unpacked.length; j++) {
-        unpacked[j] = reader.read();
-      }
-
-      LOG.debug("Output: " + TestBitPacking.toString(unpacked));
-      Assert.assertArrayEquals("width " + i, values, unpacked);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
index 1931d59..52c0cad 100644
--- a/parquet-generator/pom.xml
+++ b/parquet-generator/pom.xml
@@ -18,7 +18,7 @@
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
-    <groupId>com.twitter</groupId>
+    <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
     <version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,7 +37,7 @@
 
   <dependencies>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-common</artifactId>
       <version>${project.version}</version>
     </dependency>

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
new file mode 100644
index 0000000..641e76d
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/Generator.java
@@ -0,0 +1,37 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding;
+
+import org.apache.parquet.encoding.bitpacking.ByteBasedBitPackingGenerator;
+import org.apache.parquet.encoding.bitpacking.IntBasedBitPackingGenerator;
+
+/**
+ * main class for code generation hook in build for encodings generation
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class Generator {
+
+  public static void main(String[] args) throws Exception {
+    IntBasedBitPackingGenerator.main(args);
+    ByteBasedBitPackingGenerator.main(args);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
new file mode 100644
index 0000000..2249cbc
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
@@ -0,0 +1,259 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding.bitpacking;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.apache.parquet.bytes.BytesUtils;
+
+/**
+ *
+ * This class generates bit packers that pack the most significant bit first.
+ * The result of the generation is checked in. To regenerate the code run this class and check in the result.
+ *
+ * TODO: remove the unnecessary masks for perf
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class ByteBasedBitPackingGenerator {
+
+  private static final String CLASS_NAME_PREFIX = "ByteBitPacking";
+  private static final int PACKER_COUNT = 32;
+
+  public static void main(String[] args) throws Exception {
+    String basePath = args[0];
+    generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
+    generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
+  }
+
+  private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
+    final File file = new File(basePath + "/org/apache/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
+    if (!file.getParentFile().exists()) {
+      file.getParentFile().mkdirs();
+    }
+    FileWriter fw = new FileWriter(file);
+    fw.append("package org.apache.parquet.column.values.bitpacking;\n");
+    fw.append("\n");
+    fw.append("/**\n");
+    if (msbFirst) {
+      fw.append(" * Packs from the Most Significant Bit first\n");
+    } else {
+      fw.append(" * Packs from the Least Significant Bit first\n");
+    }
+    fw.append(" * \n");
+    fw.append(" * @author automatically generated\n");
+    fw.append(" * @see ByteBasedBitPackingGenerator\n");
+    fw.append(" *\n");
+    fw.append(" */\n");
+    fw.append("public abstract class " + className + " {\n");
+    fw.append("\n");
+    fw.append("  private static final BytePacker[] packers = new BytePacker[33];\n");
+    fw.append("  static {\n");
+    for (int i = 0; i <= PACKER_COUNT; i++) {
+      fw.append("    packers[" + i + "] = new Packer" + i + "();\n");
+    }
+    fw.append("  }\n");
+    fw.append("\n");
+    fw.append("  public static final BytePackerFactory factory = new BytePackerFactory() {\n");
+    fw.append("    public BytePacker newBytePacker(int bitWidth) {\n");
+    fw.append("      return packers[bitWidth];\n");
+    fw.append("    }\n");
+    fw.append("  };\n");
+    fw.append("\n");
+    for (int i = 0; i <= PACKER_COUNT; i++) {
+      generateClass(fw, i, msbFirst);
+      fw.append("\n");
+    }
+    fw.append("}\n");
+    fw.close();
+  }
+
+  private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
+    fw.append("  private static final class Packer" + bitWidth + " extends BytePacker {\n");
+    fw.append("\n");
+    fw.append("    private Packer" + bitWidth + "() {\n");
+    fw.append("      super("+bitWidth+");\n");
+    fw.append("    }\n");
+    fw.append("\n");
+    // Packing
+    generatePack(fw, bitWidth, 1, msbFirst);
+    generatePack(fw, bitWidth, 4, msbFirst);
+
+    // Unpacking
+    generateUnpack(fw, bitWidth, 1, msbFirst);
+    generateUnpack(fw, bitWidth, 4, msbFirst);
+
+    fw.append("  }\n");
+  }
+
+  private static int getShift(FileWriter fw, int bitWidth, boolean msbFirst,
+      int byteIndex, int valueIndex) throws IOException {
+    // relative positions of the start and end of the value to the start and end of the byte
+    int valueStartBitIndex = (valueIndex * bitWidth) - (8 * (byteIndex));
+    int valueEndBitIndex = ((valueIndex + 1) * bitWidth) - (8 * (byteIndex + 1));
+
+    // boundaries of the current value that we want
+    int valueStartBitWanted;
+    int valueEndBitWanted;
+    // boundaries of the current byte that will receive them
+    int byteStartBitWanted;
+    int byteEndBitWanted;
+
+    int shift;
+
+    if (msbFirst) {
+      valueStartBitWanted = valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1;
+      valueEndBitWanted = valueEndBitIndex > 0 ? valueEndBitIndex : 0;
+      byteStartBitWanted = valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex;
+      byteEndBitWanted = valueEndBitIndex > 0 ? 0 : -valueEndBitIndex;
+      shift = valueEndBitWanted - byteEndBitWanted;
+    } else {
+      valueStartBitWanted = bitWidth - 1 - (valueEndBitIndex > 0 ? valueEndBitIndex : 0);
+      valueEndBitWanted = bitWidth - 1 - (valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1);
+      byteStartBitWanted = 7 - (valueEndBitIndex > 0 ? 0 : -valueEndBitIndex);
+      byteEndBitWanted = 7 - (valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex);
+      shift = valueStartBitWanted - byteStartBitWanted;
+    }
+
+    visualizeAlignment(
+        fw, bitWidth, valueEndBitIndex,
+        valueStartBitWanted, valueEndBitWanted,
+        byteStartBitWanted, byteEndBitWanted,
+        shift
+        );
+    return shift;
+  }
+
+  private static void visualizeAlignment(FileWriter fw, int bitWidth,
+      int valueEndBitIndex, int valueStartBitWanted, int valueEndBitWanted,
+      int byteStartBitWanted, int byteEndBitWanted, int shift) throws IOException {
+    // ASCII art to visualize what is happening
+    fw.append("//");
+    int buf = 2 + Math.max(0, bitWidth + 8);
+    for (int i = 0; i < buf; i++) {
+      fw.append(" ");
+    }
+    fw.append("[");
+    for (int i = 7; i >= 0; i--) {
+      if (i<=byteStartBitWanted && i>=byteEndBitWanted) {
+        fw.append(String.valueOf(i));
+      } else {
+        fw.append("_");
+      }
+    }
+    fw.append("]\n          //");
+    for (int i = 0; i < buf + (8 - bitWidth + shift); i++) {
+      fw.append(" ");
+    }
+    fw.append("[");
+    for (int i = bitWidth - 1; i >= 0 ; i--) {
+      if (i<=valueStartBitWanted && i>=valueEndBitWanted) {
+        fw.append(String.valueOf(i % 10));
+      } else {
+        fw.append("_");
+      }
+    }
+    fw.append("]\n");
+    fw.append("           ");
+  }
+
+  private static void generatePack(FileWriter fw, int bitWidth, int batch, boolean msbFirst) throws IOException {
+    int mask = genMask(bitWidth);
+    fw.append("    public final void pack" + (batch * 8) + "Values(final int[] in, final int inPos, final byte[] out, final int outPos) {\n");
+    for (int byteIndex = 0; byteIndex < bitWidth * batch; ++byteIndex) {
+      fw.append("      out[" + align(byteIndex, 2) + " + outPos] = (byte)((\n");
+      int startIndex = (byteIndex * 8) / bitWidth;
+      int endIndex = ((byteIndex + 1) * 8 + bitWidth - 1) / bitWidth;
+      for (int valueIndex = startIndex; valueIndex < endIndex; valueIndex++) {
+
+        if (valueIndex == startIndex) {
+          fw.append("          ");
+        } else {
+          fw.append("\n        | ");
+        }
+        int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
+
+        String shiftString = ""; // used when shift == 0
+        if (shift > 0) {
+          shiftString = " >>> " + shift;
+        } else if (shift < 0) {
+          shiftString = " <<  " + ( - shift);
+        }
+        fw.append("((in[" + align(valueIndex, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
+      }
+      fw.append(") & 255);\n");
+    }
+    fw.append("    }\n");
+  }
+
+  private static void generateUnpack(FileWriter fw, int bitWidth, int batch, boolean msbFirst)
+      throws IOException {
+    fw.append("    public final void unpack" + (batch * 8) + "Values(final byte[] in, final int inPos, final int[] out, final int outPos) {\n");
+    if (bitWidth > 0) {
+      int mask = genMask(bitWidth);
+      for (int valueIndex = 0; valueIndex < (batch * 8); ++valueIndex) {
+        fw.append("      out[" + align(valueIndex, 2) + " + outPos] =\n");
+
+        int startIndex = valueIndex * bitWidth / 8;
+        int endIndex = BytesUtils.paddedByteCountFromBits((valueIndex + 1) * bitWidth);
+
+        for (int byteIndex = startIndex; byteIndex < endIndex; byteIndex++) {
+          if (byteIndex == startIndex) {
+            fw.append("          ");
+          } else {
+            fw.append("\n        | ");
+          }
+          int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
+
+          String shiftString = ""; // when shift == 0
+          if (shift < 0) {
+            shiftString = ">>>  " + (-shift);
+          } else if (shift > 0){
+            shiftString = "<<  " + shift;
+          }
+          fw.append(" (((((int)in[" + align(byteIndex, 2) + " + inPos]) & 255) " + shiftString + ") & " + mask + ")");
+        }
+        fw.append(";\n");
+      }
+    }
+    fw.append("    }\n");
+  }
+
+  private static int genMask(int bitWidth) {
+    int mask = 0;
+    for (int i = 0; i < bitWidth; i++) {
+      mask <<= 1;
+      mask |= 1;
+    }
+    return mask;
+  }
+
+  private static String align(int value, int digits) {
+    final String valueString = String.valueOf(value);
+    StringBuilder result = new StringBuilder();
+    for (int i = valueString.length(); i < digits; i++) {
+      result.append(" ");
+    }
+    result.append(valueString);
+    return result.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
new file mode 100644
index 0000000..e472663
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
@@ -0,0 +1,211 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.encoding.bitpacking;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+/**
+ * Scheme designed by D. Lemire
+ *
+ * This is a re-implementation of The scheme released under Apache License Version 2.0
+ * at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java
+ *
+ * It generate two classes:
+ * - LemireBitPackingLE, the original scheme, filling the LSB first
+ * - LemireBitPackingBE, the scheme modified to fill the MSB first (and match our existing bit packing)
+ *
+ * The result of the generation is checked in. To regenerate the code run this class and check in the result.
+ *
+ * The generated classes pack the values into arrays of ints (as opposed to arrays of bytes) based on a given bit width.
+ *
+ * Note: This is not really used for now as the hadoop API does not really let write int[]. We need to revisit this
+ *
+ * @author Julien Le Dem
+ *
+ */
+public class IntBasedBitPackingGenerator {
+
+  private static final String CLASS_NAME_PREFIX = "LemireBitPacking";
+
+  public static void main(String[] args) throws Exception {
+    String basePath = args[0];
+    generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
+    generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
+  }
+
+  private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
+    final File file = new File(basePath + "/org/apache/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
+    if (!file.getParentFile().exists()) {
+      file.getParentFile().mkdirs();
+    }
+    FileWriter fw = new FileWriter(file);
+    fw.append("package org.apache.parquet.column.values.bitpacking;\n");
+    fw.append("\n");
+    fw.append("/**\n");
+    fw.append(" * Based on the original implementation at at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java\n");
+    fw.append(" * Which is released under the\n");
+    fw.append(" * Apache License Version 2.0 http://www.apache.org/licenses/.\n");
+    fw.append(" * By Daniel Lemire, http://lemire.me/en/\n");
+    fw.append(" * \n");
+    fw.append(" * Scheme designed by D. Lemire\n");
+    if (msbFirst) {
+      fw.append(" * Adapted to pack from the Most Significant Bit first\n");
+    }
+    fw.append(" * \n");
+    fw.append(" * @author automatically generated\n");
+    fw.append(" * @see IntBasedBitPackingGenerator\n");
+    fw.append(" *\n");
+    fw.append(" */\n");
+    fw.append("abstract class " + className + " {\n");
+    fw.append("\n");
+    fw.append("  private static final IntPacker[] packers = new IntPacker[32];\n");
+    fw.append("  static {\n");
+    for (int i = 0; i < 32; i++) {
+      fw.append("    packers[" + i + "] = new Packer" + i + "();\n");
+    }
+    fw.append("  }\n");
+    fw.append("\n");
+    fw.append("  public static final IntPackerFactory factory = new IntPackerFactory() {\n");
+    fw.append("    public IntPacker newIntPacker(int bitWidth) {\n");
+    fw.append("      return packers[bitWidth];\n");
+    fw.append("    }\n");
+    fw.append("  };\n");
+    fw.append("\n");
+    for (int i = 0; i < 32; i++) {
+      generateClass(fw, i, msbFirst);
+      fw.append("\n");
+    }
+    fw.append("}\n");
+    fw.close();
+  }
+
+  private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
+    int mask = 0;
+    for (int i = 0; i < bitWidth; i++) {
+      mask <<= 1;
+      mask |= 1;
+    }
+    fw.append("  private static final class Packer" + bitWidth + " extends IntPacker {\n");
+    fw.append("\n");
+    fw.append("    private Packer" + bitWidth + "() {\n");
+    fw.append("      super(" + bitWidth + ");\n");
+    fw.append("    }\n");
+    fw.append("\n");
+    // Packing
+    fw.append("    public final void pack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
+    for (int i = 0; i < bitWidth; ++i) {
+      fw.append("      out[" + align(i, 2) + " + outPos] =\n");
+      int startIndex = (i * 32) / bitWidth;
+      int endIndex = ((i + 1) * 32 + bitWidth - 1) / bitWidth;
+      for (int j = startIndex; j < endIndex; j++) {
+        if (j == startIndex) {
+          fw.append("          ");
+        } else {
+          fw.append("\n        | ");
+        }
+        String shiftString = getPackShiftString(bitWidth, i, startIndex, j, msbFirst);
+        fw.append("((in[" + align(j, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
+      }
+      fw.append(";\n");
+    }
+    fw.append("    }\n");
+
+    // Unpacking
+    fw.append("    public final void unpack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
+    if (bitWidth > 0) {
+      for (int i = 0; i < 32; ++i) {
+        fw.append("      out[" + align(i, 2) + " + outPos] =");
+        int byteIndex = i * bitWidth / 32;
+        String shiftString = getUnpackShiftString(bitWidth, i, msbFirst);
+        fw.append(" ((in[" + align(byteIndex, 2) + " + inPos] " + shiftString + ") & " + mask + ")");
+        if (((i + 1) * bitWidth - 1 ) / 32 != byteIndex) {
+          // reading the end of the value from next int
+          int bitsRead = ((i + 1) * bitWidth - 1) % 32 + 1;
+          fw.append(" | ((in[" + align(byteIndex + 1, 2) + " + inPos]");
+          if (msbFirst) {
+            fw.append(") >>> " + align(32 - bitsRead, 2) + ")");
+          } else {
+            int lowerMask = 0;
+            for (int j = 0; j < bitsRead; j++) {
+              lowerMask <<= 1;
+              lowerMask |= 1;
+            }
+            fw.append(" & " + lowerMask + ") << " + align(bitWidth - bitsRead, 2) + ")");
+          }
+        }
+        fw.append(";\n");
+      }
+    }
+    fw.append("    }\n");
+    fw.append("  }\n");
+  }
+
+  private static String getUnpackShiftString(int bitWidth, int i, boolean msbFirst) {
+    final int regularShift = i * bitWidth % 32;
+    String shiftString;
+    if (msbFirst) {
+      int shift = 32 - (regularShift + bitWidth);
+      if (shift < 0) {
+        shiftString = "<<  " + align(-shift, 2);
+      } else {
+        shiftString = ">>> " + align(shift, 2);
+      }
+    } else {
+      shiftString = ">>> " + align(regularShift, 2);
+    }
+    return shiftString;
+  }
+
+  private static String getPackShiftString(int bitWidth, int integerIndex, int startIndex, int valueIndex, boolean msbFirst) {
+    String shiftString;
+    int regularShift = (valueIndex * bitWidth) % 32;
+    if (msbFirst) { // filling most significant bit first
+      int shift = 32 - (regularShift + bitWidth);
+      if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
+        // end of last value from previous int
+          shiftString = " <<  " + align(32 - (((valueIndex + 1) * bitWidth) % 32), 2);
+      } else if (shift < 0) {
+        // partial last value
+          shiftString = " >>> " + align(-shift, 2);
+      } else {
+        shiftString = " <<  " + align(shift, 2);
+      }
+    } else { // filling least significant bit first
+      if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
+        // end of last value from previous int
+        shiftString = " >>> " + align(32 - regularShift, 2);
+      } else {
+        shiftString = " <<  " + align(regularShift, 2);
+      }
+    }
+    return shiftString;
+  }
+
+  private static String align(int value, int digits) {
+    final String valueString = String.valueOf(value);
+    StringBuilder result = new StringBuilder();
+    for (int i = valueString.length(); i < digits; i++) {
+      result.append(" ");
+    }
+    result.append(valueString);
+    return result.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java b/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
new file mode 100644
index 0000000..042015c
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/filter2/Generator.java
@@ -0,0 +1,28 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.filter2;
+
+/**
+ * main class for code generation hook in build for filter2 generation
+ */
+public class Generator {
+  public static void main(String[] args) throws Exception {
+    IncrementallyUpdatedFilterPredicateGenerator.main(args);
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java b/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
new file mode 100644
index 0000000..1dfaf6f
--- /dev/null
+++ b/parquet-generator/src/main/java/org/apache/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
@@ -0,0 +1,269 @@
+/* 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.filter2;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class IncrementallyUpdatedFilterPredicateGenerator {
+
+  public static void main(String[] args) throws IOException {
+    File srcFile = new File(args[0] + "/org/apache/parquet/filter2/recordlevel/IncrementallyUpdatedFilterPredicateBuilder.java");
+    srcFile = srcFile.getAbsoluteFile();
+    File parent = srcFile.getParentFile();
+    if (!parent.exists()) {
+      if (!parent.mkdirs()) {
+        throw new IOException("Couldn't mkdirs for " + parent);
+      }
+    }
+    new IncrementallyUpdatedFilterPredicateGenerator(srcFile).run();
+  }
+
+  private final FileWriter writer;
+
+  public IncrementallyUpdatedFilterPredicateGenerator(File file) throws IOException {
+    this.writer = new FileWriter(file);
+  }
+
+  private static class TypeInfo {
+    public final String className;
+    public final String primitiveName;
+    public final boolean useComparable;
+    public final boolean supportsInequality;
+
+    private TypeInfo(String className, String primitiveName, boolean useComparable, boolean supportsInequality) {
+      this.className = className;
+      this.primitiveName = primitiveName;
+      this.useComparable = useComparable;
+      this.supportsInequality = supportsInequality;
+    }
+  }
+
+  private static final TypeInfo[] TYPES = new TypeInfo[]{
+    new TypeInfo("Integer", "int", false, true),
+    new TypeInfo("Long", "long", false, true),
+    new TypeInfo("Boolean", "boolean", false, false),
+    new TypeInfo("Float", "float", false, true),
+    new TypeInfo("Double", "double", false, true),
+    new TypeInfo("Binary", "Binary", true, true),
+  };
+
+  public void run() throws IOException {
+    add("package org.apache.parquet.filter2.recordlevel;\n" +
+        "\n" +
+        "import org.apache.parquet.hadoop.metadata.ColumnPath;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.Eq;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.Gt;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.GtEq;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.Lt;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.LtEq;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.NotEq;\n" +
+        "import org.apache.parquet.filter2.predicate.Operators.UserDefined;\n" +
+        "import org.apache.parquet.filter2.predicate.UserDefinedPredicate;\n" +
+        "import org.apache.parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;\n" +
+        "import org.apache.parquet.io.api.Binary;\n\n" +
+        "/**\n" +
+        " * This class is auto-generated by {@link parquet.filter2.IncrementallyUpdatedFilterPredicateGenerator}\n" +
+        " * Do not manually edit!\n" +
+        " * See {@link IncrementallyUpdatedFilterPredicateBuilderBase}\n" +
+        " */\n");
+
+    add("public class IncrementallyUpdatedFilterPredicateBuilder extends IncrementallyUpdatedFilterPredicateBuilderBase {\n\n");
+
+    addVisitBegin("Eq");
+    for (TypeInfo info : TYPES) {
+      addEqNotEqCase(info, true);
+    }
+    addVisitEnd();
+
+    addVisitBegin("NotEq");
+    for (TypeInfo info : TYPES) {
+      addEqNotEqCase(info, false);
+    }
+    addVisitEnd();
+
+    addVisitBegin("Lt");
+    for (TypeInfo info : TYPES) {
+      addInequalityCase(info, "<");
+    }
+    addVisitEnd();
+
+    addVisitBegin("LtEq");
+    for (TypeInfo info : TYPES) {
+      addInequalityCase(info, "<=");
+    }
+    addVisitEnd();
+
+    addVisitBegin("Gt");
+    for (TypeInfo info : TYPES) {
+      addInequalityCase(info, ">");
+    }
+    addVisitEnd();
+
+    addVisitBegin("GtEq");
+    for (TypeInfo info : TYPES) {
+      addInequalityCase(info, ">=");
+    }
+    addVisitEnd();
+
+    add("  @Override\n" +
+        "  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(UserDefined<T, U> pred) {\n");
+    addUdpBegin();
+    for (TypeInfo info : TYPES) {
+      addUdpCase(info, false);
+    }
+    addVisitEnd();
+
+    add("  @Override\n" +
+        "  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(LogicalNotUserDefined<T, U> notPred) {\n" +
+        "    UserDefined<T, U> pred = notPred.getUserDefined();\n");
+    addUdpBegin();
+    for (TypeInfo info : TYPES) {
+      addUdpCase(info, true);
+    }
+    addVisitEnd();
+
+    add("}\n");
+    writer.close();
+  }
+
+  private void addVisitBegin(String inVar) throws IOException {
+    add("  @Override\n" +
+        "  public <T extends Comparable<T>> IncrementallyUpdatedFilterPredicate visit(" + inVar + "<T> pred) {\n" +
+        "    ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
+        "    Class<T> clazz = pred.getColumn().getColumnType();\n" +
+        "\n" +
+        "    ValueInspector valueInspector = null;\n\n");
+  }
+
+  private void addVisitEnd() throws IOException {
+    add("    if (valueInspector == null) {\n" +
+        "      throw new IllegalArgumentException(\"Encountered unknown type \" + clazz);\n" +
+        "    }\n" +
+        "\n" +
+        "    addValueInspector(columnPath, valueInspector);\n" +
+        "    return valueInspector;\n" +
+        "  }\n\n");
+  }
+
+  private void addEqNotEqCase(TypeInfo info, boolean isEq) throws IOException {
+    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
+        "      if (pred.getValue() == null) {\n" +
+        "        valueInspector = new ValueInspector() {\n" +
+        "          @Override\n" +
+        "          public void updateNull() {\n" +
+        "            setResult(" + isEq + ");\n" +
+        "          }\n" +
+        "\n" +
+        "          @Override\n" +
+        "          public void update(" + info.primitiveName + " value) {\n" +
+        "            setResult(" + !isEq + ");\n" +
+        "          }\n" +
+        "        };\n" +
+        "      } else {\n" +
+        "        final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
+        "\n" +
+        "        valueInspector = new ValueInspector() {\n" +
+        "          @Override\n" +
+        "          public void updateNull() {\n" +
+        "            setResult(" + !isEq +");\n" +
+        "          }\n" +
+        "\n" +
+        "          @Override\n" +
+        "          public void update(" + info.primitiveName + " value) {\n");
+
+    if (info.useComparable) {
+      add("            setResult(" + compareEquality("value", "target", isEq) + ");\n");
+    } else {
+      add("            setResult(" + (isEq ? "value == target" : "value != target" )  + ");\n");
+    }
+
+    add("          }\n" +
+        "        };\n" +
+        "      }\n" +
+        "    }\n\n");
+  }
+
+  private void addInequalityCase(TypeInfo info, String op) throws IOException {
+    if (!info.supportsInequality) {
+      add("    if (clazz.equals(" + info.className + ".class)) {\n");
+      add("      throw new IllegalArgumentException(\"Operator " + op + " not supported for " + info.className + "\");\n");
+      add("    }\n\n");
+      return;
+    }
+
+    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
+        "      final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
+        "\n" +
+        "      valueInspector = new ValueInspector() {\n" +
+        "        @Override\n" +
+        "        public void updateNull() {\n" +
+        "          setResult(false);\n" +
+        "        }\n" +
+        "\n" +
+        "        @Override\n" +
+        "        public void update(" + info.primitiveName + " value) {\n");
+
+    if (info.useComparable) {
+      add("          setResult(value.compareTo(target) " + op + " 0);\n");
+    } else {
+      add("          setResult(value " + op + " target);\n");
+    }
+    add("        }\n" +
+        "      };\n" +
+        "    }\n\n");
+  }
+
+  private void addUdpBegin() throws IOException {
+    add("    ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
+        "    Class<T> clazz = pred.getColumn().getColumnType();\n" +
+        "\n" +
+        "    ValueInspector valueInspector = null;\n" +
+        "\n" +
+        "    final U udp = pred.getUserDefinedPredicate();\n" +
+        "\n");
+  }
+
+  private void addUdpCase(TypeInfo info, boolean invert)throws IOException {
+    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
+        "      valueInspector = new ValueInspector() {\n" +
+        "        @Override\n" +
+        "        public void updateNull() {\n" +
+        "          setResult(" + (invert ? "!" : "") + "udp.keep(null));\n" +
+        "        }\n" +
+        "\n" +
+        "        @SuppressWarnings(\"unchecked\")\n" +
+        "        @Override\n" +
+        "        public void update(" + info.primitiveName + " value) {\n" +
+        "          setResult(" + (invert ? "!" : "") + "udp.keep((T) (Object) value));\n" +
+        "        }\n" +
+        "      };\n" +
+        "    }\n\n");
+  }
+
+  private String compareEquality(String var, String target, boolean eq) {
+    return var + ".compareTo(" + target + ")" + (eq ? " == 0 " : " != 0");
+  }
+
+  private void add(String s) throws IOException {
+    writer.write(s);
+  }
+}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/Generator.java b/parquet-generator/src/main/java/parquet/encoding/Generator.java
deleted file mode 100644
index 148d838..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/Generator.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding;
-
-import parquet.encoding.bitpacking.ByteBasedBitPackingGenerator;
-import parquet.encoding.bitpacking.IntBasedBitPackingGenerator;
-
-/**
- * main class for code generation hook in build for encodings generation
- *
- * @author Julien Le Dem
- *
- */
-public class Generator {
-
-  public static void main(String[] args) throws Exception {
-    IntBasedBitPackingGenerator.main(args);
-    ByteBasedBitPackingGenerator.main(args);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java b/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
deleted file mode 100644
index 5bcf4b0..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/bitpacking/ByteBasedBitPackingGenerator.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding.bitpacking;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-import parquet.bytes.BytesUtils;
-
-/**
- *
- * This class generates bit packers that pack the most significant bit first.
- * The result of the generation is checked in. To regenerate the code run this class and check in the result.
- *
- * TODO: remove the unnecessary masks for perf
- *
- * @author Julien Le Dem
- *
- */
-public class ByteBasedBitPackingGenerator {
-
-  private static final String CLASS_NAME_PREFIX = "ByteBitPacking";
-  private static final int PACKER_COUNT = 32;
-
-  public static void main(String[] args) throws Exception {
-    String basePath = args[0];
-    generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
-    generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
-  }
-
-  private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
-    final File file = new File(basePath + "/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
-    if (!file.getParentFile().exists()) {
-      file.getParentFile().mkdirs();
-    }
-    FileWriter fw = new FileWriter(file);
-    fw.append("package parquet.column.values.bitpacking;\n");
-    fw.append("\n");
-    fw.append("/**\n");
-    if (msbFirst) {
-      fw.append(" * Packs from the Most Significant Bit first\n");
-    } else {
-      fw.append(" * Packs from the Least Significant Bit first\n");
-    }
-    fw.append(" * \n");
-    fw.append(" * @author automatically generated\n");
-    fw.append(" * @see ByteBasedBitPackingGenerator\n");
-    fw.append(" *\n");
-    fw.append(" */\n");
-    fw.append("public abstract class " + className + " {\n");
-    fw.append("\n");
-    fw.append("  private static final BytePacker[] packers = new BytePacker[33];\n");
-    fw.append("  static {\n");
-    for (int i = 0; i <= PACKER_COUNT; i++) {
-      fw.append("    packers[" + i + "] = new Packer" + i + "();\n");
-    }
-    fw.append("  }\n");
-    fw.append("\n");
-    fw.append("  public static final BytePackerFactory factory = new BytePackerFactory() {\n");
-    fw.append("    public BytePacker newBytePacker(int bitWidth) {\n");
-    fw.append("      return packers[bitWidth];\n");
-    fw.append("    }\n");
-    fw.append("  };\n");
-    fw.append("\n");
-    for (int i = 0; i <= PACKER_COUNT; i++) {
-      generateClass(fw, i, msbFirst);
-      fw.append("\n");
-    }
-    fw.append("}\n");
-    fw.close();
-  }
-
-  private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
-    fw.append("  private static final class Packer" + bitWidth + " extends BytePacker {\n");
-    fw.append("\n");
-    fw.append("    private Packer" + bitWidth + "() {\n");
-    fw.append("      super("+bitWidth+");\n");
-    fw.append("    }\n");
-    fw.append("\n");
-    // Packing
-    generatePack(fw, bitWidth, 1, msbFirst);
-    generatePack(fw, bitWidth, 4, msbFirst);
-
-    // Unpacking
-    generateUnpack(fw, bitWidth, 1, msbFirst);
-    generateUnpack(fw, bitWidth, 4, msbFirst);
-
-    fw.append("  }\n");
-  }
-
-  private static int getShift(FileWriter fw, int bitWidth, boolean msbFirst,
-      int byteIndex, int valueIndex) throws IOException {
-    // relative positions of the start and end of the value to the start and end of the byte
-    int valueStartBitIndex = (valueIndex * bitWidth) - (8 * (byteIndex));
-    int valueEndBitIndex = ((valueIndex + 1) * bitWidth) - (8 * (byteIndex + 1));
-
-    // boundaries of the current value that we want
-    int valueStartBitWanted;
-    int valueEndBitWanted;
-    // boundaries of the current byte that will receive them
-    int byteStartBitWanted;
-    int byteEndBitWanted;
-
-    int shift;
-
-    if (msbFirst) {
-      valueStartBitWanted = valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1;
-      valueEndBitWanted = valueEndBitIndex > 0 ? valueEndBitIndex : 0;
-      byteStartBitWanted = valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex;
-      byteEndBitWanted = valueEndBitIndex > 0 ? 0 : -valueEndBitIndex;
-      shift = valueEndBitWanted - byteEndBitWanted;
-    } else {
-      valueStartBitWanted = bitWidth - 1 - (valueEndBitIndex > 0 ? valueEndBitIndex : 0);
-      valueEndBitWanted = bitWidth - 1 - (valueStartBitIndex < 0 ? bitWidth - 1 + valueStartBitIndex : bitWidth - 1);
-      byteStartBitWanted = 7 - (valueEndBitIndex > 0 ? 0 : -valueEndBitIndex);
-      byteEndBitWanted = 7 - (valueStartBitIndex < 0 ? 8 : 7 - valueStartBitIndex);
-      shift = valueStartBitWanted - byteStartBitWanted;
-    }
-
-    visualizeAlignment(
-        fw, bitWidth, valueEndBitIndex,
-        valueStartBitWanted, valueEndBitWanted,
-        byteStartBitWanted, byteEndBitWanted,
-        shift
-        );
-    return shift;
-  }
-
-  private static void visualizeAlignment(FileWriter fw, int bitWidth,
-      int valueEndBitIndex, int valueStartBitWanted, int valueEndBitWanted,
-      int byteStartBitWanted, int byteEndBitWanted, int shift) throws IOException {
-    // ASCII art to visualize what is happening
-    fw.append("//");
-    int buf = 2 + Math.max(0, bitWidth + 8);
-    for (int i = 0; i < buf; i++) {
-      fw.append(" ");
-    }
-    fw.append("[");
-    for (int i = 7; i >= 0; i--) {
-      if (i<=byteStartBitWanted && i>=byteEndBitWanted) {
-        fw.append(String.valueOf(i));
-      } else {
-        fw.append("_");
-      }
-    }
-    fw.append("]\n          //");
-    for (int i = 0; i < buf + (8 - bitWidth + shift); i++) {
-      fw.append(" ");
-    }
-    fw.append("[");
-    for (int i = bitWidth - 1; i >= 0 ; i--) {
-      if (i<=valueStartBitWanted && i>=valueEndBitWanted) {
-        fw.append(String.valueOf(i % 10));
-      } else {
-        fw.append("_");
-      }
-    }
-    fw.append("]\n");
-    fw.append("           ");
-  }
-
-  private static void generatePack(FileWriter fw, int bitWidth, int batch, boolean msbFirst) throws IOException {
-    int mask = genMask(bitWidth);
-    fw.append("    public final void pack" + (batch * 8) + "Values(final int[] in, final int inPos, final byte[] out, final int outPos) {\n");
-    for (int byteIndex = 0; byteIndex < bitWidth * batch; ++byteIndex) {
-      fw.append("      out[" + align(byteIndex, 2) + " + outPos] = (byte)((\n");
-      int startIndex = (byteIndex * 8) / bitWidth;
-      int endIndex = ((byteIndex + 1) * 8 + bitWidth - 1) / bitWidth;
-      for (int valueIndex = startIndex; valueIndex < endIndex; valueIndex++) {
-
-        if (valueIndex == startIndex) {
-          fw.append("          ");
-        } else {
-          fw.append("\n        | ");
-        }
-        int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
-
-        String shiftString = ""; // used when shift == 0
-        if (shift > 0) {
-          shiftString = " >>> " + shift;
-        } else if (shift < 0) {
-          shiftString = " <<  " + ( - shift);
-        }
-        fw.append("((in[" + align(valueIndex, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
-      }
-      fw.append(") & 255);\n");
-    }
-    fw.append("    }\n");
-  }
-
-  private static void generateUnpack(FileWriter fw, int bitWidth, int batch, boolean msbFirst)
-      throws IOException {
-    fw.append("    public final void unpack" + (batch * 8) + "Values(final byte[] in, final int inPos, final int[] out, final int outPos) {\n");
-    if (bitWidth > 0) {
-      int mask = genMask(bitWidth);
-      for (int valueIndex = 0; valueIndex < (batch * 8); ++valueIndex) {
-        fw.append("      out[" + align(valueIndex, 2) + " + outPos] =\n");
-
-        int startIndex = valueIndex * bitWidth / 8;
-        int endIndex = BytesUtils.paddedByteCountFromBits((valueIndex + 1) * bitWidth);
-
-        for (int byteIndex = startIndex; byteIndex < endIndex; byteIndex++) {
-          if (byteIndex == startIndex) {
-            fw.append("          ");
-          } else {
-            fw.append("\n        | ");
-          }
-          int shift = getShift(fw, bitWidth, msbFirst, byteIndex, valueIndex);
-
-          String shiftString = ""; // when shift == 0
-          if (shift < 0) {
-            shiftString = ">>>  " + (-shift);
-          } else if (shift > 0){
-            shiftString = "<<  " + shift;
-          }
-          fw.append(" (((((int)in[" + align(byteIndex, 2) + " + inPos]) & 255) " + shiftString + ") & " + mask + ")");
-        }
-        fw.append(";\n");
-      }
-    }
-    fw.append("    }\n");
-  }
-
-  private static int genMask(int bitWidth) {
-    int mask = 0;
-    for (int i = 0; i < bitWidth; i++) {
-      mask <<= 1;
-      mask |= 1;
-    }
-    return mask;
-  }
-
-  private static String align(int value, int digits) {
-    final String valueString = String.valueOf(value);
-    StringBuilder result = new StringBuilder();
-    for (int i = valueString.length(); i < digits; i++) {
-      result.append(" ");
-    }
-    result.append(valueString);
-    return result.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java b/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
deleted file mode 100644
index 153102a..0000000
--- a/parquet-generator/src/main/java/parquet/encoding/bitpacking/IntBasedBitPackingGenerator.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.encoding.bitpacking;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-/**
- * Scheme designed by D. Lemire
- *
- * This is a re-implementation of The scheme released under Apache License Version 2.0
- * at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java
- *
- * It generate two classes:
- * - LemireBitPackingLE, the original scheme, filling the LSB first
- * - LemireBitPackingBE, the scheme modified to fill the MSB first (and match our existing bit packing)
- *
- * The result of the generation is checked in. To regenerate the code run this class and check in the result.
- *
- * The generated classes pack the values into arrays of ints (as opposed to arrays of bytes) based on a given bit width.
- *
- * Note: This is not really used for now as the hadoop API does not really let write int[]. We need to revisit this
- *
- * @author Julien Le Dem
- *
- */
-public class IntBasedBitPackingGenerator {
-
-  private static final String CLASS_NAME_PREFIX = "LemireBitPacking";
-
-  public static void main(String[] args) throws Exception {
-    String basePath = args[0];
-    generateScheme(CLASS_NAME_PREFIX + "BE", true, basePath);
-    generateScheme(CLASS_NAME_PREFIX + "LE", false, basePath);
-  }
-
-  private static void generateScheme(String className, boolean msbFirst, String basePath) throws IOException {
-    final File file = new File(basePath + "/parquet/column/values/bitpacking/" + className + ".java").getAbsoluteFile();
-    if (!file.getParentFile().exists()) {
-      file.getParentFile().mkdirs();
-    }
-    FileWriter fw = new FileWriter(file);
-    fw.append("package parquet.column.values.bitpacking;\n");
-    fw.append("\n");
-    fw.append("/**\n");
-    fw.append(" * Based on the original implementation at at https://github.com/lemire/JavaFastPFOR/blob/master/src/integercompression/BitPacking.java\n");
-    fw.append(" * Which is released under the\n");
-    fw.append(" * Apache License Version 2.0 http://www.apache.org/licenses/.\n");
-    fw.append(" * By Daniel Lemire, http://lemire.me/en/\n");
-    fw.append(" * \n");
-    fw.append(" * Scheme designed by D. Lemire\n");
-    if (msbFirst) {
-      fw.append(" * Adapted to pack from the Most Significant Bit first\n");
-    }
-    fw.append(" * \n");
-    fw.append(" * @author automatically generated\n");
-    fw.append(" * @see IntBasedBitPackingGenerator\n");
-    fw.append(" *\n");
-    fw.append(" */\n");
-    fw.append("abstract class " + className + " {\n");
-    fw.append("\n");
-    fw.append("  private static final IntPacker[] packers = new IntPacker[32];\n");
-    fw.append("  static {\n");
-    for (int i = 0; i < 32; i++) {
-      fw.append("    packers[" + i + "] = new Packer" + i + "();\n");
-    }
-    fw.append("  }\n");
-    fw.append("\n");
-    fw.append("  public static final IntPackerFactory factory = new IntPackerFactory() {\n");
-    fw.append("    public IntPacker newIntPacker(int bitWidth) {\n");
-    fw.append("      return packers[bitWidth];\n");
-    fw.append("    }\n");
-    fw.append("  };\n");
-    fw.append("\n");
-    for (int i = 0; i < 32; i++) {
-      generateClass(fw, i, msbFirst);
-      fw.append("\n");
-    }
-    fw.append("}\n");
-    fw.close();
-  }
-
-  private static void generateClass(FileWriter fw, int bitWidth, boolean msbFirst) throws IOException {
-    int mask = 0;
-    for (int i = 0; i < bitWidth; i++) {
-      mask <<= 1;
-      mask |= 1;
-    }
-    fw.append("  private static final class Packer" + bitWidth + " extends IntPacker {\n");
-    fw.append("\n");
-    fw.append("    private Packer" + bitWidth + "() {\n");
-    fw.append("      super(" + bitWidth + ");\n");
-    fw.append("    }\n");
-    fw.append("\n");
-    // Packing
-    fw.append("    public final void pack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
-    for (int i = 0; i < bitWidth; ++i) {
-      fw.append("      out[" + align(i, 2) + " + outPos] =\n");
-      int startIndex = (i * 32) / bitWidth;
-      int endIndex = ((i + 1) * 32 + bitWidth - 1) / bitWidth;
-      for (int j = startIndex; j < endIndex; j++) {
-        if (j == startIndex) {
-          fw.append("          ");
-        } else {
-          fw.append("\n        | ");
-        }
-        String shiftString = getPackShiftString(bitWidth, i, startIndex, j, msbFirst);
-        fw.append("((in[" + align(j, 2) + " + inPos] & " + mask + ")" + shiftString + ")");
-      }
-      fw.append(";\n");
-    }
-    fw.append("    }\n");
-
-    // Unpacking
-    fw.append("    public final void unpack32Values(final int[] in, final int inPos, final int[] out, final int outPos) {\n");
-    if (bitWidth > 0) {
-      for (int i = 0; i < 32; ++i) {
-        fw.append("      out[" + align(i, 2) + " + outPos] =");
-        int byteIndex = i * bitWidth / 32;
-        String shiftString = getUnpackShiftString(bitWidth, i, msbFirst);
-        fw.append(" ((in[" + align(byteIndex, 2) + " + inPos] " + shiftString + ") & " + mask + ")");
-        if (((i + 1) * bitWidth - 1 ) / 32 != byteIndex) {
-          // reading the end of the value from next int
-          int bitsRead = ((i + 1) * bitWidth - 1) % 32 + 1;
-          fw.append(" | ((in[" + align(byteIndex + 1, 2) + " + inPos]");
-          if (msbFirst) {
-            fw.append(") >>> " + align(32 - bitsRead, 2) + ")");
-          } else {
-            int lowerMask = 0;
-            for (int j = 0; j < bitsRead; j++) {
-              lowerMask <<= 1;
-              lowerMask |= 1;
-            }
-            fw.append(" & " + lowerMask + ") << " + align(bitWidth - bitsRead, 2) + ")");
-          }
-        }
-        fw.append(";\n");
-      }
-    }
-    fw.append("    }\n");
-    fw.append("  }\n");
-  }
-
-  private static String getUnpackShiftString(int bitWidth, int i, boolean msbFirst) {
-    final int regularShift = i * bitWidth % 32;
-    String shiftString;
-    if (msbFirst) {
-      int shift = 32 - (regularShift + bitWidth);
-      if (shift < 0) {
-        shiftString = "<<  " + align(-shift, 2);
-      } else {
-        shiftString = ">>> " + align(shift, 2);
-      }
-    } else {
-      shiftString = ">>> " + align(regularShift, 2);
-    }
-    return shiftString;
-  }
-
-  private static String getPackShiftString(int bitWidth, int integerIndex, int startIndex, int valueIndex, boolean msbFirst) {
-    String shiftString;
-    int regularShift = (valueIndex * bitWidth) % 32;
-    if (msbFirst) { // filling most significant bit first
-      int shift = 32 - (regularShift + bitWidth);
-      if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
-        // end of last value from previous int
-          shiftString = " <<  " + align(32 - (((valueIndex + 1) * bitWidth) % 32), 2);
-      } else if (shift < 0) {
-        // partial last value
-          shiftString = " >>> " + align(-shift, 2);
-      } else {
-        shiftString = " <<  " + align(shift, 2);
-      }
-    } else { // filling least significant bit first
-      if (valueIndex == startIndex && (integerIndex * 32) % bitWidth != 0) {
-        // end of last value from previous int
-        shiftString = " >>> " + align(32 - regularShift, 2);
-      } else {
-        shiftString = " <<  " + align(regularShift, 2);
-      }
-    }
-    return shiftString;
-  }
-
-  private static String align(int value, int digits) {
-    final String valueString = String.valueOf(value);
-    StringBuilder result = new StringBuilder();
-    for (int i = valueString.length(); i < digits; i++) {
-      result.append(" ");
-    }
-    result.append(valueString);
-    return result.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/filter2/Generator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/filter2/Generator.java b/parquet-generator/src/main/java/parquet/filter2/Generator.java
deleted file mode 100644
index 9cc38bf..0000000
--- a/parquet-generator/src/main/java/parquet/filter2/Generator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2;
-
-/**
- * main class for code generation hook in build for filter2 generation
- */
-public class Generator {
-  public static void main(String[] args) throws Exception {
-    IncrementallyUpdatedFilterPredicateGenerator.main(args);
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
----------------------------------------------------------------------
diff --git a/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java b/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
deleted file mode 100644
index 9a705de..0000000
--- a/parquet-generator/src/main/java/parquet/filter2/IncrementallyUpdatedFilterPredicateGenerator.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package parquet.filter2;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-public class IncrementallyUpdatedFilterPredicateGenerator {
-
-  public static void main(String[] args) throws IOException {
-    File srcFile = new File(args[0] + "/parquet/filter2/recordlevel/IncrementallyUpdatedFilterPredicateBuilder.java");
-    srcFile = srcFile.getAbsoluteFile();
-    File parent = srcFile.getParentFile();
-    if (!parent.exists()) {
-      if (!parent.mkdirs()) {
-        throw new IOException("Couldn't mkdirs for " + parent);
-      }
-    }
-    new IncrementallyUpdatedFilterPredicateGenerator(srcFile).run();
-  }
-
-  private final FileWriter writer;
-
-  public IncrementallyUpdatedFilterPredicateGenerator(File file) throws IOException {
-    this.writer = new FileWriter(file);
-  }
-
-  private static class TypeInfo {
-    public final String className;
-    public final String primitiveName;
-    public final boolean useComparable;
-    public final boolean supportsInequality;
-
-    private TypeInfo(String className, String primitiveName, boolean useComparable, boolean supportsInequality) {
-      this.className = className;
-      this.primitiveName = primitiveName;
-      this.useComparable = useComparable;
-      this.supportsInequality = supportsInequality;
-    }
-  }
-
-  private static final TypeInfo[] TYPES = new TypeInfo[]{
-    new TypeInfo("Integer", "int", false, true),
-    new TypeInfo("Long", "long", false, true),
-    new TypeInfo("Boolean", "boolean", false, false),
-    new TypeInfo("Float", "float", false, true),
-    new TypeInfo("Double", "double", false, true),
-    new TypeInfo("Binary", "Binary", true, true),
-  };
-
-  public void run() throws IOException {
-    add("package parquet.filter2.recordlevel;\n" +
-        "\n" +
-        "import parquet.hadoop.metadata.ColumnPath;\n" +
-        "import parquet.filter2.predicate.Operators.Eq;\n" +
-        "import parquet.filter2.predicate.Operators.Gt;\n" +
-        "import parquet.filter2.predicate.Operators.GtEq;\n" +
-        "import parquet.filter2.predicate.Operators.LogicalNotUserDefined;\n" +
-        "import parquet.filter2.predicate.Operators.Lt;\n" +
-        "import parquet.filter2.predicate.Operators.LtEq;\n" +
-        "import parquet.filter2.predicate.Operators.NotEq;\n" +
-        "import parquet.filter2.predicate.Operators.UserDefined;\n" +
-        "import parquet.filter2.predicate.UserDefinedPredicate;\n" +
-        "import parquet.filter2.recordlevel.IncrementallyUpdatedFilterPredicate.ValueInspector;\n" +
-        "import parquet.io.api.Binary;\n\n" +
-        "/**\n" +
-        " * This class is auto-generated by {@link parquet.filter2.IncrementallyUpdatedFilterPredicateGenerator}\n" +
-        " * Do not manually edit!\n" +
-        " * See {@link IncrementallyUpdatedFilterPredicateBuilderBase}\n" +
-        " */\n");
-
-    add("public class IncrementallyUpdatedFilterPredicateBuilder extends IncrementallyUpdatedFilterPredicateBuilderBase {\n\n");
-
-    addVisitBegin("Eq");
-    for (TypeInfo info : TYPES) {
-      addEqNotEqCase(info, true);
-    }
-    addVisitEnd();
-
-    addVisitBegin("NotEq");
-    for (TypeInfo info : TYPES) {
-      addEqNotEqCase(info, false);
-    }
-    addVisitEnd();
-
-    addVisitBegin("Lt");
-    for (TypeInfo info : TYPES) {
-      addInequalityCase(info, "<");
-    }
-    addVisitEnd();
-
-    addVisitBegin("LtEq");
-    for (TypeInfo info : TYPES) {
-      addInequalityCase(info, "<=");
-    }
-    addVisitEnd();
-
-    addVisitBegin("Gt");
-    for (TypeInfo info : TYPES) {
-      addInequalityCase(info, ">");
-    }
-    addVisitEnd();
-
-    addVisitBegin("GtEq");
-    for (TypeInfo info : TYPES) {
-      addInequalityCase(info, ">=");
-    }
-    addVisitEnd();
-
-    add("  @Override\n" +
-        "  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(UserDefined<T, U> pred) {\n");
-    addUdpBegin();
-    for (TypeInfo info : TYPES) {
-      addUdpCase(info, false);
-    }
-    addVisitEnd();
-
-    add("  @Override\n" +
-        "  public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> IncrementallyUpdatedFilterPredicate visit(LogicalNotUserDefined<T, U> notPred) {\n" +
-        "    UserDefined<T, U> pred = notPred.getUserDefined();\n");
-    addUdpBegin();
-    for (TypeInfo info : TYPES) {
-      addUdpCase(info, true);
-    }
-    addVisitEnd();
-
-    add("}\n");
-    writer.close();
-  }
-
-  private void addVisitBegin(String inVar) throws IOException {
-    add("  @Override\n" +
-        "  public <T extends Comparable<T>> IncrementallyUpdatedFilterPredicate visit(" + inVar + "<T> pred) {\n" +
-        "    ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
-        "    Class<T> clazz = pred.getColumn().getColumnType();\n" +
-        "\n" +
-        "    ValueInspector valueInspector = null;\n\n");
-  }
-
-  private void addVisitEnd() throws IOException {
-    add("    if (valueInspector == null) {\n" +
-        "      throw new IllegalArgumentException(\"Encountered unknown type \" + clazz);\n" +
-        "    }\n" +
-        "\n" +
-        "    addValueInspector(columnPath, valueInspector);\n" +
-        "    return valueInspector;\n" +
-        "  }\n\n");
-  }
-
-  private void addEqNotEqCase(TypeInfo info, boolean isEq) throws IOException {
-    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
-        "      if (pred.getValue() == null) {\n" +
-        "        valueInspector = new ValueInspector() {\n" +
-        "          @Override\n" +
-        "          public void updateNull() {\n" +
-        "            setResult(" + isEq + ");\n" +
-        "          }\n" +
-        "\n" +
-        "          @Override\n" +
-        "          public void update(" + info.primitiveName + " value) {\n" +
-        "            setResult(" + !isEq + ");\n" +
-        "          }\n" +
-        "        };\n" +
-        "      } else {\n" +
-        "        final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
-        "\n" +
-        "        valueInspector = new ValueInspector() {\n" +
-        "          @Override\n" +
-        "          public void updateNull() {\n" +
-        "            setResult(" + !isEq +");\n" +
-        "          }\n" +
-        "\n" +
-        "          @Override\n" +
-        "          public void update(" + info.primitiveName + " value) {\n");
-
-    if (info.useComparable) {
-      add("            setResult(" + compareEquality("value", "target", isEq) + ");\n");
-    } else {
-      add("            setResult(" + (isEq ? "value == target" : "value != target" )  + ");\n");
-    }
-
-    add("          }\n" +
-        "        };\n" +
-        "      }\n" +
-        "    }\n\n");
-  }
-
-  private void addInequalityCase(TypeInfo info, String op) throws IOException {
-    if (!info.supportsInequality) {
-      add("    if (clazz.equals(" + info.className + ".class)) {\n");
-      add("      throw new IllegalArgumentException(\"Operator " + op + " not supported for " + info.className + "\");\n");
-      add("    }\n\n");
-      return;
-    }
-
-    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
-        "      final " + info.primitiveName + " target = (" + info.className + ") (Object) pred.getValue();\n" +
-        "\n" +
-        "      valueInspector = new ValueInspector() {\n" +
-        "        @Override\n" +
-        "        public void updateNull() {\n" +
-        "          setResult(false);\n" +
-        "        }\n" +
-        "\n" +
-        "        @Override\n" +
-        "        public void update(" + info.primitiveName + " value) {\n");
-
-    if (info.useComparable) {
-      add("          setResult(value.compareTo(target) " + op + " 0);\n");
-    } else {
-      add("          setResult(value " + op + " target);\n");
-    }
-    add("        }\n" +
-        "      };\n" +
-        "    }\n\n");
-  }
-
-  private void addUdpBegin() throws IOException {
-    add("    ColumnPath columnPath = pred.getColumn().getColumnPath();\n" +
-        "    Class<T> clazz = pred.getColumn().getColumnType();\n" +
-        "\n" +
-        "    ValueInspector valueInspector = null;\n" +
-        "\n" +
-        "    final U udp = pred.getUserDefinedPredicate();\n" +
-        "\n");
-  }
-
-  private void addUdpCase(TypeInfo info, boolean invert)throws IOException {
-    add("    if (clazz.equals(" + info.className + ".class)) {\n" +
-        "      valueInspector = new ValueInspector() {\n" +
-        "        @Override\n" +
-        "        public void updateNull() {\n" +
-        "          setResult(" + (invert ? "!" : "") + "udp.keep(null));\n" +
-        "        }\n" +
-        "\n" +
-        "        @SuppressWarnings(\"unchecked\")\n" +
-        "        @Override\n" +
-        "        public void update(" + info.primitiveName + " value) {\n" +
-        "          setResult(" + (invert ? "!" : "") + "udp.keep((T) (Object) value));\n" +
-        "        }\n" +
-        "      };\n" +
-        "    }\n\n");
-  }
-
-  private String compareEquality(String var, String target, boolean eq) {
-    return var + ".compareTo(" + target + ")" + (eq ? " == 0 " : " != 0");
-  }
-
-  private void add(String s) throws IOException {
-    writer.write(s);
-  }
-}

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
index 5ae94d4..a9f4620 100644
--- a/parquet-hadoop-bundle/pom.xml
+++ b/parquet-hadoop-bundle/pom.xml
@@ -18,7 +18,7 @@
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
-    <groupId>com.twitter</groupId>
+    <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
     <version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,17 +37,17 @@
 
   <dependencies>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-common</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-column</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-hadoop</artifactId>
       <version>${project.version}</version>
     </dependency>
@@ -77,7 +77,7 @@
               </filters>
               <artifactSet>
                 <includes>
-                  <include>com.twitter:parquet-*</include>
+                  <include>org.apache.parquet:parquet-*</include>
                 </includes>
               </artifactSet>
             </configuration>

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle b/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
new file mode 100644
index 0000000..fe95886
--- /dev/null
+++ b/parquet-hadoop-bundle/src/main/resources/org/apache/parquet/bundle
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop-bundle/src/main/resources/parquet/bundle
----------------------------------------------------------------------
diff --git a/parquet-hadoop-bundle/src/main/resources/parquet/bundle b/parquet-hadoop-bundle/src/main/resources/parquet/bundle
deleted file mode 100644
index fe95886..0000000
--- a/parquet-hadoop-bundle/src/main/resources/parquet/bundle
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/b10870e4/parquet-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
index 970864f..f510401 100644
--- a/parquet-hadoop/pom.xml
+++ b/parquet-hadoop/pom.xml
@@ -18,7 +18,7 @@
   -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
-    <groupId>com.twitter</groupId>
+    <groupId>org.apache.parquet</groupId>
     <artifactId>parquet</artifactId>
     <relativePath>../pom.xml</relativePath>
     <version>1.7.0-incubating-SNAPSHOT</version>
@@ -37,12 +37,12 @@
 
   <dependencies>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-column</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-format</artifactId>
       <version>${parquet.format.version}</version>
     </dependency>
@@ -59,7 +59,7 @@
       <scope>provided</scope>
     </dependency>
     <dependency>
-      <groupId>com.twitter</groupId>
+      <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-jackson</artifactId>
       <version>${project.version}</version>
     </dependency>


Mime
View raw message