arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [1/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead of MaterializedField
Date Thu, 18 Aug 2016 23:30:34 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 246a126b2 -> e7e399db5


http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index a3763cd..4eb0d9f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -26,140 +26,139 @@ import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
-import org.apache.arrow.vector.holders.Decimal38SparseHolder;
-
-public class DecimalUtility extends CoreDecimalUtility{
-
-    public final static int MAX_DIGITS = 9;
-    public final static int DIGITS_BASE = 1000000000;
-    public final static int DIGITS_MAX = 999999999;
-    public final static int INTEGER_SIZE = (Integer.SIZE/8);
-
-    public final static String[] decimalToString = {"",
-            "0",
-            "00",
-            "000",
-            "0000",
-            "00000",
-            "000000",
-            "0000000",
-            "00000000",
-            "000000000"};
-
-    public final static long[] scale_long_constants = {
-        1,
-        10,
-        100,
-        1000,
-        10000,
-        100000,
-        1000000,
-        10000000,
-        100000000,
-        1000000000,
-        10000000000l,
-        100000000000l,
-        1000000000000l,
-        10000000000000l,
-        100000000000000l,
-        1000000000000000l,
-        10000000000000000l,
-        100000000000000000l,
-        1000000000000000000l};
-
-    /*
-     * Simple function that returns the static precomputed
-     * power of ten, instead of using Math.pow
-     */
-    public static long getPowerOfTen(int power) {
-      assert power >= 0 && power < scale_long_constants.length;
-      return scale_long_constants[(power)];
-    }
-
-    /*
-     * Math.pow returns a double and while multiplying with large digits
-     * in the decimal data type we encounter noise. So instead of multiplying
-     * with Math.pow we use the static constants to perform the multiplication
-     */
-    public static long adjustScaleMultiply(long input, int factor) {
-      int index = Math.abs(factor);
-      assert index >= 0 && index < scale_long_constants.length;
-      if (factor >= 0) {
-        return input * scale_long_constants[index];
-      } else {
-        return input / scale_long_constants[index];
-      }
-    }
 
-    public static long adjustScaleDivide(long input, int factor) {
-      int index = Math.abs(factor);
-      assert index >= 0 && index < scale_long_constants.length;
-      if (factor >= 0) {
-        return input / scale_long_constants[index];
-      } else {
-        return input * scale_long_constants[index];
-      }
+public class DecimalUtility {
+
+  public final static int MAX_DIGITS = 9;
+  public final static int DIGITS_BASE = 1000000000;
+  public final static int DIGITS_MAX = 999999999;
+  public final static int INTEGER_SIZE = (Integer.SIZE/8);
+
+  public final static String[] decimalToString = {"",
+          "0",
+          "00",
+          "000",
+          "0000",
+          "00000",
+          "000000",
+          "0000000",
+          "00000000",
+          "000000000"};
+
+  public final static long[] scale_long_constants = {
+          1,
+          10,
+          100,
+          1000,
+          10000,
+          100000,
+          1000000,
+          10000000,
+          100000000,
+          1000000000,
+          10000000000l,
+          100000000000l,
+          1000000000000l,
+          10000000000000l,
+          100000000000000l,
+          1000000000000000l,
+          10000000000000000l,
+          100000000000000000l,
+          1000000000000000000l};
+
+  /*
+   * Simple function that returns the static precomputed
+   * power of ten, instead of using Math.pow
+   */
+  public static long getPowerOfTen(int power) {
+    assert power >= 0 && power < scale_long_constants.length;
+    return scale_long_constants[(power)];
+  }
+
+  /*
+   * Math.pow returns a double and while multiplying with large digits
+   * in the decimal data type we encounter noise. So instead of multiplying
+   * with Math.pow we use the static constants to perform the multiplication
+   */
+  public static long adjustScaleMultiply(long input, int factor) {
+    int index = Math.abs(factor);
+    assert index >= 0 && index < scale_long_constants.length;
+    if (factor >= 0) {
+      return input * scale_long_constants[index];
+    } else {
+      return input / scale_long_constants[index];
     }
+  }
 
-    /* Given the number of actual digits this function returns the
-     * number of indexes it will occupy in the array of integers
-     * which are stored in base 1 billion
-     */
-    public static int roundUp(int ndigits) {
-        return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+  public static long adjustScaleDivide(long input, int factor) {
+    int index = Math.abs(factor);
+    assert index >= 0 && index < scale_long_constants.length;
+    if (factor >= 0) {
+      return input / scale_long_constants[index];
+    } else {
+      return input * scale_long_constants[index];
     }
+  }
 
-    /* Returns a string representation of the given integer
-     * If the length of the given integer is less than the
-     * passed length, this function will prepend zeroes to the string
-     */
-    public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
-        String value = ((Integer) number).toString();
-        int length = value.length();
+  /* Given the number of actual digits this function returns the
+   * number of indexes it will occupy in the array of integers
+   * which are stored in base 1 billion
+   */
+  public static int roundUp(int ndigits) {
+    return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS;
+  }
 
-        StringBuilder str = new StringBuilder();
-        str.append(decimalToString[desiredLength - length]);
-        str.append(value);
+  /* Returns a string representation of the given integer
+   * If the length of the given integer is less than the
+   * passed length, this function will prepend zeroes to the string
+   */
+  public static StringBuilder toStringWithZeroes(int number, int desiredLength) {
+    String value = ((Integer) number).toString();
+    int length = value.length();
 
-        return str;
-    }
+    StringBuilder str = new StringBuilder();
+    str.append(decimalToString[desiredLength - length]);
+    str.append(value);
 
-    public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
-        String value = ((Long) number).toString();
-        int length = value.length();
+    return str;
+  }
 
-        StringBuilder str = new StringBuilder();
+  public static StringBuilder toStringWithZeroes(long number, int desiredLength) {
+    String value = ((Long) number).toString();
+    int length = value.length();
 
-        // Desired length can be > MAX_DIGITS
-        int zeroesLength = desiredLength - length;
-        while (zeroesLength > MAX_DIGITS) {
-            str.append(decimalToString[MAX_DIGITS]);
-            zeroesLength -= MAX_DIGITS;
-        }
-        str.append(decimalToString[zeroesLength]);
-        str.append(value);
+    StringBuilder str = new StringBuilder();
 
-        return str;
+    // Desired length can be > MAX_DIGITS
+    int zeroesLength = desiredLength - length;
+    while (zeroesLength > MAX_DIGITS) {
+      str.append(decimalToString[MAX_DIGITS]);
+      zeroesLength -= MAX_DIGITS;
     }
+    str.append(decimalToString[zeroesLength]);
+    str.append(value);
+
+    return str;
+  }
 
   public static BigDecimal getBigDecimalFromIntermediate(ByteBuf data, int startIndex, int nDecimalDigits, int scale) {
 
-        // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
-        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
-    }
+    // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false
+    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, false);
+  }
 
-    public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
+  public static BigDecimal getBigDecimalFromSparse(ArrowBuf data, int startIndex, int nDecimalDigits, int scale) {
 
-        // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
-        return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
-    }
+    // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate
+    return getBigDecimalFromArrowBuf(data, startIndex, nDecimalDigits, scale, true);
+  }
 
-    public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
-      byte[] value = new byte[length];
-      bytebuf.getBytes(start, value, 0, length);
-      BigInteger unscaledValue = new BigInteger(value);
-      return new BigDecimal(unscaledValue, scale);
-    }
+  public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int start, int length, int scale) {
+    byte[] value = new byte[length];
+    bytebuf.getBytes(start, value, 0, length);
+    BigInteger unscaledValue = new BigInteger(value);
+    return new BigDecimal(unscaledValue, scale);
+  }
 
   public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int start, int length, int scale) {
     byte[] value = new byte[length];
@@ -168,115 +167,123 @@ public class DecimalUtility extends CoreDecimalUtility{
     return new BigDecimal(unscaledValue, scale);
   }
 
-    /* Create a BigDecimal object using the data in the ArrowBuf.
-     * This function assumes that data is provided in a non-dense format
-     * It works on both sparse and intermediate representations.
-     */
+  public static void writeBigDecimalToArrowBuf(ArrowBuf bytebuf, int startIndex, BigDecimal value) {
+    byte[] bytes = value.unscaledValue().toByteArray();
+    if (bytes.length > 16) {
+      throw new UnsupportedOperationException("Decimal size greater than 16 bytes");
+    }
+    bytebuf.setBytes(startIndex + 16 - bytes.length, bytes, 0, bytes.length);
+  }
+
+  /* Create a BigDecimal object using the data in the ArrowBuf.
+   * This function assumes that data is provided in a non-dense format
+   * It works on both sparse and intermediate representations.
+   */
   public static BigDecimal getBigDecimalFromArrowBuf(ByteBuf data, int startIndex, int nDecimalDigits, int scale,
-      boolean truncateScale) {
+                                                     boolean truncateScale) {
 
-        // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
-        int actualDigits;
+    // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal.
+    int actualDigits;
 
-        // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
-        BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
+    // Initialize the BigDecimal, first digit in the ArrowBuf has the sign so mask it out
+    BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF);
 
-        BigInteger base = BigInteger.valueOf(DIGITS_BASE);
+    BigInteger base = BigInteger.valueOf(DIGITS_BASE);
 
-        for (int i = 1; i < nDecimalDigits; i++) {
+    for (int i = 1; i < nDecimalDigits; i++) {
 
-            BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
-            decimalDigits = decimalDigits.multiply(base);
-            decimalDigits = decimalDigits.add(temp);
-        }
+      BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * INTEGER_SIZE)));
+      decimalDigits = decimalDigits.multiply(base);
+      decimalDigits = decimalDigits.add(temp);
+    }
 
-        // Truncate any additional padding we might have added
-        if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
-            BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
-            decimalDigits = decimalDigits.divide(truncate);
-        }
+    // Truncate any additional padding we might have added
+    if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) {
+      BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits)));
+      decimalDigits = decimalDigits.divide(truncate);
+    }
 
-        // set the sign
-        if ((data.getInt(startIndex) & 0x80000000) != 0) {
-            decimalDigits = decimalDigits.negate();
-        }
+    // set the sign
+    if ((data.getInt(startIndex) & 0x80000000) != 0) {
+      decimalDigits = decimalDigits.negate();
+    }
 
-        BigDecimal decimal = new BigDecimal(decimalDigits, scale);
+    BigDecimal decimal = new BigDecimal(decimalDigits, scale);
 
-        return decimal;
-    }
+    return decimal;
+  }
 
-    /* This function returns a BigDecimal object from the dense decimal representation.
-     * First step is to convert the dense representation into an intermediate representation
-     * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
-     */
-    public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
+  /* This function returns a BigDecimal object from the dense decimal representation.
+   * First step is to convert the dense representation into an intermediate representation
+   * and then invoke getBigDecimalFromArrowBuf() to get the BigDecimal object
+   */
+  public static BigDecimal getBigDecimalFromDense(ArrowBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) {
 
         /* This method converts the dense representation to
          * an intermediate representation. The intermediate
          * representation has one more integer than the dense
          * representation.
          */
-        byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
-
-        // Start storing from the least significant byte of the first integer
-        int intermediateIndex = 3;
-
-        int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
-        int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
-
-        int maskIndex;
-        int shiftOrder;
-        byte shiftBits;
-
-        // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
-        if (maxPrecision == 38) {
-            maskIndex = 0;
-            shiftOrder = 6;
-            shiftBits = 0x00;
-            intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
-        } else if (maxPrecision == 28) {
-            maskIndex = 1;
-            shiftOrder = 4;
-            shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
-            intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
-        } else {
-            throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
-        }
+    byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * INTEGER_SIZE)];
+
+    // Start storing from the least significant byte of the first integer
+    int intermediateIndex = 3;
+
+    int[] mask = {0x03, 0x0F, 0x3F, 0xFF};
+    int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00};
+
+    int maskIndex;
+    int shiftOrder;
+    byte shiftBits;
+
+    // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code
+    if (maxPrecision == 38) {
+      maskIndex = 0;
+      shiftOrder = 6;
+      shiftBits = 0x00;
+      intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F);
+    } else if (maxPrecision == 28) {
+      maskIndex = 1;
+      shiftOrder = 4;
+      shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder);
+      intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2);
+    } else {
+      throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported");
+    }
 
-        int inputIndex = 1;
-        boolean sign = false;
+    int inputIndex = 1;
+    boolean sign = false;
 
-        if ((data.getByte(startIndex) & 0x80) != 0) {
-            sign = true;
-        }
+    if ((data.getByte(startIndex) & 0x80) != 0) {
+      sign = true;
+    }
 
-        while (inputIndex < width) {
+    while (inputIndex < width) {
 
-            intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
+      intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder)));
 
-            shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
+      shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder);
 
-            inputIndex++;
-            intermediateIndex++;
+      inputIndex++;
+      intermediateIndex++;
 
-            if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
-                shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
-                maskIndex++;
-                shiftOrder -= 2;
-            }
+      if (((inputIndex - 1) % INTEGER_SIZE) == 0) {
+        shiftBits = (byte) ((shiftBits & 0xFF) >>> 2);
+        maskIndex++;
+        shiftOrder -= 2;
+      }
 
-        }
+    }
         /* copy the last byte */
-        intermediateBytes[intermediateIndex] = shiftBits;
+    intermediateBytes[intermediateIndex] = shiftBits;
 
-        if (sign == true) {
-            intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
-        }
+    if (sign == true) {
+      intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80);
+    }
 
     final ByteBuf intermediate = UnpooledByteBufAllocator.DEFAULT.buffer(intermediateBytes.length);
     try {
-        intermediate.setBytes(0, intermediateBytes);
+      intermediate.setBytes(0, intermediateBytes);
 
       BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale);
       return ret;
@@ -284,299 +291,296 @@ public class DecimalUtility extends CoreDecimalUtility{
       intermediate.release();
     }
 
-    }
+  }
 
-    /*
-     * Function converts the BigDecimal and stores it in out internal sparse representation
-     */
-  public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
-      int nDecimalDigits) {
+ public static void getSparseFromBigDecimal(BigDecimal input, ByteBuf data, int startIndex, int scale, int precision,
+                                             int nDecimalDigits) {
 
-        // Initialize the buffer
-        for (int i = 0; i < nDecimalDigits; i++) {
-          data.setInt(startIndex + (i * INTEGER_SIZE), 0);
-        }
+    // Initialize the buffer
+    for (int i = 0; i < nDecimalDigits; i++) {
+      data.setInt(startIndex + (i * INTEGER_SIZE), 0);
+    }
 
-        boolean sign = false;
+    boolean sign = false;
 
-        if (input.signum() == -1) {
-            // negative input
-            sign = true;
-            input = input.abs();
-        }
+    if (input.signum() == -1) {
+      // negative input
+      sign = true;
+      input = input.abs();
+    }
 
-        // Truncate the input as per the scale provided
-        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
+    // Truncate the input as per the scale provided
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-        // Separate out the integer part
-        BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
+    // Separate out the integer part
+    BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN);
 
-        int destIndex = nDecimalDigits - roundUp(scale) - 1;
+    int destIndex = nDecimalDigits - roundUp(scale) - 1;
 
-        // we use base 1 billion integer digits for out integernal representation
-        BigDecimal base = new BigDecimal(DIGITS_BASE);
+    // we use base 1 billion integer digits for out integernal representation
+    BigDecimal base = new BigDecimal(DIGITS_BASE);
 
-        while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
-            // store the modulo as the integer value
-            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
-            destIndex--;
-            // Divide by base 1 billion
-            integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
-        }
+    while (integerPart.compareTo(BigDecimal.ZERO) == 1) {
+      // store the modulo as the integer value
+      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (integerPart.remainder(base)).intValue());
+      destIndex--;
+      // Divide by base 1 billion
+      integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN);
+    }
 
         /* Sparse representation contains padding of additional zeroes
          * so each digit contains MAX_DIGITS for ease of arithmetic
          */
-        int actualDigits;
-        if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
-            // Pad additional zeroes
-            scale = scale + (MAX_DIGITS - actualDigits);
-            input = input.setScale(scale, BigDecimal.ROUND_DOWN);
-        }
-
-        //separate out the fractional part
-        BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
+    int actualDigits;
+    if ((actualDigits = (scale % MAX_DIGITS)) != 0) {
+      // Pad additional zeroes
+      scale = scale + (MAX_DIGITS - actualDigits);
+      input = input.setScale(scale, BigDecimal.ROUND_DOWN);
+    }
 
-        destIndex = nDecimalDigits - 1;
+    //separate out the fractional part
+    BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale);
 
-        while (scale > 0) {
-            // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
-            fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
-            BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
+    destIndex = nDecimalDigits - 1;
 
-            data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
-            destIndex--;
+    while (scale > 0) {
+      // Get next set of MAX_DIGITS (9) store it in the ArrowBuf
+      fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS);
+      BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE);
 
-            fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
-            scale -= MAX_DIGITS;
-        }
+      data.setInt(startIndex + (destIndex * INTEGER_SIZE), (temp.unscaledValue().intValue()));
+      destIndex--;
 
-        // Set the negative sign
-        if (sign == true) {
-            data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
-        }
+      fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN);
+      scale -= MAX_DIGITS;
+    }
 
+    // Set the negative sign
+    if (sign == true) {
+      data.setInt(startIndex, data.getInt(startIndex) | 0x80000000);
     }
 
+  }
 
-    public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
-        // Truncate or pad to set the input to the correct scale
-        input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-        return (input.unscaledValue().longValue());
-    }
+  public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) {
+    // Truncate or pad to set the input to the correct scale
+    input = input.setScale(scale, BigDecimal.ROUND_HALF_UP);
 
-    public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
-      return BigDecimal.valueOf(input, scale);
-    }
+    return (input.unscaledValue().longValue());
+  }
 
-    public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
-      return BigDecimal.valueOf(input, scale);
-    }
+  public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) {
+    return BigDecimal.valueOf(input, scale);
+  }
+
+  public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) {
+    return BigDecimal.valueOf(input, scale);
+  }
 
 
-    public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
+  public static int compareDenseBytes(ArrowBuf left, int leftStart, boolean leftSign, ArrowBuf right, int rightStart, boolean rightSign, int width) {
 
-      int invert = 1;
+    int invert = 1;
 
       /* If signs are different then simply look at the
        * sign of the two inputs and determine which is greater
        */
-      if (leftSign != rightSign) {
+    if (leftSign != rightSign) {
 
-        return((leftSign == true) ? -1 : 1);
-      } else if(leftSign == true) {
+      return((leftSign == true) ? -1 : 1);
+    } else if(leftSign == true) {
         /* Both inputs are negative, at the end we will
          * have to invert the comparison
          */
-        invert = -1;
-      }
-
-      int cmp = 0;
-
-      for (int i = 0; i < width; i++) {
-        byte leftByte  = left.getByte(leftStart + i);
-        byte rightByte = right.getByte(rightStart + i);
-        // Unsigned byte comparison
-        if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
-          cmp = 1;
-          break;
-        } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
-          cmp = -1;
-          break;
-        }
-      }
-      cmp *= invert; // invert the comparison if both were negative values
-
-      return cmp;
+      invert = -1;
     }
 
-    public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
-      int value = buffer.getInt(start + (index * 4));
+    int cmp = 0;
 
-      if (index == 0) {
-        /* the first byte contains sign bit, return value without it */
-        value = (value & 0x7FFFFFFF);
+    for (int i = 0; i < width; i++) {
+      byte leftByte  = left.getByte(leftStart + i);
+      byte rightByte = right.getByte(rightStart + i);
+      // Unsigned byte comparison
+      if ((leftByte & 0xFF) > (rightByte & 0xFF)) {
+        cmp = 1;
+        break;
+      } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) {
+        cmp = -1;
+        break;
       }
-      return value;
     }
+    cmp *= invert; // invert the comparison if both were negative values
 
-    public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
-      buffer.setInt(start + (index * 4), value);
+    return cmp;
+  }
+
+  public static int getIntegerFromSparseBuffer(ArrowBuf buffer, int start, int index) {
+    int value = buffer.getInt(start + (index * 4));
+
+    if (index == 0) {
+        /* the first byte contains sign bit, return value without it */
+      value = (value & 0x7FFFFFFF);
     }
+    return value;
+  }
 
-    public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
+  public static void setInteger(ArrowBuf buffer, int start, int index, int value) {
+    buffer.setInt(start + (index * 4), value);
+  }
 
-      int invert = 1;
+  public static int compareSparseBytes(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) {
 
-      if (absCompare == false) {
-        if (leftSign != rightSign) {
-          return (leftSign == true) ? -1 : 1;
-        }
+    int invert = 1;
 
-        // Both values are negative invert the outcome of the comparison
-        if (leftSign == true) {
-          invert = -1;
-        }
+    if (absCompare == false) {
+      if (leftSign != rightSign) {
+        return (leftSign == true) ? -1 : 1;
       }
 
-      int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
-      return cmp * invert;
+      // Both values are negative invert the outcome of the comparison
+      if (leftSign == true) {
+        invert = -1;
+      }
     }
-    public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
+
+    int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits);
+    return cmp * invert;
+  }
+  public static int compareSparseBytesInner(ArrowBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, ArrowBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) {
       /* compute the number of integer digits in each decimal */
-      int leftInt  = leftPrecision - leftScale;
-      int rightInt = rightPrecision - rightScale;
+    int leftInt  = leftPrecision - leftScale;
+    int rightInt = rightPrecision - rightScale;
 
       /* compute the number of indexes required for storing integer digits */
-      int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
-      int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
+    int leftIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftInt);
+    int rightIntRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightInt);
 
       /* compute number of indexes required for storing scale */
-      int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
-      int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
+    int leftScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(leftScale);
+    int rightScaleRoundedUp = org.apache.arrow.vector.util.DecimalUtility.roundUp(rightScale);
 
       /* compute index of the most significant integer digits */
-      int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-      int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+    int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+    int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
 
-      int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
-      int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
+    int leftStopIndex = nDecimalDigits - leftScaleRoundedUp;
+    int rightStopIndex = nDecimalDigits - rightScaleRoundedUp;
 
       /* Discard the zeroes in the integer part */
-      while (leftIndex1 < leftStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-          break;
-        }
+    while (leftIndex1 < leftStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+        break;
+      }
 
         /* Digit in this location is zero, decrement the actual number
          * of integer digits
          */
-        leftIntRoundedUp--;
-        leftIndex1++;
-      }
+      leftIntRoundedUp--;
+      leftIndex1++;
+    }
 
       /* If we reached the stop index then the number of integers is zero */
-      if (leftIndex1 == leftStopIndex) {
-        leftIntRoundedUp = 0;
-      }
+    if (leftIndex1 == leftStopIndex) {
+      leftIntRoundedUp = 0;
+    }
 
-      while (rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-          break;
-        }
+    while (rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+        break;
+      }
 
         /* Digit in this location is zero, decrement the actual number
          * of integer digits
          */
-        rightIntRoundedUp--;
-        rightIndex1++;
-      }
+      rightIntRoundedUp--;
+      rightIndex1++;
+    }
 
-      if (rightIndex1 == rightStopIndex) {
-        rightIntRoundedUp = 0;
-      }
+    if (rightIndex1 == rightStopIndex) {
+      rightIntRoundedUp = 0;
+    }
 
       /* We have the accurate number of non-zero integer digits,
        * if the number of integer digits are different then we can determine
        * which decimal is larger and needn't go down to comparing individual values
        */
-      if (leftIntRoundedUp > rightIntRoundedUp) {
-        return 1;
-      }
-      else if (rightIntRoundedUp > leftIntRoundedUp) {
-        return -1;
-      }
+    if (leftIntRoundedUp > rightIntRoundedUp) {
+      return 1;
+    }
+    else if (rightIntRoundedUp > leftIntRoundedUp) {
+      return -1;
+    }
 
       /* The number of integer digits are the same, set the each index
        * to the first non-zero integer and compare each digit
        */
-      leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
-      rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
+    leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp;
+    rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp;
 
-      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-          return 1;
-        }
-        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-          return -1;
-        }
-
-        leftIndex1++;
-        rightIndex1++;
+    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+        return 1;
+      }
+      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+        return -1;
       }
 
+      leftIndex1++;
+      rightIndex1++;
+    }
+
       /* The integer part of both the decimal's are equal, now compare
        * each individual fractional part. Set the index to be at the
        * beginning of the fractional part
        */
-      leftIndex1 = leftStopIndex;
-      rightIndex1 = rightStopIndex;
+    leftIndex1 = leftStopIndex;
+    rightIndex1 = rightStopIndex;
 
       /* Stop indexes will be the end of the array */
-      leftStopIndex = nDecimalDigits;
-      rightStopIndex = nDecimalDigits;
+    leftStopIndex = nDecimalDigits;
+    rightStopIndex = nDecimalDigits;
 
       /* compare the two fractional parts of the decimal */
-      while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
-          return 1;
-        }
-        else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
-          return -1;
-        }
-
-        leftIndex1++;
-        rightIndex1++;
+    while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) {
+        return 1;
+      }
+      else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) {
+        return -1;
       }
 
+      leftIndex1++;
+      rightIndex1++;
+    }
+
       /* Till now the fractional part of the decimals are equal, check
        * if one of the decimal has fractional part that is remaining
        * and is non-zero
        */
-      while (leftIndex1 < leftStopIndex) {
-        if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
-          return 1;
-        }
-        leftIndex1++;
+    while (leftIndex1 < leftStopIndex) {
+      if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) {
+        return 1;
       }
+      leftIndex1++;
+    }
 
-      while(rightIndex1 < rightStopIndex) {
-        if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
-          return -1;
-        }
-        rightIndex1++;
+    while(rightIndex1 < rightStopIndex) {
+      if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) {
+        return -1;
       }
+      rightIndex1++;
+    }
 
       /* Both decimal values are equal */
-      return 0;
-    }
+    return 0;
+  }
 
-    public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
-      byte[] value = Arrays.copyOfRange(bytes, start, start + length);
-      BigInteger unscaledValue = new BigInteger(value);
-      return new BigDecimal(unscaledValue, scale);
-    }
+  public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) {
+    byte[] value = Arrays.copyOfRange(bytes, start, start + length);
+    BigInteger unscaledValue = new BigInteger(value);
+    return new BigDecimal(unscaledValue, scale);
+  }
 
   public static void roundDecimal(ArrowBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) {
     int newScaleRoundedUp  = org.apache.arrow.vector.util.DecimalUtility.roundUp(desiredScale);
@@ -704,34 +708,6 @@ public class DecimalUtility extends CoreDecimalUtility{
     int index = nDecimalDigits - roundUp(scale);
     return (int) (adjustScaleDivide(data.getInt(start + (index * INTEGER_SIZE)), MAX_DIGITS - 1));
   }
-
-  public static int compareSparseSamePrecScale(ArrowBuf left, int lStart, byte[] right, int length) {
-    // check the sign first
-    boolean lSign = (left.getInt(lStart) & 0x80000000) != 0;
-    boolean rSign = ByteFunctionHelpers.getSign(right);
-    int cmp = 0;
-
-    if (lSign != rSign) {
-      return (lSign == false) ? 1 : -1;
-    }
-
-    // invert the comparison if we are comparing negative numbers
-    int invert = (lSign == true) ? -1 : 1;
-
-    // compare byte by byte
-    int n = 0;
-    int lPos = lStart;
-    int rPos = 0;
-    while (n < length/4) {
-      int leftInt = Decimal38SparseHolder.getInteger(n, lStart, left);
-      int rightInt = ByteFunctionHelpers.getInteger(right, n);
-      if (leftInt != rightInt) {
-        cmp =  (leftInt - rightInt ) > 0 ? 1 : -1;
-        break;
-      }
-      n++;
-    }
-    return cmp * invert;
-  }
 }
 
+

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index dea433e..d7f9d38 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -18,7 +18,9 @@
 package org.apache.arrow.vector.util;
 
 import java.util.AbstractMap;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -241,6 +243,16 @@ public class MapWithOrdinal<K, V> implements Map<K, V> {
     return delegate.keySet();
   }
 
+  public List<K> keyList() {
+    int size = size();
+    Set<K> keys = keySet();
+    List<K> children = new ArrayList<>(size);
+    for (K key : keys) {
+      children.add(getOrdinal(key), key);
+    }
+    return children;
+  }
+
   @Override
   public Set<Entry<K, V>> entrySet() {
     return delegate.entrySet();

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
new file mode 100644
index 0000000..7ab7db3
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.util.DecimalUtility;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDecimalVector {
+
+  private static long[] intValues;
+
+  static {
+    intValues = new long[30];
+    for (int i = 0; i < intValues.length; i++) {
+      intValues[i] = 1 << i + 1;
+    }
+  }
+  private int scale = 3;
+
+  @Test
+  public void test() {
+    BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+    NullableDecimalVector decimalVector = new NullableDecimalVector("decimal", allocator, 10, scale);
+    decimalVector.allocateNew();
+    BigDecimal[] values = new BigDecimal[intValues.length];
+    for (int i = 0; i < intValues.length; i++) {
+      BigDecimal decimal = new BigDecimal(BigInteger.valueOf(intValues[i]), scale);
+      values[i] = decimal;
+      decimalVector.getMutator().setIndexDefined(i);
+      DecimalUtility.writeBigDecimalToArrowBuf(decimalVector.getBuffer(), i * 16, decimal);
+    }
+
+    decimalVector.getMutator().setValueCount(intValues.length);
+
+    for (int i = 0; i < intValues.length; i++) {
+      BigDecimal value = decimalVector.getAccessor().getObject(i);
+      assertEquals(values[i], value);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
index 4dee86c..9baebc5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java
@@ -20,8 +20,6 @@ package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
@@ -53,8 +51,7 @@ public class TestOversizedAllocationForValueVector {
 
   @Test(expected = OversizedAllocationException.class)
   public void testFixedVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final UInt4Vector vector = new UInt4Vector(field, allocator);
+    final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: buffer size = max value capacity
     final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
     try {
@@ -78,8 +75,7 @@ public class TestOversizedAllocationForValueVector {
 
   @Test(expected = OversizedAllocationException.class)
   public void testBitVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final BitVector vector = new BitVector(field, allocator);
+    final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: buffer size ~ max value capacity
     final int expectedValueCapacity = 1 << 29;
     try {
@@ -109,8 +105,7 @@ public class TestOversizedAllocationForValueVector {
 
   @Test(expected = OversizedAllocationException.class)
   public void testVariableVectorReallocation() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
-    final VarCharVector vector = new VarCharVector(field, allocator);
+    final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator);
     // edge case 1: value count = MAX_VALUE_ALLOCATION
     final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;
     final int expectedOffsetSize = 10;

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index e4d28c3..1bb50b7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -22,8 +22,6 @@ import static org.junit.Assert.assertEquals;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.holders.NullableUInt4Holder;
-import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.types.Types;
 import org.junit.After;
 import org.junit.Before;
@@ -46,13 +44,12 @@ public class TestUnionVector {
 
   @Test
   public void testUnionVector() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
     final NullableUInt4Holder uInt4Holder = new NullableUInt4Holder();
     uInt4Holder.value = 100;
     uInt4Holder.isSet = 1;
 
-    try (UnionVector unionVector = new UnionVector(field, allocator, null)) {
+    try (UnionVector unionVector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       unionVector.allocateNew();
 
       // write some data

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index ce091ab..21cdc4f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -19,15 +19,7 @@ package org.apache.arrow.vector;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.holders.*;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
 import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
@@ -50,9 +42,9 @@ public class TestValueVector {
   }
 
   private final static Charset utf8Charset = Charset.forName("UTF-8");
-  private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset);
-  private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset);
-  private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset);
+  private final static byte[] STR1 = "AAAAA1".getBytes(utf8Charset);
+  private final static byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset);
+  private final static byte[] STR3 = "CCCC3".getBytes(utf8Charset);
 
   @After
   public void terminate() throws Exception {
@@ -61,10 +53,9 @@ public class TestValueVector {
 
   @Test
   public void testFixedType() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final UInt4Vector vector = new UInt4Vector(field, allocator)) {
+    try (final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
       final UInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -86,10 +77,9 @@ public class TestValueVector {
 
   @Test
   public void testNullableVarLen2() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
+    try (final NullableVarCharVector vector = new NullableVarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
@@ -116,44 +106,10 @@ public class TestValueVector {
   }
 
   @Test
-  public void testRepeatedIntVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
-
-    // Create a new value vector.
-    try (final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator)) {
-
-      // Populate the vector.
-      final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
-      final int nRecords = 7;
-      final int nElements = values.length;
-      vector1.allocateNew(nRecords, nRecords * nElements);
-      final RepeatedIntVector.Mutator mutator = vector1.getMutator();
-      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
-        mutator.startNewValue(recordIndex);
-        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
-          mutator.add(recordIndex, recordIndex * values[elementIndex]);
-        }
-      }
-      mutator.setValueCount(nRecords);
-
-      // Verify the contents.
-      final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
-      assertEquals(nRecords, accessor1.getValueCount());
-      for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
-        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
-          final int value = accessor1.get(recordIndex, elementIndex);
-          assertEquals(recordIndex * values[elementIndex], value);
-        }
-      }
-    }
-  }
-
-  @Test
   public void testNullableFixedType() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
 
     // Create a new value vector for 1024 integers.
-    try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
+    try (final NullableUInt4Vector vector = new NullableUInt4Vector(EMPTY_SCHEMA_PATH, allocator)) {
       final NullableUInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -222,10 +178,8 @@ public class TestValueVector {
 
   @Test
   public void testNullableFloat() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -271,10 +225,8 @@ public class TestValueVector {
 
   @Test
   public void testBitVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final BitVector vector = new BitVector(field, allocator)) {
+    try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
       final BitVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -311,10 +263,8 @@ public class TestValueVector {
 
   @Test
   public void testReAllocNullableFixedWidthVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableFloat4Vector vector = (NullableFloat4Vector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
@@ -346,10 +296,8 @@ public class TestValueVector {
 
   @Test
   public void testReAllocNullableVariableWidthVector() {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
-
     // Create a new value vector for 1024 integers
-    try (final NullableVarCharVector vector = (NullableVarCharVector) BasicTypeHelper.getNewVector(field, allocator)) {
+    try (final NullableVarCharVector vector = (NullableVarCharVector) MinorType.VARCHAR.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {
       final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
@@ -376,69 +324,4 @@ public class TestValueVector {
     }
   }
 
-  @Test
-  public void testVVInitialCapacity() throws Exception {
-    final MaterializedField[] fields = new MaterializedField[9];
-    final ValueVector[] valueVectors = new ValueVector[9];
-
-    fields[0] = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
-    fields[1] = MaterializedField.create(EMPTY_SCHEMA_PATH, IntHolder.TYPE);
-    fields[2] = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
-    fields[3] = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVar16CharHolder.TYPE);
-    fields[4] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedFloat4Holder.TYPE);
-    fields[5] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedVarBinaryHolder.TYPE);
-
-    fields[6] = MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE);
-    fields[6].addChild(fields[0] /*bit*/);
-    fields[6].addChild(fields[2] /*varchar*/);
-
-    fields[7] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE);
-    fields[7].addChild(fields[1] /*int*/);
-    fields[7].addChild(fields[3] /*optional var16char*/);
-
-    fields[8] = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE);
-    fields[8].addChild(fields[1] /*int*/);
-
-    final int initialCapacity = 1024;
-
-    try {
-      for (int i = 0; i < valueVectors.length; i++) {
-        valueVectors[i] = BasicTypeHelper.getNewVector(fields[i], allocator);
-        valueVectors[i].setInitialCapacity(initialCapacity);
-        valueVectors[i].allocateNew();
-      }
-
-      for (int i = 0; i < valueVectors.length; i++) {
-        final ValueVector vv = valueVectors[i];
-        final int vvCapacity = vv.getValueCapacity();
-
-        // this can't be equality because Nullables will be allocated using power of two sized buffers (thus need 1025
-        // spots in one vector > power of two is 2048, available capacity will be 2048 => 2047)
-        assertTrue(String.format("Incorrect value capacity for %s [%d]", vv.getField(), vvCapacity),
-                initialCapacity <= vvCapacity);
-      }
-    } finally {
-      for (ValueVector v : valueVectors) {
-        v.close();
-      }
-    }
-  }
-
-  @Test
-  public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH,
-            Types.optional(MinorType.LIST));
-    ListVector vector = new ListVector(field, allocator, null);
-    ListVector vectorFrom = new ListVector(field, allocator, null);
-    vectorFrom.allocateNew();
-
-    for (int i = 0; i < 10000; i++) {
-      vector.allocateNew();
-      vector.copyFromSafe(0, 0, vectorFrom);
-      vector.clear();
-    }
-
-    vectorFrom.clear();
-    vector.clear();
-  }
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 4c24444..24f00f1 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,7 +27,7 @@ import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.holders.UInt4Holder;
-import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -49,10 +49,9 @@ public class TestPromotableWriter {
 
   @Test
   public void testPromoteToUnion() throws Exception {
-    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
-    try (final AbstractMapVector container = new MapVector(field, allocator, null);
-         final MapVector v = container.addOrGet("test", MapVector.TYPE, MapVector.class);
+    try (final AbstractMapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+         final MapVector v = container.addOrGet("test", MinorType.MAP, MapVector.class);
          final PromotableWriter writer = new PromotableWriter(v, container)) {
 
       container.allocateNew();

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
new file mode 100644
index 0000000..bc17a2b
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex.writer;
+
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.complex.UnionVector;
+import org.apache.arrow.vector.complex.impl.ComplexWriterImpl;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.impl.UnionReader;
+import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestComplexWriter {
+
+  static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+
+  private static final int COUNT = 100;
+
+  @Test
+  public void simpleNestedTypes() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    IntWriter intWriter = rootWriter.integer("int");
+    BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+    for (int i = 0; i < COUNT; i++) {
+      intWriter.setPosition(i);
+      intWriter.writeInt(i);
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    writer.setValueCount(COUNT);
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < COUNT; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue());
+      Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue());
+    }
+
+    parent.close();
+  }
+
+  @Test
+  public void listScalarType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        listWriter.writeInt(j);
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        Assert.assertEquals(j, listReader.reader().readInteger().intValue());
+      }
+    }
+  }
+
+
+  @Test
+  public void listMapType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    MapWriter mapWriter = listWriter.map();
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        mapWriter.start();
+        mapWriter.integer("int").writeInt(j);
+        mapWriter.bigInt("bigInt").writeBigInt(j);
+        mapWriter.end();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        Assert.assertEquals("record: " + i, j, listReader.reader().reader("int").readInteger().intValue());
+        Assert.assertEquals(j, listReader.reader().reader("bigInt").readLong().longValue());
+      }
+    }
+  }
+
+  @Test
+  public void listListType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        ListWriter innerListWriter = listWriter.list();
+        innerListWriter.startList();
+        for (int k = 0; k < i % 13; k++) {
+          innerListWriter.integer().writeInt(k);
+        }
+        innerListWriter.endList();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        FieldReader innerListReader = listReader.reader();
+        for (int k = 0; k < i % 13; k++) {
+          innerListReader.next();
+          Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+        }
+      }
+    }
+    listVector.clear();
+  }
+
+  @Test
+  public void unionListListType() {
+    ListVector listVector = new ListVector("list", allocator, null);
+    listVector.allocateNew();
+    UnionListWriter listWriter = new UnionListWriter(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listWriter.setPosition(i);
+      listWriter.startList();
+      for (int j = 0; j < i % 7; j++) {
+        ListWriter innerListWriter = listWriter.list();
+        innerListWriter.startList();
+        for (int k = 0; k < i % 13; k++) {
+          if (k % 2 == 0) {
+            innerListWriter.integer().writeInt(k);
+          } else {
+            innerListWriter.bigInt().writeBigInt(k);
+          }
+        }
+        innerListWriter.endList();
+      }
+      listWriter.endList();
+    }
+    listWriter.setValueCount(COUNT);
+    UnionListReader listReader = new UnionListReader(listVector);
+    for (int i = 0; i < COUNT; i++) {
+      listReader.setPosition(i);
+      for (int j = 0; j < i % 7; j++) {
+        listReader.next();
+        FieldReader innerListReader = listReader.reader();
+        for (int k = 0; k < i % 13; k++) {
+          innerListReader.next();
+          if (k % 2 == 0) {
+            Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue());
+          } else {
+            Assert.assertEquals("record: " + i, k, innerListReader.reader().readLong().longValue());
+          }
+        }
+      }
+    }
+    listVector.clear();
+  }
+
+  @Test
+  public void simpleUnion() {
+    UnionVector vector = new UnionVector("union", allocator, null);
+    UnionWriter unionWriter = new UnionWriter(vector);
+    unionWriter.allocate();
+    for (int i = 0; i < COUNT; i++) {
+      unionWriter.setPosition(i);
+      if (i % 2 == 0) {
+        unionWriter.writeInt(i);
+      } else {
+        unionWriter.writeFloat4((float) i);
+      }
+    }
+    vector.getMutator().setValueCount(COUNT);
+    UnionReader unionReader = new UnionReader(vector);
+    for (int i = 0; i < COUNT; i++) {
+      unionReader.setPosition(i);
+      if (i % 2 == 0) {
+        Assert.assertEquals(i, i, unionReader.readInteger());
+      } else {
+        Assert.assertEquals((float) i, unionReader.readFloat(), 1e-12);
+      }
+    }
+    vector.close();
+  }
+
+  @Test
+  public void promotableWriter() {
+    MapVector parent = new MapVector("parent", allocator, null);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    for (int i = 0; i < 100; i++) {
+      BigIntWriter bigIntWriter = rootWriter.bigInt("a");
+      bigIntWriter.setPosition(i);
+      bigIntWriter.writeBigInt(i);
+    }
+    Field field = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("a", field.getName());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeType());
+    Int intType = (Int) field.getType();
+
+    Assert.assertEquals(64, intType.getBitWidth());
+    Assert.assertTrue(intType.getIsSigned());
+    for (int i = 100; i < 200; i++) {
+      VarCharWriter varCharWriter = rootWriter.varChar("a");
+      varCharWriter.setPosition(i);
+      byte[] bytes = Integer.toString(i).getBytes();
+      ArrowBuf tempBuf = allocator.buffer(bytes.length);
+      tempBuf.setBytes(0, bytes);
+      varCharWriter.writeVarChar(0, bytes.length, tempBuf);
+    }
+    field = parent.getField().getChildren().get(0).getChildren().get(0);
+    Assert.assertEquals("a", field.getName());
+    Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+    Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+    Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+    MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+    for (int i = 0; i < 100; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(i, rootReader.reader("a").readLong().intValue());
+    }
+    for (int i = 100; i < 200; i++) {
+      rootReader.setPosition(i);
+      Assert.assertEquals(Integer.toString(i), rootReader.reader("a").readText().toString());
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
new file mode 100644
index 0000000..06a1149
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.pojo;
+
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test conversion between Flatbuf and Pojo field representations
+ */
+public class TestConvert {
+
+  @Test
+  public void simple() {
+    Field initialField = new Field("a", true, new Int(32, true), null);
+    run(initialField);
+  }
+
+  @Test
+  public void complex() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+
+    Field initialField = new Field("a", true, Tuple.INSTANCE, childrenBuilder.build());
+    run(initialField);
+  }
+
+  @Test
+  public void schema() {
+    ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+    childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
+    childrenBuilder.add(new Field("child2", true, new FloatingPoint(0), ImmutableList.<Field>of()));
+    Schema initialSchema = new Schema(childrenBuilder.build());
+    run(initialSchema);
+
+  }
+
+  private void run(Field initialField) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(initialField.getField(builder));
+    org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer());
+    Field finalField = Field.convertField(flatBufField);
+    assertEquals(initialField, finalField);
+  }
+
+  private void run(Schema initialSchema) {
+    FlatBufferBuilder builder = new FlatBufferBuilder();
+    builder.finish(initialSchema.getSchema(builder));
+    org.apache.arrow.flatbuf.Schema flatBufSchema = org.apache.arrow.flatbuf.Schema.getRootAsSchema(builder.dataBuffer());
+    Schema finalSchema = Schema.convertSchema(flatBufSchema);
+    assertEquals(initialSchema, finalSchema);
+  }
+}


Mime
View raw message