hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [35/43] hive git commit: HIVE-11253. Move SearchArgument and VectorizedRowBatch classes to storage-api. (omalley reviewed by prasanthj)
Date Fri, 31 Jul 2015 00:43:35 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/9ae70cb4/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
new file mode 100644
index 0000000..d27ac16
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
@@ -0,0 +1,687 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.sarg;
+
+import java.sql.Timestamp;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The implementation of SearchArguments.
+ */
+final class SearchArgumentImpl implements SearchArgument {
+  public static final Log LOG = LogFactory.getLog(SearchArgumentImpl.class);
+
+  static final class PredicateLeafImpl implements PredicateLeaf {
+    private final Operator operator;
+    private final Type type;
+    private final String columnName;
+    private final Object literal;
+    private final List<Object> literalList;
+
+    // Used by kryo
+    @SuppressWarnings("unused")
+    PredicateLeafImpl() {
+      operator = null;
+      type = null;
+      columnName = null;
+      literal = null;
+      literalList = null;
+    }
+
+    PredicateLeafImpl(Operator operator,
+                      Type type,
+                      String columnName,
+                      Object literal,
+                      List<Object> literalList) {
+      this.operator = operator;
+      this.type = type;
+      this.columnName = columnName;
+      this.literal = literal;
+      if (literal != null) {
+        if (literal.getClass() != type.getValueClass()) {
+          throw new IllegalArgumentException("Wrong value class " +
+              literal.getClass().getName() + " for " + type + "." + operator +
+              " leaf");
+        }
+      }
+      this.literalList = literalList;
+      if (literalList != null) {
+        Class valueCls = type.getValueClass();
+        for(Object lit: literalList) {
+          if (lit != null && lit.getClass() != valueCls) {
+            throw new IllegalArgumentException("Wrong value class item " +
+                lit.getClass().getName() + " for " + type + "." + operator +
+                " leaf");
+          }
+        }
+      }
+    }
+
+    @Override
+    public Operator getOperator() {
+      return operator;
+    }
+
+    @Override
+    public Type getType(){
+      return type;
+    }
+
+    @Override
+    public String getColumnName() {
+      return columnName;
+    }
+
+    @Override
+    public Object getLiteral() {
+      // To get around a kryo 2.22 bug while deserialize a Timestamp into Date
+      // (https://github.com/EsotericSoftware/kryo/issues/88)
+      // When we see a Date, convert back into Timestamp
+      if (literal instanceof java.util.Date) {
+        return new Timestamp(((java.util.Date)literal).getTime());
+      }
+      return literal;
+    }
+
+    @Override
+    public List<Object> getLiteralList() {
+      return literalList;
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder buffer = new StringBuilder();
+      buffer.append('(');
+      buffer.append(operator);
+      buffer.append(' ');
+      buffer.append(columnName);
+      if (literal != null) {
+        buffer.append(' ');
+        buffer.append(literal);
+      } else if (literalList != null) {
+        for(Object lit: literalList) {
+          buffer.append(' ');
+          buffer.append(lit == null ? "null" : lit.toString());
+        }
+      }
+      buffer.append(')');
+      return buffer.toString();
+    }
+
+    private static boolean isEqual(Object left, Object right) {
+
+      return left == right ||
+          (left != null && right != null && left.equals(right));
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      if (other == null || other.getClass() != getClass()) {
+        return false;
+      } else if (other == this) {
+        return true;
+      } else {
+        PredicateLeafImpl o = (PredicateLeafImpl) other;
+        return operator == o.operator &&
+            type == o.type &&
+            columnName.equals(o.columnName) &&
+            isEqual(literal, o.literal) &&
+            isEqual(literalList, o.literalList);
+      }
+    }
+
+    @Override
+    public int hashCode() {
+      return operator.hashCode() +
+             type.hashCode() * 17 +
+             columnName.hashCode() * 3 * 17+
+             (literal == null ? 0 : literal.hashCode()) * 101 * 3 * 17 +
+             (literalList == null ? 0 : literalList.hashCode()) *
+                 103 * 101 * 3 * 17;
+    }
+  }
+
+
+  private final List<PredicateLeaf> leaves;
+  private final ExpressionTree expression;
+
+  SearchArgumentImpl(ExpressionTree expression, List<PredicateLeaf> leaves) {
+    this.expression = expression;
+    this.leaves = leaves;
+  }
+
+  // Used by kyro
+  @SuppressWarnings("unused")
+  SearchArgumentImpl() {
+        leaves = null;
+        expression = null;
+  }
+
+  @Override
+  public List<PredicateLeaf> getLeaves() {
+    return leaves;
+  }
+
+  @Override
+  public TruthValue evaluate(TruthValue[] leaves) {
+    return expression == null ? TruthValue.YES : expression.evaluate(leaves);
+  }
+
+  @Override
+  public ExpressionTree getExpression() {
+    return expression;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder buffer = new StringBuilder();
+    for(int i=0; i < leaves.size(); ++i) {
+      buffer.append("leaf-");
+      buffer.append(i);
+      buffer.append(" = ");
+      buffer.append(leaves.get(i).toString());
+      buffer.append('\n');
+    }
+    buffer.append("expr = ");
+    buffer.append(expression);
+    return buffer.toString();
+  }
+
+  static class BuilderImpl implements Builder {
+
+    // max threshold for CNF conversion. having >8 elements in andList will be
+    // converted to maybe
+    private static final int CNF_COMBINATIONS_THRESHOLD = 256;
+
+    private final Deque<ExpressionTree> currentTree =
+        new ArrayDeque<ExpressionTree>();
+    private final Map<PredicateLeaf, Integer> leaves =
+        new HashMap<PredicateLeaf, Integer>();
+    private final ExpressionTree root =
+        new ExpressionTree(ExpressionTree.Operator.AND);
+    {
+      currentTree.add(root);
+    }
+
+    @Override
+    public Builder startOr() {
+      ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.OR);
+      currentTree.getFirst().getChildren().add(node);
+      currentTree.addFirst(node);
+      return this;
+    }
+
+    @Override
+    public Builder startAnd() {
+      ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.AND);
+      currentTree.getFirst().getChildren().add(node);
+      currentTree.addFirst(node);
+      return this;
+    }
+
+    @Override
+    public Builder startNot() {
+      ExpressionTree node = new ExpressionTree(ExpressionTree.Operator.NOT);
+      currentTree.getFirst().getChildren().add(node);
+      currentTree.addFirst(node);
+      return this;
+    }
+
+    @Override
+    public Builder end() {
+      ExpressionTree current = currentTree.removeFirst();
+      if (current.getChildren().size() == 0) {
+        throw new IllegalArgumentException("Can't create expression " + root +
+            " with no children.");
+      }
+      if (current.getOperator() == ExpressionTree.Operator.NOT &&
+          current.getChildren().size() != 1) {
+        throw new IllegalArgumentException("Can't create not expression " +
+            current + " with more than 1 child.");
+      }
+      return this;
+    }
+
+    private int addLeaf(PredicateLeaf leaf) {
+      Integer result = leaves.get(leaf);
+      if (result == null) {
+        int id = leaves.size();
+        leaves.put(leaf, id);
+        return id;
+      } else {
+        return result;
+      }
+    }
+
+    @Override
+    public Builder lessThan(String column, PredicateLeaf.Type type,
+                            Object literal) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null || literal == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN,
+                type, column, literal, null);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder lessThanEquals(String column, PredicateLeaf.Type type,
+                                  Object literal) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null || literal == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.LESS_THAN_EQUALS,
+                type, column, literal, null);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder equals(String column, PredicateLeaf.Type type,
+                          Object literal) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null || literal == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.EQUALS,
+                type, column, literal, null);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder nullSafeEquals(String column, PredicateLeaf.Type type,
+                                  Object literal) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null || literal == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.NULL_SAFE_EQUALS,
+                type, column, literal, null);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder in(String column, PredicateLeaf.Type type,
+                      Object... literal) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column  == null || literal == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        if (literal.length == 0) {
+          throw new IllegalArgumentException("Can't create in expression with "
+              + "no arguments");
+        }
+        List<Object> argList = new ArrayList<Object>();
+        argList.addAll(Arrays.asList(literal));
+
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.IN,
+                type, column, null, argList);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder isNull(String column, PredicateLeaf.Type type) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.IS_NULL,
+                type, column, null, null);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder between(String column, PredicateLeaf.Type type, Object lower,
+                           Object upper) {
+      ExpressionTree parent = currentTree.getFirst();
+      if (column == null || lower == null || upper == null) {
+        parent.getChildren().add(new ExpressionTree(TruthValue.YES_NO_NULL));
+      } else {
+        List<Object> argList = new ArrayList<Object>();
+        argList.add(lower);
+        argList.add(upper);
+        PredicateLeaf leaf =
+            new PredicateLeafImpl(PredicateLeaf.Operator.BETWEEN,
+                type, column, null, argList);
+        parent.getChildren().add(new ExpressionTree(addLeaf(leaf)));
+      }
+      return this;
+    }
+
+    @Override
+    public Builder literal(TruthValue truth) {
+      ExpressionTree parent = currentTree.getFirst();
+      parent.getChildren().add(new ExpressionTree(truth));
+      return this;
+    }
+
+    /**
+     * Recursively explore the tree to find the leaves that are still reachable
+     * after optimizations.
+     * @param tree the node to check next
+     * @param next the next available leaf id
+     * @param leafReorder
+     * @return the next available leaf id
+     */
+    static int compactLeaves(ExpressionTree tree, int next, int[] leafReorder) {
+      if (tree.getOperator() == ExpressionTree.Operator.LEAF) {
+        int oldLeaf = tree.getLeaf();
+        if (leafReorder[oldLeaf] == -1) {
+          leafReorder[oldLeaf] = next++;
+        }
+      } else if (tree.getChildren() != null){
+        for(ExpressionTree child: tree.getChildren()) {
+          next = compactLeaves(child, next, leafReorder);
+        }
+      }
+      return next;
+    }
+
+    /**
+     * Rewrite expression tree to update the leaves.
+     * @param root the root of the tree to fix
+     * @param leafReorder a map from old leaf ids to new leaf ids
+     * @return the fixed root
+     */
+    static ExpressionTree rewriteLeaves(ExpressionTree root,
+                                        int[] leafReorder) {
+      if (root.getOperator() == ExpressionTree.Operator.LEAF) {
+        return new ExpressionTree(leafReorder[root.getLeaf()]);
+      } else if (root.getChildren() != null){
+        List<ExpressionTree> children = root.getChildren();
+        for(int i=0; i < children.size(); ++i) {
+          children.set(i, rewriteLeaves(children.get(i), leafReorder));
+        }
+      }
+      return root;
+    }
+
+    @Override
+    public SearchArgument build() {
+      if (currentTree.size() != 1) {
+        throw new IllegalArgumentException("Failed to end " +
+            currentTree.size() + " operations.");
+      }
+      ExpressionTree optimized = pushDownNot(root);
+      optimized = foldMaybe(optimized);
+      optimized = flatten(optimized);
+      optimized = convertToCNF(optimized);
+      optimized = flatten(optimized);
+      int leafReorder[] = new int[leaves.size()];
+      Arrays.fill(leafReorder, -1);
+      int newLeafCount = compactLeaves(optimized, 0, leafReorder);
+      optimized = rewriteLeaves(optimized, leafReorder);
+      ArrayList<PredicateLeaf> leafList = new ArrayList<>(newLeafCount);
+      // expand list to correct size
+      for(int i=0; i < newLeafCount; ++i) {
+        leafList.add(null);
+      }
+      // build the new list
+      for(Map.Entry<PredicateLeaf, Integer> elem: leaves.entrySet()) {
+        int newLoc = leafReorder[elem.getValue()];
+        if (newLoc != -1) {
+          leafList.set(newLoc, elem.getKey());
+        }
+      }
+      return new SearchArgumentImpl(optimized, leafList);
+    }
+
+    /**
+     * Push the negations all the way to just before the leaves. Also remove
+     * double negatives.
+     * @param root the expression to normalize
+     * @return the normalized expression, which may share some or all of the
+     * nodes of the original expression.
+     */
+    static ExpressionTree pushDownNot(ExpressionTree root) {
+      if (root.getOperator() == ExpressionTree.Operator.NOT) {
+        ExpressionTree child = root.getChildren().get(0);
+        switch (child.getOperator()) {
+          case NOT:
+            return pushDownNot(child.getChildren().get(0));
+          case CONSTANT:
+            return  new ExpressionTree(child.getConstant().not());
+          case AND:
+            root = new ExpressionTree(ExpressionTree.Operator.OR);
+            for(ExpressionTree kid: child.getChildren()) {
+              root.getChildren().add(pushDownNot(new
+                  ExpressionTree(ExpressionTree.Operator.NOT, kid)));
+            }
+            break;
+          case OR:
+            root = new ExpressionTree(ExpressionTree.Operator.AND);
+            for(ExpressionTree kid: child.getChildren()) {
+              root.getChildren().add(pushDownNot(new ExpressionTree
+                  (ExpressionTree.Operator.NOT, kid)));
+            }
+            break;
+          // for leaf, we don't do anything
+          default:
+            break;
+        }
+      } else if (root.getChildren() != null) {
+        // iterate through children and push down not for each one
+        for(int i=0; i < root.getChildren().size(); ++i) {
+          root.getChildren().set(i, pushDownNot(root.getChildren().get(i)));
+        }
+      }
+      return root;
+    }
+
+    /**
+     * Remove MAYBE values from the expression. If they are in an AND operator,
+     * they are dropped. If they are in an OR operator, they kill their parent.
+     * This assumes that pushDownNot has already been called.
+     * @param expr The expression to clean up
+     * @return The cleaned up expression
+     */
+    static ExpressionTree foldMaybe(ExpressionTree expr) {
+      if (expr.getChildren() != null) {
+        for(int i=0; i < expr.getChildren().size(); ++i) {
+          ExpressionTree child = foldMaybe(expr.getChildren().get(i));
+          if (child.getConstant() == TruthValue.YES_NO_NULL) {
+            switch (expr.getOperator()) {
+              case AND:
+                expr.getChildren().remove(i);
+                i -= 1;
+                break;
+              case OR:
+                // a maybe will kill the or condition
+                return child;
+              default:
+                throw new IllegalStateException("Got a maybe as child of " +
+                    expr);
+            }
+          } else {
+            expr.getChildren().set(i, child);
+          }
+        }
+        if (expr.getChildren().isEmpty()) {
+          return new ExpressionTree(TruthValue.YES_NO_NULL);
+        }
+      }
+      return expr;
+    }
+
+    /**
+     * Converts multi-level ands and ors into single level ones.
+     * @param root the expression to flatten
+     * @return the flattened expression, which will always be root with
+     *   potentially modified children.
+     */
+    static ExpressionTree flatten(ExpressionTree root) {
+      if (root.getChildren() != null) {
+        // iterate through the index, so that if we add more children,
+        // they don't get re-visited
+        for(int i=0; i < root.getChildren().size(); ++i) {
+          ExpressionTree child = flatten(root.getChildren().get(i));
+          // do we need to flatten?
+          if (child.getOperator() == root.getOperator() &&
+              child.getOperator() != ExpressionTree.Operator.NOT) {
+            boolean first = true;
+            for(ExpressionTree grandkid: child.getChildren()) {
+              // for the first grandkid replace the original parent
+              if (first) {
+                first = false;
+                root.getChildren().set(i, grandkid);
+              } else {
+                root.getChildren().add(++i, grandkid);
+              }
+            }
+          } else {
+            root.getChildren().set(i, child);
+          }
+        }
+        // if we have a singleton AND or OR, just return the child
+        if ((root.getOperator() == ExpressionTree.Operator.OR ||
+            root.getOperator() == ExpressionTree.Operator.AND) &&
+            root.getChildren().size() == 1) {
+          return root.getChildren().get(0);
+        }
+      }
+      return root;
+    }
+
+    /**
+     * Generate all combinations of items on the andList. For each item on the
+     * andList, it generates all combinations of one child from each and
+     * expression. Thus, (and a b) (and c d) will be expanded to: (or a c)
+     * (or a d) (or b c) (or b d). If there are items on the nonAndList, they
+     * are added to each or expression.
+     * @param result a list to put the results onto
+     * @param andList a list of and expressions
+     * @param nonAndList a list of non-and expressions
+     */
+    private static void generateAllCombinations(List<ExpressionTree> result,
+                                                List<ExpressionTree> andList,
+                                                List<ExpressionTree> nonAndList
+    ) {
+      List<ExpressionTree> kids = andList.get(0).getChildren();
+      if (result.isEmpty()) {
+        for(ExpressionTree kid: kids) {
+          ExpressionTree or = new ExpressionTree(ExpressionTree.Operator.OR);
+          result.add(or);
+          for(ExpressionTree node: nonAndList) {
+            or.getChildren().add(new ExpressionTree(node));
+          }
+          or.getChildren().add(kid);
+        }
+      } else {
+        List<ExpressionTree> work = new ArrayList<ExpressionTree>(result);
+        result.clear();
+        for(ExpressionTree kid: kids) {
+          for(ExpressionTree or: work) {
+            ExpressionTree copy = new ExpressionTree(or);
+            copy.getChildren().add(kid);
+            result.add(copy);
+          }
+        }
+      }
+      if (andList.size() > 1) {
+        generateAllCombinations(result, andList.subList(1, andList.size()),
+            nonAndList);
+      }
+    }
+
+    /**
+     * Convert an expression so that the top level operator is AND with OR
+     * operators under it. This routine assumes that all of the NOT operators
+     * have been pushed to the leaves via pushdDownNot.
+     * @param root the expression
+     * @return the normalized expression
+     */
+    static ExpressionTree convertToCNF(ExpressionTree root) {
+      if (root.getChildren() != null) {
+        // convert all of the children to CNF
+        int size = root.getChildren().size();
+        for(int i=0; i < size; ++i) {
+          root.getChildren().set(i, convertToCNF(root.getChildren().get(i)));
+        }
+        if (root.getOperator() == ExpressionTree.Operator.OR) {
+          // a list of leaves that weren't under AND expressions
+          List<ExpressionTree> nonAndList = new ArrayList<ExpressionTree>();
+          // a list of AND expressions that we need to distribute
+          List<ExpressionTree> andList = new ArrayList<ExpressionTree>();
+          for(ExpressionTree child: root.getChildren()) {
+            if (child.getOperator() == ExpressionTree.Operator.AND) {
+              andList.add(child);
+            } else if (child.getOperator() == ExpressionTree.Operator.OR) {
+              // pull apart the kids of the OR expression
+              for(ExpressionTree grandkid: child.getChildren()) {
+                nonAndList.add(grandkid);
+              }
+            } else {
+              nonAndList.add(child);
+            }
+          }
+          if (!andList.isEmpty()) {
+            if (checkCombinationsThreshold(andList)) {
+              root = new ExpressionTree(ExpressionTree.Operator.AND);
+              generateAllCombinations(root.getChildren(), andList, nonAndList);
+            } else {
+              root = new ExpressionTree(TruthValue.YES_NO_NULL);
+            }
+          }
+        }
+      }
+      return root;
+    }
+
+    private static boolean checkCombinationsThreshold(List<ExpressionTree> andList)
{
+      int numComb = 1;
+      for (ExpressionTree tree : andList) {
+        numComb *= tree.getChildren().size();
+        if (numComb > CNF_COMBINATIONS_THRESHOLD) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9ae70cb4/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
new file mode 100644
index 0000000..9890771
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
+
+public class HiveDecimalWritable implements WritableComparable<HiveDecimalWritable>
{
+
+  static final private Log LOG = LogFactory.getLog(HiveDecimalWritable.class);
+
+  private byte[] internalStorage = new byte[0];
+  private int scale;
+
+  public HiveDecimalWritable() {
+  }
+
+  public HiveDecimalWritable(String value) {
+    set(HiveDecimal.create(value));
+  }
+
+  public HiveDecimalWritable(byte[] bytes, int scale) {
+    set(bytes, scale);
+  }
+
+  public HiveDecimalWritable(HiveDecimalWritable writable) {
+    set(writable.getHiveDecimal());
+  }
+
+  public HiveDecimalWritable(HiveDecimal value) {
+    set(value);
+  }
+
+  public HiveDecimalWritable(long value) {
+    set((HiveDecimal.create(value)));
+  }
+
+  public void set(HiveDecimal value) {
+    set(value.unscaledValue().toByteArray(), value.scale());
+  }
+
+  public void set(HiveDecimal value, int maxPrecision, int maxScale) {
+    set(HiveDecimal.enforcePrecisionScale(value, maxPrecision, maxScale));
+  }
+
+  public void set(HiveDecimalWritable writable) {
+    set(writable.getHiveDecimal());
+  }
+
+  public void set(byte[] bytes, int scale) {
+    this.internalStorage = bytes;
+    this.scale = scale;
+  }
+
+  public HiveDecimal getHiveDecimal() {
+    return HiveDecimal.create(new BigInteger(internalStorage), scale);
+  }
+
+  /**
+   * Get a HiveDecimal instance from the writable and constraint it with maximum precision/scale.
+   *
+   * @param maxPrecision maximum precision
+   * @param maxScale maximum scale
+   * @return HiveDecimal instance
+   */
+  public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) {
+     return HiveDecimal.enforcePrecisionScale(HiveDecimal.
+             create(new BigInteger(internalStorage), scale),
+         maxPrecision, maxScale);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    scale = WritableUtils.readVInt(in);
+    int byteArrayLen = WritableUtils.readVInt(in);
+    if (internalStorage.length != byteArrayLen) {
+      internalStorage = new byte[byteArrayLen];
+    }
+    in.readFully(internalStorage);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    WritableUtils.writeVInt(out, scale);
+    WritableUtils.writeVInt(out, internalStorage.length);
+    out.write(internalStorage);
+  }
+
+  @Override
+  public int compareTo(HiveDecimalWritable that) {
+    return getHiveDecimal().compareTo(that.getHiveDecimal());
+  }
+
+  @Override
+  public String toString() {
+    return getHiveDecimal().toString();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (this == other) {
+      return true;
+    }
+    if (other == null || getClass() != other.getClass()) {
+      return false;
+    }
+    HiveDecimalWritable bdw = (HiveDecimalWritable) other;
+
+    // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want
+    // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is
+    // the same as 3.140). 'Equals' returns true iff equal and the same scale
+    // is set in the decimals (e.g.: 3.14 is not the same as 3.140)
+    return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0;
+  }
+
+  @Override
+  public int hashCode() {
+    return getHiveDecimal().hashCode();
+  }
+
+  /* (non-Javadoc)
+   * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the
+   * internal storage bytes and scale.
+   * @return
+   */
+  public byte[] getInternalStorage() {
+    return internalStorage;
+  }
+
+  /* (non-Javadoc)
+   * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the
+   * internal storage bytes and scale.
+   */
+  public int getScale() {
+    return scale;
+  }
+
+  public static
+  HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable,
+                                            int precision, int scale) {
+    if (writable == null) {
+      return null;
+    }
+
+    HiveDecimal dec =
+        HiveDecimal.enforcePrecisionScale(writable.getHiveDecimal(), precision,
+            scale);
+    return dec == null ? null : new HiveDecimalWritable(dec);
+  }
+}


Mime
View raw message