drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [3/3] drill git commit: DRILL-5293: Change seed for distribution hash function to differ from that of the hash table
Date Fri, 03 Mar 2017 22:10:23 GMT
DRILL-5293: Change seed for distribution hash function to differ from that of the hash table

close #765


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3dfb4972
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3dfb4972
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3dfb4972

Branch: refs/heads/master
Commit: 3dfb497293a177164a158b246000ee8291ef258c
Parents: f0057e7
Author: Boaz Ben-Zvi <boazben-zvi@BBenZvi-E754-MBP13.local>
Authored: Fri Feb 24 16:48:42 2017 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Fri Mar 3 10:20:37 2017 -0800

----------------------------------------------------------------------
 .../drill/exec/planner/physical/HashPrelUtil.java     | 14 ++++++++++----
 .../physical/visitor/InsertLocalExchangeVisitor.java  |  6 +++---
 2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
index caf21bc..4300c82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
@@ -24,6 +24,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.ValueExpressions;
 import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
 
 import java.util.ArrayList;
@@ -36,6 +37,7 @@ public class HashPrelUtil {
 
   public static final String HASH_EXPR_NAME = "E_X_P_R_H_A_S_H_F_I_E_L_D";
 
+  public static final int DIST_SEED = 1301011; // distribution seed
   /**
    * Interface for creating different forms of hash expression types.
    * @param <T>
@@ -72,8 +74,9 @@ public class HashPrelUtil {
    */
   public static <T> T createHashBasedPartitionExpression(
       List<T> distFields,
+      T seed,
       HashExpressionCreatorHelper<T> helper) {
-    return createHashExpression(distFields, helper, true /*for distribution always hash as
double*/);
+    return createHashExpression(distFields, seed, helper, true /*for distribution always
hash as double*/);
   }
 
   /**
@@ -89,6 +92,7 @@ public class HashPrelUtil {
    */
   public static <T> T createHashExpression(
       List<T> inputExprs,
+      T seed,
       HashExpressionCreatorHelper<T> helper,
       boolean hashAsDouble) {
 
@@ -96,7 +100,7 @@ public class HashPrelUtil {
 
     final String functionName = hashAsDouble ? HASH32_DOUBLE_FUNCTION_NAME : HASH32_FUNCTION_NAME;
 
-    T func = helper.createCall(functionName,  ImmutableList.of(inputExprs.get(0)));
+    T func = helper.createCall(functionName,  ImmutableList.of(inputExprs.get(0), seed ));
     for (int i = 1; i<inputExprs.size(); i++) {
       func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(i), func));
     }
@@ -108,7 +112,8 @@ public class HashPrelUtil {
    * Return a hash expression :  hash32(field1, hash32(field2, hash32(field3, 0)));
    */
   public static LogicalExpression getHashExpression(List<LogicalExpression> fields,
boolean hashAsDouble){
-    return createHashExpression(fields, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
+    final LogicalExpression seed = ValueExpressions.getInt(0); // Hash Table seed
+    return createHashExpression(fields, seed, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
   }
 
 
@@ -134,6 +139,7 @@ public class HashPrelUtil {
       expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN));
     }
 
-    return createHashBasedPartitionExpression(expressions, HASH_HELPER_LOGICALEXPRESSION);
+    final LogicalExpression distSeed = ValueExpressions.getInt(DIST_SEED);
+    return createHashBasedPartitionExpression(expressions, distSeed, HASH_HELPER_LOGICALEXPRESSION);
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
index 6ebdba4..fba4cbe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
@@ -19,14 +19,12 @@ package org.apache.drill.exec.planner.physical.visitor;
 
 import com.google.common.collect.Lists;
 
-import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.exec.planner.physical.ExchangePrel;
 import org.apache.drill.exec.planner.physical.HashPrelUtil;
 import org.apache.drill.exec.planner.physical.HashPrelUtil.HashExpressionCreatorHelper;
 import org.apache.drill.exec.planner.physical.HashToRandomExchangePrel;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
 import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.drill.exec.planner.physical.ProjectPrel;
 import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
 import org.apache.drill.exec.planner.physical.UnorderedDeMuxExchangePrel;
@@ -40,6 +38,7 @@ import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
 
+import java.math.BigDecimal;
 import java.util.Collections;
 import java.util.List;
 
@@ -119,7 +118,8 @@ public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel,
Void, Runt
       }
 
       outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
-      updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, hashHelper));
+      final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED));
// distribution seed
+      updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed,
hashHelper));
 
       RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(),
updatedExpr, outputFieldNames);
 


Mime
View raw message