hive-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Gopal V (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HIVE-9065) Multiple join Predicate Push Down error
Date Wed, 10 Dec 2014 09:13:12 GMT

     [ https://issues.apache.org/jira/browse/HIVE-9065?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Gopal V updated HIVE-9065:
--------------------------
    Description: 
Example sql
SELECT count(*)
    FROM
    tmp.ls_cardlastsetl_tmp a 
    INNER JOIN tmp.ls_distinct_nbkentrycard b ON a.cardid=b.cswcardid 
    INNER JOIN default.s_tbl_bank d ON b.bankid=d.id 
    INNER JOIN tmp.ls_cardlastmod_tmp e ON b.cswcardid=e.cardid
WHERE to_date(b.cardcreatetime)<"2014-11-20" AND to_date(from_unixtime(e.lastmodtime))>"2014-11-30"
;

plan:
{code}
FS
   SEL
      GBY
         RS
            GBY
               SEL
                     JOIN
                        RS
                           JOIN
                              RS
                                 TS[a]
                              RS
                                 FIL
                                    TS[b]
                              RS
                                 TS[e]
                        RS
                           TS[d]
{code}

Table e of FIL be lost
Reason:
When e into a, b, leftAlias not updated
Fix patch:
 
Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
===================================================================
--- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(revision 85984)
+++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(working copy)
@@ -363,4 +363,22 @@
   public List<ASTNode> getPostJoinFilters() {
     return postJoinFilters;
   }
+  /**
+   * after merge join tree,update alias
+  * @Title: updateAlias
+  * @Description: TODO  
+  * @return void     
+  * @throws
+   */
+  public void updateAlias(){
+	  if(this.joinSrc == null){
+		  return ;
+	  }
+	  joinSrc.updateAlias();
+	  String[] leftAlias = new String[joinSrc.getLeftAliases().length+joinSrc.getRightAliases().length];
+	  System.arraycopy(joinSrc.getLeftAliases(), 0, leftAlias, 0, joinSrc.getLeftAliases().length);
+	  System.arraycopy(joinSrc.getRightAliases(), 0, leftAlias, joinSrc.getLeftAliases().length,
joinSrc.getRightAliases().length);
+	  this.leftAliases = leftAlias;
+  }
+  
 }
Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
===================================================================
--- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(revision 85984)
+++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(working copy)
@@ -8938,6 +8938,7 @@
           }
         }
         mergeJoinTree(qb);
+        qb.getQbJoinTree().updateAlias();
       }
 
       // if any filters are present in the join tree, push them on top of the


  was:
Example sql
SELECT count(*)
    FROM
    tmp.ls_cardlastsetl_tmp a 
    INNER JOIN tmp.ls_distinct_nbkentrycard b ON a.cardid=b.cswcardid 
    INNER JOIN default.s_tbl_bank d ON b.bankid=d.id 
    INNER JOIN tmp.ls_cardlastmod_tmp e ON b.cswcardid=e.cardid
WHERE to_date(b.cardcreatetime)<"2014-11-20" AND to_date(from_unixtime(e.lastmodtime))>"2014-11-30"
;

plan:
FS
   SEL
      GBY
         RS
            GBY
               SEL
                     JOIN
                        RS
                           JOIN
                              RS
                                 TS[a]
                              RS
                                 FIL
                                    TS[b]
                              RS
                                 TS[e]
                        RS
                           TS[d]
Table e of FIL be lost
Reason:
When e into a, b, leftAlias not updated
Fix patch:
 
Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
===================================================================
--- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(revision 85984)
+++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(working copy)
@@ -363,4 +363,22 @@
   public List<ASTNode> getPostJoinFilters() {
     return postJoinFilters;
   }
+  /**
+   * after merge join tree,update alias
+  * @Title: updateAlias
+  * @Description: TODO  
+  * @return void     
+  * @throws
+   */
+  public void updateAlias(){
+	  if(this.joinSrc == null){
+		  return ;
+	  }
+	  joinSrc.updateAlias();
+	  String[] leftAlias = new String[joinSrc.getLeftAliases().length+joinSrc.getRightAliases().length];
+	  System.arraycopy(joinSrc.getLeftAliases(), 0, leftAlias, 0, joinSrc.getLeftAliases().length);
+	  System.arraycopy(joinSrc.getRightAliases(), 0, leftAlias, joinSrc.getLeftAliases().length,
joinSrc.getRightAliases().length);
+	  this.leftAliases = leftAlias;
+  }
+  
 }
Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
===================================================================
--- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(revision 85984)
+++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(working copy)
@@ -8938,6 +8938,7 @@
           }
         }
         mergeJoinTree(qb);
+        qb.getQbJoinTree().updateAlias();
       }
 
       // if any filters are present in the join tree, push them on top of the



> Multiple join Predicate Push Down error
> ---------------------------------------
>
>                 Key: HIVE-9065
>                 URL: https://issues.apache.org/jira/browse/HIVE-9065
>             Project: Hive
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 0.13.0
>         Environment: hadoop2.5  jdk1.7
>            Reporter: yuebing.fu
>              Labels: patch
>             Fix For: 0.13.0
>
>   Original Estimate: 0.4h
>  Remaining Estimate: 0.4h
>
> Example sql
> SELECT count(*)
>     FROM
>     tmp.ls_cardlastsetl_tmp a 
>     INNER JOIN tmp.ls_distinct_nbkentrycard b ON a.cardid=b.cswcardid 
>     INNER JOIN default.s_tbl_bank d ON b.bankid=d.id 
>     INNER JOIN tmp.ls_cardlastmod_tmp e ON b.cswcardid=e.cardid
> WHERE to_date(b.cardcreatetime)<"2014-11-20" AND to_date(from_unixtime(e.lastmodtime))>"2014-11-30"
;
> plan:
> {code}
> FS
>    SEL
>       GBY
>          RS
>             GBY
>                SEL
>                      JOIN
>                         RS
>                            JOIN
>                               RS
>                                  TS[a]
>                               RS
>                                  FIL
>                                     TS[b]
>                               RS
>                                  TS[e]
>                         RS
>                            TS[d]
> {code}
> Table e of FIL be lost
> Reason:
> When e into a, b, leftAlias not updated
> Fix patch:
>  
> Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
> ===================================================================
> --- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(revision 85984)
> +++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
(working copy)
> @@ -363,4 +363,22 @@
>    public List<ASTNode> getPostJoinFilters() {
>      return postJoinFilters;
>    }
> +  /**
> +   * after merge join tree,update alias
> +  * @Title: updateAlias
> +  * @Description: TODO  
> +  * @return void     
> +  * @throws
> +   */
> +  public void updateAlias(){
> +	  if(this.joinSrc == null){
> +		  return ;
> +	  }
> +	  joinSrc.updateAlias();
> +	  String[] leftAlias = new String[joinSrc.getLeftAliases().length+joinSrc.getRightAliases().length];
> +	  System.arraycopy(joinSrc.getLeftAliases(), 0, leftAlias, 0, joinSrc.getLeftAliases().length);
> +	  System.arraycopy(joinSrc.getRightAliases(), 0, leftAlias, joinSrc.getLeftAliases().length,
joinSrc.getRightAliases().length);
> +	  this.leftAliases = leftAlias;
> +  }
> +  
>  }
> Index: /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
> ===================================================================
> --- /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(revision 85984)
> +++ /BI/dp/hive-0.13.1-cdh5.2.0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(working copy)
> @@ -8938,6 +8938,7 @@
>            }
>          }
>          mergeJoinTree(qb);
> +        qb.getQbJoinTree().updateAlias();
>        }
>  
>        // if any filters are present in the join tree, push them on top of the



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message