tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject git commit: TAJO-914: join queries with constant values can cause schema mismatch in logical plan.
Date Wed, 23 Jul 2014 09:27:50 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 45559ce60 -> 326be451d


TAJO-914: join queries with constant values can cause schema mismatch in logical plan.

Closes #78


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/326be451
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/326be451
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/326be451

Branch: refs/heads/master
Commit: 326be451ded71099b88bdb6a50c40946df0af206
Parents: 45559ce
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Wed Jul 23 18:26:15 2014 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Wed Jul 23 18:26:15 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |  6 ++++
 .../apache/tajo/engine/utils/SchemaUtil.java    | 20 +++++++++--
 .../tajo/engine/query/TestCaseByCases.java      | 38 ++++++++++++++++++--
 .../TestCaseByCases/testTAJO914Case1.sql        | 10 ++++++
 .../TestCaseByCases/testTAJO914Case2.sql        |  9 +++++
 .../TestCaseByCases/testTAJO914Case3.sql        | 10 ++++++
 .../TestCaseByCases/testTAJO914Case4.sql        | 10 ++++++
 .../TestCaseByCases/testTAJO914Case1.result     |  5 +++
 .../TestCaseByCases/testTAJO914Case2.result     |  5 +++
 .../TestCaseByCases/testTAJO914Case3.result     |  5 +++
 .../TestCaseByCases/testTAJO914Case4.result     |  5 +++
 11 files changed, 118 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2be9b26..a67625d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -97,6 +97,12 @@ Release 0.9.0 - unreleased
 
   BUG FIXES
 
+    TAJO-914: join queries with constant values can cause schema mismatch in
+    logical plan. (hyunsik)
+
+    TAJO-969: Distributed sort on a large data set may result in incorrect
+    results. (hyunsik)
+
     TAJO-972: Broadcast join with left outer join returns duplicated rows.
     (Hyoungjun Kim via jaehwa)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java
index c882607..981b572 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java
@@ -23,6 +23,16 @@ import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.TableDesc;
 
 public class SchemaUtil {
+  // See TAJO-914 bug.
+  //
+  // Its essential problem is that constant value is evaluated multiple times at each scan.
+  // As a result, join nodes can take the child nodes which have the same named fields.
+  // Because current schema does not allow the same name and ignore the duplicated schema,
+  // it finally causes the in-out schema mismatch between the parent and child nodes.
+  //
+  // tmpColumnSeq is a hack to avoid the above problem by keeping duplicated constant values
as different name fields.
+  // The essential solution would be https://issues.apache.org/jira/browse/TAJO-895.
+  static int tmpColumnSeq = 0;
   public static Schema merge(Schema left, Schema right) {
     Schema merged = new Schema();
     for(Column col : left.getColumns()) {
@@ -31,11 +41,17 @@ public class SchemaUtil {
       }
     }
     for(Column col : right.getColumns()) {
-      if (!merged.containsByQualifiedName(col.getQualifiedName())) {
+      if (merged.containsByQualifiedName(col.getQualifiedName())) {
+        merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType());
+      } else {
         merged.addColumn(col);
       }
     }
-    
+
+    // if overflow
+    if (tmpColumnSeq < 0) {
+      tmpColumnSeq = 0;
+    }
     return merged;
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
index 459a4c1..846c290 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java
@@ -71,7 +71,7 @@ public class TestCaseByCases extends QueryTestCaseBase {
   }
 
   @Test
-  public final void testTAJO880_1() throws Exception {
+  public final void testTAJO880Case1() throws Exception {
     //TAJO-880: NULL in CASE clause occurs Exception.
     ResultSet res = executeString(
         "select case when l_returnflag != 'R' then l_orderkey else null end from lineitem"
@@ -91,7 +91,7 @@ public class TestCaseByCases extends QueryTestCaseBase {
   }
 
   @Test
-  public final void testTAJO880_2() throws Exception {
+  public final void testTAJO880Case2() throws Exception {
     //TAJO-880: NULL in CASE clause occurs Exception.
     ResultSet res = executeString(
         "select case when l_returnflag != 'R' then null else l_orderkey end from lineitem"
@@ -111,7 +111,7 @@ public class TestCaseByCases extends QueryTestCaseBase {
   }
 
   @Test
-  public final void testTAJO880_3() throws Exception {
+  public final void testTAJO880Case3() throws Exception {
     //TAJO-880: NULL in CASE clause occurs Exception.
     ResultSet res = executeString(
         "select case " +
@@ -135,6 +135,38 @@ public class TestCaseByCases extends QueryTestCaseBase {
   }
 
   @Test
+  public final void testTAJO914Case1() throws Exception {
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
+  public final void testTAJO914Case2() throws Exception {
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
+  public final void testTAJO914Case3() throws Exception {
+    executeString("CREATE TABLE T3 (l_orderkey bigint, col1 text);").close();
+    ResultSet res = executeQuery();
+    res.close();
+
+    res = executeString("select * from T3;");
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
+  public final void testTAJO914Case4() throws Exception {
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
   public final void testTAJO917Case1() throws Exception {
     ResultSet res = executeQuery();
     assertResultSet(res);

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case1.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case1.sql
new file mode 100644
index 0000000..765110a
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case1.sql
@@ -0,0 +1,10 @@
+select
+  l_orderkey,
+  '##' as col1
+from
+  lineitem
+  join orders on l_orderkey = o_orderkey
+group by
+  l_orderkey
+order by
+  l_orderkey;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case2.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case2.sql
new file mode 100644
index 0000000..f404088
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case2.sql
@@ -0,0 +1,9 @@
+select
+  l_orderkey,
+  '##' as col1
+from
+  lineitem join orders on l_orderkey = o_orderkey
+group by
+  l_orderkey, col1
+order by
+  l_orderkey;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case3.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case3.sql
new file mode 100644
index 0000000..f9ee355
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case3.sql
@@ -0,0 +1,10 @@
+insert OVERWRITE into T3
+  select
+    l_orderkey,
+    '##' as col1
+  from
+    lineitem join orders on l_orderkey = o_orderkey
+  group by
+    l_orderkey, col1
+  order by
+    l_orderkey;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case4.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case4.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case4.sql
new file mode 100644
index 0000000..ba99b70
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO914Case4.sql
@@ -0,0 +1,10 @@
+  select
+    l_orderkey,
+    '##' as col1,
+    sum(l_orderkey) as s1
+  from
+    lineitem join orders o1 on l_orderkey = o1.o_orderkey join orders o2 on l_orderkey =
o2.o_orderkey
+  group by
+    l_orderkey, col1
+  order by
+    l_orderkey;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case1.result
b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case1.result
new file mode 100644
index 0000000..01e467c
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case1.result
@@ -0,0 +1,5 @@
+l_orderkey,col1
+-------------------------------
+1,##
+2,##
+3,##
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case2.result
b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case2.result
new file mode 100644
index 0000000..01e467c
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case2.result
@@ -0,0 +1,5 @@
+l_orderkey,col1
+-------------------------------
+1,##
+2,##
+3,##
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case3.result
b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case3.result
new file mode 100644
index 0000000..01e467c
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case3.result
@@ -0,0 +1,5 @@
+l_orderkey,col1
+-------------------------------
+1,##
+2,##
+3,##
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/326be451/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case4.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case4.result
b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case4.result
new file mode 100644
index 0000000..d3b79d1
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO914Case4.result
@@ -0,0 +1,5 @@
+l_orderkey,col1,s1
+-------------------------------
+1,##,2
+2,##,2
+3,##,6
\ No newline at end of file


Mime
View raw message