tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject tajo git commit: TAJO-1752: NameResolver cannot find nested records properly.
Date Wed, 12 Aug 2015 09:38:12 GMT
Repository: tajo
Updated Branches:
  refs/heads/master 5982cf1ff -> fd6a95180


TAJO-1752: NameResolver cannot find nested records properly.

Closes #686


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd6a9518
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd6a9518
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd6a9518

Branch: refs/heads/master
Commit: fd6a95180ecfa05df7cd0b1a8a957057c9788ad6
Parents: 5982cf1
Author: Jihoon Son <jihoonson@apache.org>
Authored: Wed Aug 12 18:37:57 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Wed Aug 12 18:37:57 2015 +0900

----------------------------------------------------------------------
 CHANGES                                                |  2 ++
 .../java/org/apache/tajo/catalog/NestedPathUtil.java   |  2 +-
 .../main/java/org/apache/tajo/util/StringUtils.java    |  3 ++-
 .../tajo/engine/query/TestSelectNestedRecord.java      | 10 ++++++++++
 .../TestSelectNestedRecord/sample2/sample2.json        |  1 +
 .../queries/TestSelectNestedRecord/sample2_ddl.sql     | 10 +++++++---
 .../queries/TestSelectNestedRecord/testSelect3.sql     |  1 +
 .../results/TestSelectNestedRecord/testSelect3.result  |  3 +++
 .../apache/tajo/plan/nameresolver/NameResolver.java    | 13 ++++++++-----
 9 files changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f572ebc..c7a8fde 100644
--- a/CHANGES
+++ b/CHANGES
@@ -216,6 +216,8 @@ Release 0.11.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1752: NameResolver cannot find nested records properly. (jihoon)
+
     TAJO-1732: During filter push down phase, join conditions are not set properly. 
     (jihoon)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
index dec9a55..9c750e8 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -79,7 +79,7 @@ public class NestedPathUtil {
     }
 
     if (found != null) {
-      if (found.getDataType().getType() == Type.RECORD) {
+      if (found.getDataType().getType() == Type.RECORD && paths.length > depth
+ 1) {
         return lookupColumnInternal(found, paths, depth + 1);
       } else {
         return found;

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 61dc855..7a9f0c3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -432,7 +432,8 @@ public class StringUtils {
   public static String join(Object[] objects, String delimiter, int startIndex, int length)
{
     boolean first = true;
     StringBuilder sb = new StringBuilder();
-    for(int i = startIndex; i + startIndex < length; i++) {
+    int endIndex = startIndex + length;
+    for(int i = startIndex; i + startIndex < endIndex; i++) {
       if (first) {
         first = false;
       } else {

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
index ffd6f08..41df0fa 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -60,6 +60,16 @@ public class TestSelectNestedRecord extends QueryTestCaseBase {
   }
 
   @Test
+  public final void testSelect3() throws Exception {
+    List<String> tables = executeDDL("sample2_ddl.sql", "sample2", "sample5");
+    assertEquals(TUtil.newList("sample5"), tables);
+
+    ResultSet res = executeQuery();
+    assertResultSet(res);
+    cleanupQuery(res);
+  }
+
+  @Test
   public final void testNestedFieldAsGroupbyKey1() throws Exception {
     List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
     assertEquals(TUtil.newList("tweets"), tables);

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
new file mode 100644
index 0000000..20bab62
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
@@ -0,0 +1 @@
+{ "glossary": { "title": "example glossary", "GlossDiv": { "title": "S", "GlossList": { "GlossEntry":
{ "ID": "SGML", "SortAs": "SGML", "GlossTerm": "Standard Generalized Markup Language", "Acronym":
"SGML", "Abbrev": "ISO 8879:1986", "GlossDef": { "para": "A meta-markup language, used to
create markup languages such as DocBook.", "GlossSeeAlso": ["GML", "XML"] }, "GlossSee": "markup"
} } } } }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
index 9537c3e..bc2d6e2 100644
--- a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
@@ -2,18 +2,22 @@ CREATE EXTERNAL TABLE ${0} (
   glossary RECORD (
     title TEXT,
     "GlossDiv" RECORD (
-      "GlossEntry" RECORD (
+      title TEXT,
+      "GlossList" RECORD (
+        "GlossEntry" RECORD (
         "ID" TEXT,
         "SortAs" TEXT,
         "GlossTerm" TEXT,
         "Acronym" TEXT,
         "Abbrev" TEXT,
         "GlossDef" RECORD (
-          para TEXT
+          para TEXT,
+          "GlossSeeAlso" TEXT
         ),
 
         "GlossSee" TEXT
+        )
       )
     )
   )
-) USING JSON LOCATION ${path};
\ No newline at end of file
+) USING JSON LOCATION ${table.path};
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
new file mode 100644
index 0000000..dfc79e9
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
@@ -0,0 +1 @@
+select glossary.title, glossary."GlossDiv"."title" from sample5;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
new file mode 100644
index 0000000..47e3fe5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
@@ -0,0 +1,3 @@
+glossary/title,glossary/GlossDiv/title
+-------------------------------
+example glossary,S
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index a6f3d35..c346ce9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -24,6 +24,7 @@ import com.google.common.collect.Maps;
 import org.apache.tajo.algebra.ColumnReferenceExpr;
 import org.apache.tajo.catalog.CatalogUtil;
 import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.NestedPathUtil;
 import org.apache.tajo.catalog.Schema;
 import org.apache.tajo.catalog.exception.AmbiguousTableException;
 import org.apache.tajo.catalog.exception.UndefinedColumnException;
@@ -346,8 +347,9 @@ public abstract class NameResolver {
     }
 
     // column.nested_fieldX...
-    if (guessedRelations.size() == 0 && qualifierParts.length == 1) {
-      Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]);
+    if (guessedRelations.size() == 0 && qualifierParts.length > 0) {
+      Collection<RelationNode> rels = lookupTableByColumns(block, StringUtils.join(qualifierParts,
+          NestedPathUtil.PATH_DELIMITER, 0));
 
       if (rels.size() > 1) {
         throw new AmbiguousColumnException(columnRef.getCanonicalName());
@@ -376,12 +378,13 @@ public abstract class NameResolver {
       columnName = qualifierParts[columnNamePosition];
 
       // if qualifierParts include nested field names
-      if (qualifierParts.length > columnNamePosition) {
-        columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length);
+      if (qualifierParts.length > columnNamePosition + 1) {
+        columnName += NestedPathUtil.PATH_DELIMITER + StringUtils.join(qualifierParts, NestedPathUtil.PATH_DELIMITER,
+            columnNamePosition + 1, qualifierParts.length);
       }
 
       // columnRef always has a leaf field name.
-      columnName += "/" + columnRef.getName();
+      columnName += NestedPathUtil.PATH_DELIMITER + columnRef.getName();
     }
 
     return new Pair<String, String>(qualifier, columnName);


Mime
View raw message