tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sir...@apache.org
Subject tajo git commit: TAJO-1283: ORDER BY with the first descending order causes wrong results. (Keuntae Park)
Date Tue, 03 Feb 2015 07:59:50 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.10.0 9ad117f07 -> 02c6c266c


TAJO-1283: ORDER BY with the first descending order causes wrong results. (Keuntae Park)

Closes #364


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/02c6c266
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/02c6c266
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/02c6c266

Branch: refs/heads/branch-0.10.0
Commit: 02c6c266c013d8174a287bc57a6d4131da51ba96
Parents: 9ad117f
Author: Keuntae Park <sirpkt@apache.org>
Authored: Tue Feb 3 16:57:13 2015 +0900
Committer: Keuntae Park <sirpkt@apache.org>
Committed: Tue Feb 3 16:57:13 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  3 ++
 .../apache/tajo/querymaster/Repartitioner.java  |  9 ++----
 .../java/org/apache/tajo/querymaster/Stage.java |  3 +-
 .../apache/tajo/engine/query/TestSortQuery.java | 34 ++++++++++++++++++++
 .../queries/TestSortQuery/testSortFirstDesc.sql |  1 +
 .../TestSortQuery/testSortFirstDesc.result      | 10 ++++++
 .../org/apache/tajo/storage/TupleRange.java     |  9 ------
 7 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 47eaf9c..5fe46e8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -176,6 +176,9 @@ Release 0.10.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1283: ORDER BY with the first descending order causes wrong results.
+    (Keuntae Park)
+
     TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho)
 
     TAJO-1321: Cli prints wrong response time. (jihoon)

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index 5a35674..7c2d9f4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -725,13 +725,8 @@ public class Repartitioner {
       }
     }
 
-    boolean ascendingFirstKey = sortSpecs[0].isAscending();
     SortedMap<TupleRange, Collection<FetchImpl>> map;
-    if (ascendingFirstKey) {
-      map = new TreeMap<TupleRange, Collection<FetchImpl>>();
-    } else {
-      map = new TreeMap<TupleRange, Collection<FetchImpl>>(new TupleRange.DescendingTupleRangeComparator());
-    }
+    map = new TreeMap<TupleRange, Collection<FetchImpl>>();
 
     Set<FetchImpl> fetchSet;
     try {
@@ -740,7 +735,7 @@ public class Repartitioner {
         fetchSet = new HashSet<FetchImpl>();
         for (FetchImpl fetch: fetches) {
           String rangeParam =
-              TupleUtil.rangeToQuery(ranges[i], ascendingFirstKey ? i == (ranges.length -
1) : i == 0, encoder);
+              TupleUtil.rangeToQuery(ranges[i], i == (ranges.length - 1) , encoder);
           FetchImpl copy = null;
           try {
             copy = fetch.clone();

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 5673d5b..efadaa7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -1007,7 +1007,8 @@ public class Stage implements EventHandler<StageEvent> {
       int mb = (int) Math.ceil((double)volume / 1048576);
       LOG.info(stage.getId() + ", Table's volume is approximately " + mb + " MB");
       // determine the number of task per 64MB
-      int maxTaskNum = Math.max(1, (int) Math.ceil((double)mb / 64));
+      int minTaskNum = Math.max(1, stage.getContext().getQueryMasterContext().getConf().getInt(ConfVars.$TEST_MIN_TASK_NUM.varname,
1));
+      int maxTaskNum = Math.max(minTaskNum, (int) Math.ceil((double)mb / 64));
       LOG.info(stage.getId() + ", The determined number of non-leaf tasks is " + maxTaskNum);
       return maxTaskNum;
     }

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 9c911f9..ff91177 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -102,6 +102,40 @@ public class TestSortQuery extends QueryTestCaseBase {
     cleanupQuery(res);
   }
 
+
+  @Test
+  public final void testSortFirstDesc() throws Exception {
+    try {
+      testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
+      KeyValueSet tableOptions = new KeyValueSet();
+      tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+      tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N");
+
+      Schema schema = new Schema();
+      schema.addColumn("col1", Type.INT4);
+      schema.addColumn("col2", Type.TEXT);
+      String[] data = new String[]{
+          "1|abc",
+          "3|dfa",
+          "3|das",
+          "1|abb",
+          "1|abc",
+          "3|dfb",
+          "3|dat",
+          "1|abe"
+      };
+      TajoTestingCluster.createTable("sortfirstdesc", schema, tableOptions, data, 2);
+
+      ResultSet res = executeQuery();
+      assertResultSet(res);
+      cleanupQuery(res);
+    } finally {
+      testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0");
+      executeString("DROP TABLE sortfirstdesc PURGE;").close();
+    }
+  }
+
+
   @Test
   public final void testTopK() throws Exception {
     ResultSet res = executeQuery();

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
new file mode 100644
index 0000000..644feb0
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
@@ -0,0 +1 @@
+select col1, col2 from sortfirstdesc order by col1 desc, col2;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
new file mode 100644
index 0000000..2ed9235
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
@@ -0,0 +1,10 @@
+col1,col2
+-------------------------------
+3,das
+3,dat
+3,dfa
+3,dfb
+1,abb
+1,abc
+1,abc
+1,abe
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
index e824b99..043409a 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
@@ -94,15 +94,6 @@ public class TupleRange implements Comparable<TupleRange>, Cloneable
{
     }
   }
 
-  public static class DescendingTupleRangeComparator
-      implements Comparator<TupleRange> {
-
-    @Override
-    public int compare(TupleRange left, TupleRange right) {
-      return right.compareTo(left);
-    }
-  }
-
   public TupleRange clone() throws CloneNotSupportedException {
     TupleRange newRange = (TupleRange) super.clone();
     newRange.setStart(start.clone());


Mime
View raw message