tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject git commit: TAJO-1081: Non-forwarded (simple) query shows wrong rows.
Date Thu, 02 Oct 2014 07:28:10 GMT
Repository: tajo
Updated Branches:
  refs/heads/master d1022fbb8 -> 3d029fa90


TAJO-1081: Non-forwarded (simple) query shows wrong rows.

Closes #175


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3d029fa9
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3d029fa9
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3d029fa9

Branch: refs/heads/master
Commit: 3d029fa90e483e6d0678b8b6c72c4a2410e97399
Parents: d1022fb
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Thu Oct 2 00:27:24 2014 -0700
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Thu Oct 2 00:27:24 2014 -0700

----------------------------------------------------------------------
 CHANGES                                         |  2 ++
 .../tajo/cli/DefaultTajoCliOutputFormatter.java | 33 ++++++++++++++------
 .../org/apache/tajo/cli/DescTableCommand.java   |  6 +++-
 .../java/org/apache/tajo/client/TajoClient.java |  4 ++-
 .../apache/tajo/engine/planner/PlannerUtil.java |  3 +-
 .../org/apache/tajo/master/GlobalEngine.java    |  6 ++++
 .../tajo/storage/AbstractStorageManager.java    |  2 +-
 7 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c73490c..3876a1e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -155,6 +155,8 @@ Release 0.9.0 - unreleased
 
   BUG FIXES
 
+    TAJO-1081: Non-forwarded (simple) query shows wrong rows. (hyunsik)
+
     TAJO-981: Help command (\?) in tsql takes too long time. (YeonSu Han via jaehwa)
 
     TAJO-962: Column reference used in LIMIT clause incurs NPE.

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
index 543ac8c..b36f521 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
@@ -24,6 +24,7 @@ import org.apache.tajo.SessionVars;
 import org.apache.tajo.catalog.TableDesc;
 import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.client.QueryStatus;
+import org.apache.tajo.client.TajoClient;
 import org.apache.tajo.util.FileUtil;
 
 import java.io.InputStream;
@@ -50,13 +51,25 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
     this.printPause = false;
   }
 
-  private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows,
String postfix) {
+  private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows,
String postfix,
+                                        boolean endOfTuple) {
     TableStats stat = tableDesc.getStats();
-    String volume = stat == null ? "0 B" : FileUtil.humanReadableByteCount(stat.getNumBytes(),
false);
-    long resultRows = stat == null ? 0 : stat.getNumRows();
+    String volume = stat == null ? "unknown bytes" : FileUtil.humanReadableByteCount(stat.getNumBytes(),
false);
+    long resultRows = stat == null ? TajoClient.UNKNOWN_ROW_NUMBER : stat.getNumRows();
 
-    long realNumRows = resultRows != 0 ? resultRows : totalPrintedRows;
-    return "(" + realNumRows + " rows, " + getResponseTimeReadable(responseTime) + ", " +
volume + " " + postfix + ")";
+    String displayRowNum;
+    if (resultRows == TajoClient.UNKNOWN_ROW_NUMBER) {
+
+      if (endOfTuple) {
+        displayRowNum = totalPrintedRows + " rows";
+      } else {
+        displayRowNum = "unknown row number";
+      }
+
+    } else {
+      displayRowNum = resultRows + " rows";
+    }
+    return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume
+ " " + postfix + ")";
   }
 
   protected String getResponseTimeReadable(float responseTime) {
@@ -66,13 +79,13 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
   @Override
   public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc,
                           float responseTime, ResultSet res) throws Exception {
-    long resultRows = tableDesc.getStats() == null ? 0 : tableDesc.getStats().getNumRows();
-    if (resultRows == 0) {
+    long resultRows = tableDesc.getStats() == null ? -1 : tableDesc.getStats().getNumRows();
+    if (resultRows == -1) {
       resultRows = Integer.MAX_VALUE;
     }
 
     if (res == null) {
-      sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted"));
+      sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted", true));
       return;
     }
     ResultSetMetaData rsmd = res.getMetaData();
@@ -86,6 +99,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
 
     int numOfPrintedRows = 0;
     int totalPrintedRows = 0;
+    boolean endOfTuple = true;
     while (res.next()) {
       for (int i = 1; i <= numOfColumns; i++) {
         if (i > 1) sout.print(",  ");
@@ -109,6 +123,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
         sout.flush();
         if (sin != null) {
           if (sin.read() == 'q') {
+            endOfTuple = false;
             sout.println();
             break;
           }
@@ -117,7 +132,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
         sout.println();
       }
     }
-    sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected"));
+    sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected",
endOfTuple));
     sout.flush();
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
index d8023f2..8fab138 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.tajo.catalog.Column;
 import org.apache.tajo.catalog.TableDesc;
 import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.client.TajoClient;
 import org.apache.tajo.util.FileUtil;
 import org.apache.tajo.util.TUtil;
 
@@ -79,7 +80,10 @@ public class DescTableCommand extends TajoShellCommand {
     sb.append("table path: ").append(desc.getPath()).append("\n");
     sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n");
     if (desc.getStats() != null) {
-      sb.append("number of rows: ").append(desc.getStats().getNumRows()).append("\n");
+
+      long row = desc.getStats().getNumRows();
+      String rowText = row == TajoClient.UNKNOWN_ROW_NUMBER ? "unknown" : row + "";
+      sb.append("number of rows: ").append(rowText).append("\n");
       sb.append("volume: ").append(
           FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(),
               true)).append("\n");

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
index 3d61cce..2288851 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
@@ -73,6 +73,8 @@ import static org.apache.tajo.ipc.ClientProtos.SerializedResultSet;
 
 @ThreadSafe
 public class TajoClient implements Closeable {
+  public static final int UNKNOWN_ROW_NUMBER = -1;
+
   private final Log LOG = LogFactory.getLog(TajoClient.class);
 
   private final TajoConf conf;
@@ -413,7 +415,7 @@ public class TajoClient implements Closeable {
       }
     } else {
       // If a non-forwarded insert into query
-      if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum()
< 0) {
+      if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum()
== 0) {
         return this.createNullResultSet(queryId);
       } else {
         if (response.hasResultSet() || response.hasTableDesc()) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index c4cc254..827be83 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -38,6 +38,7 @@ import org.apache.tajo.engine.eval.*;
 import org.apache.tajo.engine.exception.InvalidQueryException;
 import org.apache.tajo.engine.planner.logical.*;
 import org.apache.tajo.engine.utils.SchemaUtil;
+import org.apache.tajo.storage.StorageManager;
 import org.apache.tajo.storage.TupleComparator;
 import org.apache.tajo.storage.fragment.FileFragment;
 import org.apache.tajo.storage.fragment.FragmentConvertor;
@@ -854,7 +855,7 @@ public class PlannerUtil {
                                          int startFileIndex, int numResultFiles,
                                          AtomicInteger currentFileIndex) throws IOException
{
     if (fs.isDirectory(path)) {
-      FileStatus[] files = fs.listStatus(path);
+      FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter);
       if (files != null && files.length > 0) {
         for (FileStatus eachFile : files) {
           if (result.size() >= numResultFiles) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 9612eeb..1537c89 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -39,6 +39,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
 import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.client.TajoClient;
 import org.apache.tajo.common.TajoDataTypes;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.datum.DatumFactory;
@@ -750,6 +751,11 @@ public class GlobalEngine extends AbstractService {
 
     TableStats stats = new TableStats();
     stats.setNumBytes(totalSize);
+
+    if (isExternal) { // if it is an external table, there is no way to know the exact row
number without processing.
+      stats.setNumRows(TajoClient.UNKNOWN_ROW_NUMBER);
+    }
+
     TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName),
         schema, meta, path, isExternal);
     desc.setStats(stats);

http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
index c6be73b..4bf0057 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
@@ -304,7 +304,7 @@ public abstract class AbstractStorageManager {
   // FileInputFormat Area
   /////////////////////////////////////////////////////////////////////////////
 
-  private static final PathFilter hiddenFileFilter = new PathFilter() {
+  public static final PathFilter hiddenFileFilter = new PathFilter() {
     public boolean accept(Path p) {
       String name = p.getName();
       return !name.startsWith("_") && !name.startsWith(".");


Mime
View raw message