tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject git commit: TAJO-706: In the case of very quick query, client can't get query status. (hyoungjunkim via hyunsik)
Date Sun, 30 Mar 2014 14:59:25 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.8.0 8c2703c2d -> c7e361676


TAJO-706: In the case of very quick query, client can't get query status. (hyoungjunkim via
hyunsik)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/c7e36167
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/c7e36167
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/c7e36167

Branch: refs/heads/branch-0.8.0
Commit: c7e361676470e823b2e2718f98bfd05ab35ea992
Parents: 8c2703c
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Sun Mar 30 22:16:28 2014 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Sun Mar 30 23:53:22 2014 +0900

----------------------------------------------------------------------
 .../java/org/apache/tajo/client/TajoDump.java   | 20 ++++---
 .../tajo/master/TajoMasterClientService.java    | 26 +++++++--
 .../master/querymaster/QueryJobManager.java     |  6 +++
 .../org/apache/tajo/client/TestTajoClient.java  | 56 ++++++++++++++++++++
 4 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java
index 224de45..924a65d 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoDump.java
@@ -150,14 +150,20 @@ public class TajoDump {
     List<String> tableNames = TUtil.newList(client.getTableList(databaseName));
     Collections.sort(tableNames);
     for (String tableName : tableNames) {
-      TableDesc table =
-          client.getTableDesc(CatalogUtil.denormalizeIdentifier(CatalogUtil.buildFQName(databaseName,tableName)));
-      if (table.isExternal()) {
-        writer.write(DDLBuilder.buildDDLForExternalTable(table));
-      } else {
-        writer.write(DDLBuilder.buildDDLForBaseTable(table));
+      try {
+        TableDesc table =
+            client.getTableDesc(CatalogUtil.denormalizeIdentifier(CatalogUtil.buildFQName(databaseName,tableName)));
+        if (table.isExternal()) {
+          writer.write(DDLBuilder.buildDDLForExternalTable(table));
+        } else {
+          writer.write(DDLBuilder.buildDDLForBaseTable(table));
+        }
+        writer.write("\n\n");
+      } catch (Exception e) {
+        // dump for each table can throw any exception. We need to skip the exception case.
+        // here, the error message prints out via stderr.
+        System.err.println("ERROR:" + tableName + "," + e.getMessage());
       }
-      writer.write("\n\n");
     }
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 924be95..42aba7c 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -328,9 +328,24 @@ public class TajoMasterClientService extends AbstractService {
         context.getSessionManager().touch(request.getSessionId().getId());
         QueryId queryId = new QueryId(request.getQueryId());
         QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId);
+
+        // if we cannot get a QueryInProgress instance from QueryJobManager,
+        // the instance can be in the finished query list.
+        if (queryInProgress == null) {
+          queryInProgress = context.getQueryJobManager().getFinishedQuery(queryId);
+        }
+
+        GetQueryResultResponse.Builder builder = GetQueryResultResponse.newBuilder();
+
+        // If we cannot the QueryInProgress instance from the finished list,
+        // the query result was expired due to timeout.
+        // In this case, we will result in error.
+        if (queryInProgress == null) {
+          builder.setErrorMessage("No such query: " + queryId.toString());
+          return builder.build();
+        }
+
         QueryInfo queryInfo = queryInProgress.getQueryInfo();
-        GetQueryResultResponse.Builder builder
-            = GetQueryResultResponse.newBuilder();
 
         try {
           //TODO After implementation Tajo's user security feature, Should be modified.
@@ -448,6 +463,11 @@ public class TajoMasterClientService extends AbstractService {
           builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED);
         } else {
           QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId);
+
+          // It will try to find a query status from a finished query list.
+          if (queryInProgress == null) {
+            queryInProgress = context.getQueryJobManager().getFinishedQuery(queryId);
+          }
           if (queryInProgress != null) {
             QueryInfo queryInfo = queryInProgress.getQueryInfo();
             builder.setResultCode(ResultCode.OK);
@@ -458,8 +478,6 @@ public class TajoMasterClientService extends AbstractService {
               builder.setQueryMasterHost(queryInfo.getQueryMasterHost());
               builder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
             }
-            //builder.setInitTime(queryJobManager.getInitializationTime());
-            //builder.setHasResult(!queryJobManager.isCreateTableStmt());
             if (queryInfo.getQueryState() == TajoProtos.QueryState.QUERY_SUCCEEDED) {
               builder.setFinishTime(queryInfo.getFinishTime());
             } else {

http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
index 0c28c52..97513a0 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
@@ -137,6 +137,12 @@ public class QueryJobManager extends CompositeService {
     }
   }
 
+  public QueryInProgress getFinishedQuery(QueryId queryId) {
+    synchronized(finishedQueries) {
+      return finishedQueries.get(queryId);
+    }
+  }
+
   public void stopQuery(QueryId queryId) {
     LOG.info("Stop QueryInProgress:" + queryId);
     QueryInProgress queryInProgress = getQueryInProgress(queryId);

http://git-wip-us.apache.org/repos/asf/tajo/blob/c7e36167/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index c8b11a8..5eb8af2 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -33,12 +33,14 @@ import org.apache.tajo.catalog.TableDesc;
 import org.apache.tajo.catalog.proto.CatalogProtos;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.ipc.ClientProtos;
+import org.apache.tajo.jdbc.TajoResultSet;
 import org.apache.tajo.storage.StorageUtil;
 import org.apache.tajo.util.CommonTestingUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.Timeout;
 
 import java.io.IOException;
 import java.sql.ResultSet;
@@ -608,4 +610,58 @@ public class TestTajoClient {
 
     resultSet.close();
   }
+
+  /**
+   * The main objective of this test is to get the status of a query which is actually finished.
+   * Statuses of queries regardless of its status should be available for a specified time
duration.
+   */
+  @Test(timeout = 20 * 1000)
+  public final void testGetQueryStatusAndResultAfterFinish() throws Exception {
+    String sql = "select * from lineitem";
+    ClientProtos.GetQueryStatusResponse response = client.executeQuery(sql);
+
+    assertNotNull(response);
+    QueryId queryId = new QueryId(response.getQueryId());
+
+    try {
+      long startTime = System.currentTimeMillis();
+      while (true) {
+        Thread.sleep(5 * 1000);
+
+        List<ClientProtos.BriefQueryInfo> finishedQueries = client.getFinishedQueryList();
+        boolean finished = false;
+        if (finishedQueries != null) {
+          for (ClientProtos.BriefQueryInfo eachQuery: finishedQueries) {
+            if (eachQuery.getQueryId().equals(queryId.getProto())) {
+              finished = true;
+              break;
+            }
+          }
+        }
+
+        if (finished) {
+          break;
+        }
+        if(System.currentTimeMillis() - startTime > 20 * 1000) {
+          fail("Too long time execution query");
+        }
+      }
+
+      QueryStatus queryStatus = client.getQueryStatus(queryId);
+      assertNotNull(queryStatus);
+      assertTrue(!TajoClient.isQueryRunnning(queryStatus.getState()));
+
+      TajoResultSet resultSet = (TajoResultSet) client.getQueryResult(queryId);
+      assertNotNull(resultSet);
+
+      int count = 0;
+      while(resultSet.next()) {
+        count++;
+      }
+
+      assertEquals(5, count);
+    } finally {
+      client.closeQuery(queryId);
+    }
+  }
 }


Mime
View raw message