quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianq...@apache.org
Subject incubator-quickstep git commit: Fixed the command execution bug in the distributed version.
Date Mon, 04 Jun 2018 21:37:58 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 8b7c38010 -> 835281ae8


Fixed the command execution bug in the distributed version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/835281ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/835281ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/835281ae

Branch: refs/heads/master
Commit: 835281ae894eadfdd121e444313bbf001e9b884b
Parents: 8b7c380
Author: Zuyu Zhang <zuyu@cs.wisc.edu>
Authored: Mon May 21 16:39:42 2018 -0500
Committer: Zuyu Zhang <zuyu@cs.wisc.edu>
Committed: Mon May 21 16:47:52 2018 -0500

----------------------------------------------------------------------
 cli/tests/CMakeLists.txt                        |  8 +-
 cli/tests/CommandExecutorTestRunner.cpp         |  2 +-
 cli/tests/DistributedCommandExecutorTest.cpp    |  2 -
 .../DistributedCommandExecutorTestRunner.cpp    | 87 +++++++-------------
 .../DistributedCommandExecutorTestRunner.hpp    | 15 +---
 cli/tests/command_executor/CMakeLists.txt       |  6 ++
 cli/tests/command_executor/D.test               | 25 ++----
 cli/tests/command_executor/Dt.test              |  8 ++
 8 files changed, 60 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/tests/CMakeLists.txt b/cli/tests/CMakeLists.txt
index c450e4d..18d321d 100644
--- a/cli/tests/CMakeLists.txt
+++ b/cli/tests/CMakeLists.txt
@@ -76,8 +76,9 @@ if (ENABLE_DISTRIBUTED)
                         glog
                         gtest
                         quickstep_catalog_CatalogTypedefs
-                        quickstep_cli_CommandExecutorUtil
+                        quickstep_cli_CommandExecutor
                         quickstep_cli_Constants
+                        quickstep_cli_DefaultsConfigurator
                         quickstep_cli_DropRelation
                         quickstep_cli_PrintToScreen
                         quickstep_parser_ParseStatement
@@ -90,11 +91,10 @@ if (ENABLE_DISTRIBUTED)
                         quickstep_queryexecution_Shiftboss
                         quickstep_queryexecution_Worker
                         quickstep_queryexecution_WorkerDirectory
-                        quickstep_queryoptimizer_Optimizer
-                        quickstep_queryoptimizer_OptimizerContext
                         quickstep_queryoptimizer_QueryHandle
-                        quickstep_queryoptimizer_tests_TestDatabaseLoader
+                        quickstep_queryoptimizer_QueryProcessor
                         quickstep_storage_DataExchangerAsync
+                        quickstep_storage_StorageConstants
                         quickstep_storage_StorageManager
                         quickstep_utility_Macros
                         quickstep_utility_MemStream

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 4ab94d8..a8307db 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -69,7 +69,7 @@ void CommandExecutorTestRunner::runTestCase(
       std::printf("%s\n", parse_statement.toString().c_str());
       try {
         if (parse_statement.getStatementType() == ParseStatement::kCommand) {
-          quickstep::cli::executeCommand(
+          cli::executeCommand(
               *result.parsed_statement,
               *query_processor_->getDefaultDatabase(),
               main_thread_client_id_,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/DistributedCommandExecutorTest.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/DistributedCommandExecutorTest.cpp b/cli/tests/DistributedCommandExecutorTest.cpp
index b41a70f..c8574b9 100644
--- a/cli/tests/DistributedCommandExecutorTest.cpp
+++ b/cli/tests/DistributedCommandExecutorTest.cpp
@@ -49,8 +49,6 @@ int main(int argc, char** argv) {
 
   auto test_runner = make_unique<quickstep::DistributedCommandExecutorTestRunner>(argv[3]);
   test_driver = make_unique<quickstep::TextBasedTestDriver>(&input_file, test_runner.get());
-  test_driver->registerOption(
-      quickstep::DistributedCommandExecutorTestRunner::kResetOption);
 
   ::testing::InitGoogleTest(&argc, argv);
   const int success = RUN_ALL_TESTS();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/DistributedCommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/DistributedCommandExecutorTestRunner.cpp b/cli/tests/DistributedCommandExecutorTestRunner.cpp
index a029975..d0baf9d 100644
--- a/cli/tests/DistributedCommandExecutorTestRunner.cpp
+++ b/cli/tests/DistributedCommandExecutorTestRunner.cpp
@@ -20,6 +20,7 @@
 #include "cli/tests/DistributedCommandExecutorTestRunner.hpp"
 
 #include <cstdio>
+#include <cstdlib>
 #include <memory>
 #include <set>
 #include <string>
@@ -27,8 +28,9 @@
 #include <vector>
 
 #include "catalog/CatalogTypedefs.hpp"
-#include "cli/CommandExecutorUtil.hpp"
+#include "cli/CommandExecutor.hpp"
 #include "cli/Constants.hpp"
+#include "cli/DefaultsConfigurator.hpp"
 #include "cli/DropRelation.hpp"
 #include "cli/PrintToScreen.hpp"
 #include "parser/ParseStatement.hpp"
@@ -37,11 +39,10 @@
 #include "query_execution/ForemanDistributed.hpp"
 #include "query_execution/QueryExecutionTypedefs.hpp"
 #include "query_execution/QueryExecutionUtil.hpp"
-#include "query_optimizer/Optimizer.hpp"
-#include "query_optimizer/OptimizerContext.hpp"
 #include "query_optimizer/QueryHandle.hpp"
-#include "query_optimizer/tests/TestDatabaseLoader.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
 #include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageConstants.hpp"
 #include "storage/StorageManager.hpp"
 #include "utility/MemStream.hpp"
 #include "utility/SqlError.hpp"
@@ -62,13 +63,9 @@ namespace quickstep {
 
 class CatalogRelation;
 
-namespace C = cli;
-
-const char *DistributedCommandExecutorTestRunner::kResetOption =
-    "reset_before_execution";
-
 DistributedCommandExecutorTestRunner::DistributedCommandExecutorTestRunner(const string &storage_path)
-    : query_id_(0) {
+    : catalog_path_(storage_path + kCatalogFilename),
+      query_id_(0) {
   bus_.Initialize();
 
   cli_id_ = bus_.Connect();
@@ -82,23 +79,13 @@ DistributedCommandExecutorTestRunner::DistributedCommandExecutorTestRunner(const
   block_locator_ = make_unique<BlockLocator>(&bus_);
   block_locator_->start();
 
-  test_database_loader_ = make_unique<optimizer::TestDatabaseLoader>(
-      storage_path,
-      block_locator::getBlockDomain(
-          test_database_loader_data_exchanger_.network_address(), cli_id_, &locator_client_id_,
&bus_),
-      locator_client_id_,
-      &bus_);
-  DCHECK_EQ(block_locator_->getBusClientID(), locator_client_id_);
-  test_database_loader_data_exchanger_.set_storage_manager(test_database_loader_->storage_manager());
-  test_database_loader_data_exchanger_.start();
-
-  test_database_loader_->createTestRelation(false /* allow_vchar */);
-  test_database_loader_->loadTestRelation();
+  DefaultsConfigurator::InitializeDefaultDatabase(storage_path, catalog_path_);
+  query_processor_ = std::make_unique<QueryProcessor>(std::string(catalog_path_));
 
   // NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
   // could receive a registration message from the latter.
-  foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, test_database_loader_->catalog_database(),
-                                             nullptr /* query_processor */);
+  foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, query_processor_->getDefaultDatabase(),
+                                             query_processor_.get());
   foreman_->start();
 
   // We don't use the NUMA aware version of worker code.
@@ -139,17 +126,18 @@ DistributedCommandExecutorTestRunner::~DistributedCommandExecutorTestRunner()
{
 
   foreman_->join();
 
-  test_database_loader_data_exchanger_.shutdown();
-  test_database_loader_.reset();
   data_exchanger_.shutdown();
   storage_manager_.reset();
 
   CHECK(MessageBus::SendStatus::kOK ==
       QueryExecutionUtil::SendTMBMessage(&bus_, cli_id_, locator_client_id_, TaggedMessage(kPoisonMessage)));
 
-  test_database_loader_data_exchanger_.join();
   data_exchanger_.join();
   block_locator_->join();
+
+  const std::string command = "rm -f " + catalog_path_;
+  CHECK(!std::system(command.c_str()))
+      << "Failed when attempting to remove catalog proto file: " << catalog_path_;
 }
 
 void DistributedCommandExecutorTestRunner::runTestCase(
@@ -158,12 +146,6 @@ void DistributedCommandExecutorTestRunner::runTestCase(
 
   VLOG(4) << "Test SQL(s): " << input;
 
-  if (options.find(kResetOption) != options.end()) {
-    test_database_loader_->clear();
-    test_database_loader_->createTestRelation(false /* allow_vchar */);
-    test_database_loader_->loadTestRelation();
-  }
-
   MemStream output_stream;
   sql_parser_.feedNextBuffer(new string(input));
 
@@ -181,32 +163,19 @@ void DistributedCommandExecutorTestRunner::runTestCase(
 
     try {
       if (parse_statement.getStatementType() == ParseStatement::kCommand) {
-        const ParseCommand &command = static_cast<const ParseCommand &>(parse_statement);
-        const PtrVector<ParseString> &arguments = *(command.arguments());
-        const string &command_str = command.command()->value();
-
-        string command_response;
-        if (command_str == C::kDescribeDatabaseCommand) {
-          command_response = C::ExecuteDescribeDatabase(arguments, *test_database_loader_->catalog_database());
-        } else if (command_str == C::kDescribeTableCommand) {
-          if (arguments.empty()) {
-            command_response = C::ExecuteDescribeDatabase(arguments, *test_database_loader_->catalog_database());
-          } else {
-            command_response = C::ExecuteDescribeTable(arguments, *test_database_loader_->catalog_database());
-          }
-        } else {
-          THROW_SQL_ERROR_AT(command.command()) << "Unsupported command";
-        }
-
-        std::fprintf(output_stream.file(), "%s", command_response.c_str());
+        cli::executeCommand(
+            *result.parsed_statement,
+            *query_processor_->getDefaultDatabase(),
+            cli_id_,
+            foreman_->getBusClientID(),
+            &bus_,
+            storage_manager_.get(),
+            query_processor_.get(),
+            output_stream.file());
       } else {
-        optimizer::OptimizerContext optimizer_context;
         auto query_handle = std::make_unique<QueryHandle>(query_id_++, cli_id_);
+        query_processor_->generateQueryHandle(parse_statement, query_handle.get());
 
-        optimizer_.generateQueryHandle(parse_statement,
-                                       test_database_loader_->catalog_database(),
-                                       &optimizer_context,
-                                       query_handle.get());
         const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
 
         QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
@@ -217,11 +186,11 @@ void DistributedCommandExecutorTestRunner::runTestCase(
 
         if (query_result_relation) {
           PrintToScreen::PrintRelation(*query_result_relation,
-                                       test_database_loader_->storage_manager(),
+                                       storage_manager_.get(),
                                        output_stream.file());
           DropRelation::Drop(*query_result_relation,
-                             test_database_loader_->catalog_database(),
-                             test_database_loader_->storage_manager());
+                             query_processor_->getDefaultDatabase(),
+                             storage_manager_.get());
         }
       }
     } catch (const SqlError &error) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/DistributedCommandExecutorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/cli/tests/DistributedCommandExecutorTestRunner.hpp b/cli/tests/DistributedCommandExecutorTestRunner.hpp
index 0427a85..5b03b2c 100644
--- a/cli/tests/DistributedCommandExecutorTestRunner.hpp
+++ b/cli/tests/DistributedCommandExecutorTestRunner.hpp
@@ -33,8 +33,7 @@
 #include "query_execution/Shiftboss.hpp"
 #include "query_execution/Worker.hpp"
 #include "query_execution/WorkerDirectory.hpp"
-#include "query_optimizer/Optimizer.hpp"
-#include "query_optimizer/tests/TestDatabaseLoader.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
 #include "storage/DataExchangerAsync.hpp"
 #include "storage/StorageManager.hpp"
 #include "utility/Macros.hpp"
@@ -53,12 +52,6 @@ namespace quickstep {
 class DistributedCommandExecutorTestRunner : public TextBasedTestRunner {
  public:
   /**
-   * @brief If this option is enabled, recreate the entire database and
-   *        repopulate the data before every test.
-   */
-  static const char *kResetOption;
-
-  /**
    * @brief Constructor.
    */
   explicit DistributedCommandExecutorTestRunner(const std::string &storage_path);
@@ -70,18 +63,18 @@ class DistributedCommandExecutorTestRunner : public TextBasedTestRunner
{
                    std::string *output) override;
 
  private:
+  const std::string catalog_path_;
+
   std::size_t query_id_;
 
   SqlParserWrapper sql_parser_;
-  std::unique_ptr<optimizer::TestDatabaseLoader> test_database_loader_;
-  DataExchangerAsync test_database_loader_data_exchanger_;
-  optimizer::Optimizer optimizer_;
 
   MessageBusImpl bus_;
   tmb::client_id cli_id_, locator_client_id_;
 
   std::unique_ptr<BlockLocator> block_locator_;
 
+  std::unique_ptr<QueryProcessor> query_processor_;
   std::unique_ptr<ForemanDistributed> foreman_;
 
   MessageBusImpl bus_local_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/command_executor/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/CMakeLists.txt b/cli/tests/command_executor/CMakeLists.txt
index 3850de7..86ebe98 100644
--- a/cli/tests/command_executor/CMakeLists.txt
+++ b/cli/tests/command_executor/CMakeLists.txt
@@ -32,6 +32,11 @@ add_test(quickstep_cli_tests_commandexecutor_dt
          "${CMAKE_CURRENT_BINARY_DIR}/Dt/")
 
 if (ENABLE_DISTRIBUTED)
+  add_test(quickstep_cli_tests_commandexecutor_analyze_distributed
+           "../quickstep_cli_tests_CommandExecutorTest"
+           "${CMAKE_CURRENT_SOURCE_DIR}/Analyze.test"
+           "${CMAKE_CURRENT_BINARY_DIR}/AnalyzeDistributed.test"
+           "${CMAKE_CURRENT_BINARY_DIR}/AnalyzeDistributed/")
   add_test(quickstep_cli_tests_commandexecutor_d_distributed
            "../quickstep_cli_tests_DistributedCommandExecutorTest"
            "${CMAKE_CURRENT_SOURCE_DIR}/D.test"
@@ -51,6 +56,7 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/D)
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Dt)
 
 if (ENABLE_DISTRIBUTED)
+  file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/AnalyzeDistributed)
   file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DDistributed)
   file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DtDistributed)
 endif(ENABLE_DISTRIBUTED)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 2d96b47..1001ee8 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -58,7 +58,6 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
 --
 ==
 
-
 \d foo
 --
  Table "foo"
@@ -139,23 +138,17 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
 
 ==
 
-\d
---
-       List of relations
-
- Name                                  | Type  | Blocks 
-+--------------------------------------+-------+---------+
- foo                                   | table | 1      
- foo2                                  | table | 1      
- foo3                                  | table | 1      
- foo4                                  | table | 0      
- foo_hash_part                         | table | 4      
- averylongtablenamethatseemstoneverend | table | 1      
-
-==
-
 \d invalidtable
 --
 ERROR:  Unrecognized relation invalidtable (1 : 4)
 \d invalidtable
    ^
+==
+
+DROP TABLE foo;
+DROP TABLE foo2;
+DROP TABLE foo3;
+DROP TABLE foo4;
+DROP TABLE foo_hash_part;
+--
+==

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index f735fd3..046040c 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -80,3 +80,11 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
 ERROR:  Unrecognized relation invalidtable (1 : 5)
 \dt invalidtable
     ^
+==
+
+DROP TABLE foo;
+DROP TABLE foo2;
+DROP TABLE foo3;
+DROP TABLE foo4;
+DROP TABLE averylongtablenamethatseemstoneverend;
+--


Mime
View raw message