quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zu...@apache.org
Subject incubator-quickstep git commit: Moved hdfs-related flags into a file.
Date Sun, 05 Feb 2017 01:32:55 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 43a9b3981 -> dda085c7a


Moved hdfs-related flags into a file.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dda085c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dda085c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dda085c7

Branch: refs/heads/master
Commit: dda085c7aeaa4f7f353c3392b547ca788d1507b8
Parents: 43a9b39
Author: Zuyu Zhang <zuyuz@apache.org>
Authored: Sat Feb 4 14:46:44 2017 -0800
Committer: Zuyu Zhang <zuyuz@apache.org>
Committed: Sat Feb 4 16:25:03 2017 -0800

----------------------------------------------------------------------
 CMakeLists.txt              |  1 +
 cli/QuickstepCli.cpp        |  1 +
 storage/CMakeLists.txt      | 13 ++++++---
 storage/FileManagerHdfs.cpp | 35 +----------------------
 storage/FileManagerHdfs.hpp |  6 ----
 storage/Flags.cpp           | 62 ++++++++++++++++++++++++++++++++++++++++
 storage/Flags.hpp           | 51 +++++++++++++++++++++++++++++++++
 storage/StorageManager.cpp  |  5 +---
 storage/StorageManager.hpp  |  4 ---
 9 files changed, 126 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 85210ef..e74a274 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -789,6 +789,7 @@ target_link_libraries(quickstep_cli_shell
                       quickstep_queryexecution_WorkerDirectory
                       quickstep_queryoptimizer_QueryHandle
                       quickstep_queryoptimizer_QueryProcessor
+                      quickstep_storage_Flags
                       quickstep_storage_PreloaderThread
                       quickstep_storage_StorageConstants
                       quickstep_storage_StorageManager

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 9db7577..02ec4ec 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -58,6 +58,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
 #include "query_execution/WorkerDirectory.hpp"
 #include "query_optimizer/QueryHandle.hpp"
 #include "query_optimizer/QueryProcessor.hpp"
+#include "storage/Flags.hpp"
 #include "storage/StorageConfig.h"  // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
 
 #ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index fddea1f..a44c3a7 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -207,6 +207,7 @@ if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
 elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
   add_library(quickstep_storage_FileManagerWindows FileManagerWindows.cpp FileManagerWindows.hpp)
 endif()
+add_library(quickstep_storage_Flags Flags.cpp Flags.hpp)
 add_library(quickstep_storage_HashTable ../empty_src.cpp HashTable.hpp)
 add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
 add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
@@ -679,8 +680,8 @@ target_link_libraries(quickstep_storage_FileManager
 if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
   target_link_libraries(quickstep_storage_FileManagerHdfs
                         glog
-                        ${GFLAGS_LIB_NAME}
                         quickstep_storage_FileManager
+                        quickstep_storage_Flags
                         quickstep_storage_StorageBlockInfo
                         quickstep_storage_StorageConstants
                         quickstep_storage_StorageErrors
@@ -711,6 +712,8 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
                         quickstep_utility_Macros
                         quickstep_utility_StringUtil)
 endif()
+target_link_libraries(quickstep_storage_Flags
+                      ${GFLAGS_LIB_NAME})
 target_link_libraries(quickstep_storage_HashTable
                       quickstep_catalog_CatalogTypedefs
                       quickstep_storage_HashTableBase
@@ -1000,6 +1003,7 @@ target_link_libraries(quickstep_storage_StorageManager
                       quickstep_storage_EvictionPolicy
                       quickstep_storage_FileManager
                       quickstep_storage_FileManagerLocal
+                      quickstep_storage_Flags
                       quickstep_storage_StorageBlob
                       quickstep_storage_StorageBlock
                       quickstep_storage_StorageBlockBase
@@ -1121,17 +1125,18 @@ target_link_libraries(quickstep_storage
                       quickstep_storage_CompressedTupleStorageSubBlock
                       quickstep_storage_CountedReference
                       quickstep_storage_EvictionPolicy
-                      quickstep_storage_FileManager
-                      quickstep_storage_FileManagerLocal
                       quickstep_storage_FastHashTable
                       quickstep_storage_FastHashTableFactory
                       quickstep_storage_FastSeparateChainingHashTable
+                      quickstep_storage_FileManager
+                      quickstep_storage_FileManagerLocal
+                      quickstep_storage_Flags
                       quickstep_storage_HashTable
-                      quickstep_storage_HashTable_proto
                       quickstep_storage_HashTableBase
                       quickstep_storage_HashTableFactory
                       quickstep_storage_HashTableKeyManager
                       quickstep_storage_HashTablePool
+                      quickstep_storage_HashTable_proto
                       quickstep_storage_IndexSubBlock
                       quickstep_storage_IndexSubBlockDescriptionFactory
                       quickstep_storage_InsertDestination

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index e28ed3b..937750a 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -31,12 +31,12 @@
 #include <cstring>
 #include <string>
 
+#include "storage/Flags.hpp"
 #include "storage/StorageBlockInfo.hpp"
 #include "storage/StorageConstants.hpp"
 #include "storage/StorageErrors.hpp"
 #include "utility/StringUtil.hpp"
 
-#include "gflags/gflags.h"
 #include "glog/logging.h"
 
 using std::size_t;
@@ -46,41 +46,8 @@ using std::string;
 
 namespace quickstep {
 
-DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
-
-static bool ValidateHdfsNameNodePort(const char *flagname,
-                                     std::int32_t value) {
-  if (value > 0 && value < 65536) {
-    return true;
-  } else {
-    std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
-    return false;
-  }
-}
-DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
-static const bool hdfs_namenode_port_dummy
-    = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
-
-static bool ValidateHdfsNumReplications(const char *flagname,
-                                        std::int32_t value) {
-  // The default dfs.namenode.replication.min is 1, while the default
-  // dfs.replication.max is 512.
-  if (value > 0) {
-    return true;
-  } else {
-    std::fprintf(stderr, "--%s must be at least 1\n", flagname);
-    return false;
-  }
-}
-DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
-static const bool hdfs_num_replications_dummy
-    = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
-
 FileManagerHdfs::FileManagerHdfs(const string &storage_path)
     : FileManager(storage_path) {
-  DCHECK(hdfs_namenode_port_dummy);
-  DCHECK(hdfs_num_replications_dummy);
-
   struct hdfsBuilder *builder = hdfsNewBuilder();
   hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
   hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.hpp b/storage/FileManagerHdfs.hpp
index 3c751b5..f47e4a8 100644
--- a/storage/FileManagerHdfs.hpp
+++ b/storage/FileManagerHdfs.hpp
@@ -29,14 +29,8 @@
 #include "storage/StorageBlockInfo.hpp"
 #include "utility/Macros.hpp"
 
-#include "gflags/gflags.h"
-
 namespace quickstep {
 
-DECLARE_string(hdfs_namenode_host);
-DECLARE_int32(hdfs_namenode_port);
-DECLARE_int32(hdfs_num_replications);
-
 /** \addtogroup Storage
  *  @{
  */

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.cpp
----------------------------------------------------------------------
diff --git a/storage/Flags.cpp b/storage/Flags.cpp
new file mode 100644
index 0000000..4672f81
--- /dev/null
+++ b/storage/Flags.cpp
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "storage/Flags.hpp"
+
+#include "storage/StorageConfig.h"
+
+#include "gflags/gflags.h"
+
+namespace quickstep {
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
+
+DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
+
+static bool ValidateHdfsNameNodePort(const char *flagname,
+                                     std::int32_t value) {
+  if (value > 0 && value < 65536) {
+    return true;
+  } else {
+    std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
+    return false;
+  }
+}
+DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
+static const bool hdfs_namenode_port_dummy
+    = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
+
+static bool ValidateHdfsNumReplications(const char *flagname,
+                                        std::int32_t value) {
+  // The default dfs.namenode.replication.min is 1, while the default
+  // dfs.replication.max is 512.
+  if (value > 0) {
+    return true;
+  } else {
+    std::fprintf(stderr, "--%s must be at least 1\n", flagname);
+    return false;
+  }
+}
+DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
+static const bool hdfs_num_replications_dummy
+    = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
+#endif
+
+}  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.hpp
----------------------------------------------------------------------
diff --git a/storage/Flags.hpp b/storage/Flags.hpp
new file mode 100644
index 0000000..1d5527c
--- /dev/null
+++ b/storage/Flags.hpp
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FLAGS_HPP_
+#define QUICKSTEP_STORAGE_FLAGS_HPP_
+
+#include "storage/StorageConfig.h"  // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
+
+#include "gflags/gflags_declare.h"
+
+namespace quickstep {
+
+/** \addtogroup STORAGE
+ *  @{
+ */
+
+/**
+ * @brief A collection of common flags shared by Quickstep STORAGEs in both the
+ * single-node and the distributed version.
+ **/
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DECLARE_bool(use_hdfs);
+
+DECLARE_string(hdfs_namenode_host);
+DECLARE_int32(hdfs_namenode_port);
+DECLARE_int32(hdfs_num_replications);
+
+#endif
+
+/** @} */
+
+}  // namespace quickstep
+
+#endif  // QUICKSTEP_STORAGE_FLAGS_HPP_

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 6299cda..6f7d38b 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -76,6 +76,7 @@
 
 #include "storage/EvictionPolicy.hpp"
 #include "storage/FileManagerLocal.hpp"
+#include "storage/Flags.hpp"
 #include "storage/StorageBlob.hpp"
 #include "storage/StorageBlock.hpp"
 #include "storage/StorageBlockBase.hpp"
@@ -189,10 +190,6 @@ DEFINE_uint64(buffer_pool_slots, 0,
 static const volatile bool buffer_pool_slots_dummy
     = gflags::RegisterFlagValidator(&FLAGS_buffer_pool_slots, &SetOrValidateBufferPoolSlots);
 
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
-#endif
-
 StorageManager::StorageManager(
     const std::string &storage_path,
     const block_id_domain block_domain,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index b61f10a..42176ee 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -63,10 +63,6 @@ namespace quickstep {
 DECLARE_int32(block_domain);
 DECLARE_uint64(buffer_pool_slots);
 
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DECLARE_bool(use_hdfs);
-#endif
-
 class CatalogRelationSchema;
 
 #ifdef QUICKSTEP_DISTRIBUTED


Mime
View raw message