impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [1/2] impala git commit: IMPALA-3271: organise and warn on removed startup flags
Date Sun, 11 Feb 2018 01:32:13 GMT
Repository: impala
Updated Branches:
  refs/heads/master 315142190 -> 2e6034786


IMPALA-3271: organise and warn on removed startup flags

Gather all the removed flags in a single place and establish consistent
behaviour.

If the flag is set, a warning is logged. GFlags are initialised before
logging, so the warning goes to stderr instead of the WARNING log.

The affected flags are:
be_service_threads, cgroup_hierarchy_path, enable_accept_queue_server,
enable_partitioned_aggregation, enable_partitioned_hash_join,
enable_phj_probe_side_filtering, enable_rm, llama_addresses,
llama_callback_port, llama_host, llama_max_request_attempts,
llama_port, llama_registration_timeout_secs,
llama_registration_wait_secs, local_nodemanager_url,
resource_broker_cnxn_attempts, resource_broker_cnxn_retry_interval_ms,
resource_broker_recv_timeout, resource_broker_send_timeout,
rm_always_use_defaults, rm_default_cpu_vcores, rm_default_memory,
rpc_cnxn_attempts, rpc_cnxn_retry_interval_ms, staging_cgroup,
suppress_unknown_disk_id_warnings, use_statestore,

Testing:
Ran "start-impala-cluster.py --impalad_args=--enable_rm=true"
and verified that the warning appeared in
logs/cluster/impalad-error.log

Cherry-picks: not for 2.x

Change-Id: Ic9bb4792e1b18840aa85aa5d755a09f2b5461f34
Reviewed-on: http://gerrit.cloudera.org:8080/9173
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0047f813
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0047f813
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0047f813

Branch: refs/heads/master
Commit: 0047f813abc4dc8fb08a15ac8b9d0f582ab8a5f5
Parents: 3151421
Author: Tim Armstrong <tarmstrong@cloudera.com>
Authored: Wed Jan 31 15:13:04 2018 -0800
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Fri Feb 9 20:01:33 2018 +0000

----------------------------------------------------------------------
 be/src/common/global-flags.cc             | 56 +++++++++++++++++++++++++-
 be/src/exec/exec-node.cc                  |  2 -
 be/src/exec/hdfs-scan-node-base.cc        |  3 --
 be/src/exec/partitioned-hash-join-node.cc |  2 -
 be/src/rpc/thrift-server.cc               |  3 --
 be/src/runtime/exec-env.cc                | 17 --------
 be/src/scheduling/query-schedule.cc       |  5 ---
 be/src/service/impala-server.cc           |  5 ---
 be/src/service/impalad-main.cc            |  8 ----
 9 files changed, 54 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 2d832da..8e81d23 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -160,8 +160,6 @@ DEFINE_int32(kudu_operation_timeout_ms, 3 * 60 * 1000, "Timeout (milliseconds)
s
     "all Kudu operations. This must be a positive value, and there is no way to disable "
     "timeouts.");
 
-DEFINE_bool_hidden(enable_accept_queue_server, true, "Deprecated");
-
 DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
     "incremental stats the catalog is allowed to serialize per table. "
     "This limit is set as a safety check, to prevent the JVM from "
@@ -194,3 +192,57 @@ DEFINE_string(reserved_words_version, "3.0.0", "Reserved words compatibility
ver
     "Reserved words cannot be used as identifiers in SQL. This flag determines the impala"
     " version from which the reserved word list is taken. The value must be one of "
     "[\"2.11.0\", \"3.0.0\"].");
+
+// ++========================++
+// || Startup flag graveyard ||
+// ++========================++
+//
+//                       -----------
+//           -----------/   R I P   ╲
+//          /   R I P   ╲ -----------|-----------
+//          |-----------|           |/   R I P   ╲
+//          |           |   LLAMA   ||-----------|
+//          | Old Aggs  |           ||           |
+//          |           |    --     || Old Joins |
+//          |    --     |           ||           |
+//          |           |           ||    --     |
+//          |           |~.~~.~~.~~~~|           |
+//          ~~.~~.~~.~~~~            |           |
+//                                   ~~.~~.~~.~~~~
+// The flags have no effect but we don't want to prevent Impala from starting when they
+// are provided on the command line after an upgrade. We issue a warning if the flag is
+// set from the command line.
+#define REMOVED_FLAG(flagname) \
+  DEFINE_string_hidden(flagname, "__UNSET__", "Removed"); \
+  DEFINE_validator(flagname, [](const char* name, const string& val) { \
+      if (val != "__UNSET__") LOG(WARNING) << "Ignoring removed flag " << name;
\
+      return true; \
+    });
+
+REMOVED_FLAG(be_service_threads);
+REMOVED_FLAG(cgroup_hierarchy_path);
+REMOVED_FLAG(enable_accept_queue_server);
+REMOVED_FLAG(enable_partitioned_aggregation);
+REMOVED_FLAG(enable_partitioned_hash_join);
+REMOVED_FLAG(enable_phj_probe_side_filtering);
+REMOVED_FLAG(enable_rm);
+REMOVED_FLAG(llama_addresses);
+REMOVED_FLAG(llama_callback_port);
+REMOVED_FLAG(llama_host);
+REMOVED_FLAG(llama_max_request_attempts);
+REMOVED_FLAG(llama_port);
+REMOVED_FLAG(llama_registration_timeout_secs);
+REMOVED_FLAG(llama_registration_wait_secs);
+REMOVED_FLAG(local_nodemanager_url);
+REMOVED_FLAG(resource_broker_cnxn_attempts);
+REMOVED_FLAG(resource_broker_cnxn_retry_interval_ms);
+REMOVED_FLAG(resource_broker_recv_timeout);
+REMOVED_FLAG(resource_broker_send_timeout);
+REMOVED_FLAG(rm_always_use_defaults);
+REMOVED_FLAG(rm_default_cpu_vcores);
+REMOVED_FLAG(rm_default_memory);
+REMOVED_FLAG(rpc_cnxn_attempts);
+REMOVED_FLAG(rpc_cnxn_retry_interval_ms);
+REMOVED_FLAG(staging_cgroup);
+REMOVED_FLAG(suppress_unknown_disk_id_warnings);
+REMOVED_FLAG(use_statestore);

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index eaf8dd1..08856f1 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -70,8 +70,6 @@ using strings::Substitute;
 
 DECLARE_int32(be_port);
 DECLARE_string(hostname);
-DEFINE_bool_hidden(enable_partitioned_hash_join, true, "Deprecated - has no effect");
-DEFINE_bool_hidden(enable_partitioned_aggregation, true, "Deprecated - has no effect");
 
 namespace impala {
 

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index 8b065fa..b40fb7e 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -46,9 +46,6 @@
 
 #include "common/names.h"
 
-// TODO: Remove this flag in a compatibility-breaking release.
-DEFINE_bool(suppress_unknown_disk_id_warnings, false, "Deprecated.");
-
 #ifndef NDEBUG
 DECLARE_bool(skip_file_runtime_filtering);
 #endif

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/exec/partitioned-hash-join-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-hash-join-node.cc b/be/src/exec/partitioned-hash-join-node.cc
index e76a9ba..dd3efd1 100644
--- a/be/src/exec/partitioned-hash-join-node.cc
+++ b/be/src/exec/partitioned-hash-join-node.cc
@@ -39,8 +39,6 @@
 
 #include "common/names.h"
 
-DEFINE_bool_hidden(enable_phj_probe_side_filtering, true, "Deprecated.");
-
 static const string PREPARE_FOR_READ_FAILED_ERROR_MSG =
     "Failed to acquire initial read buffer for stream in hash join node $0. Reducing "
     "query concurrency or increasing the memory limit may help this query to complete "

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/rpc/thrift-server.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-server.cc b/be/src/rpc/thrift-server.cc
index 48fb1b9..8f948cd 100644
--- a/be/src/rpc/thrift-server.cc
+++ b/be/src/rpc/thrift-server.cc
@@ -58,9 +58,6 @@ using namespace apache::thrift::server;
 using namespace apache::thrift::transport;
 using namespace apache::thrift;
 
-DEFINE_int32_hidden(rpc_cnxn_attempts, 10, "Deprecated");
-DEFINE_int32_hidden(rpc_cnxn_retry_interval_ms, 2000, "Deprecated");
-
 DECLARE_string(principal);
 DECLARE_string(keytab_file);
 

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 1c3ab7a..ff6a374 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -70,7 +70,6 @@ using boost::algorithm::join;
 using kudu::rpc::ServiceIf;
 using namespace strings;
 
-DEFINE_bool_hidden(use_statestore, true, "Deprecated, do not use");
 DEFINE_string(catalog_service_host, "localhost",
     "hostname where CatalogService is running");
 DEFINE_bool(enable_webserver, true, "If true, debug webserver is enabled");
@@ -103,22 +102,6 @@ DECLARE_bool(is_coordinator);
 DECLARE_int32(webserver_port);
 DECLARE_int64(tcmalloc_max_total_thread_cache_bytes);
 
-// TODO: Remove the following RM-related flags in Impala 3.0.
-DEFINE_bool_hidden(enable_rm, false, "Deprecated");
-DEFINE_int32_hidden(llama_callback_port, 28000, "Deprecated");
-DEFINE_string_hidden(llama_host, "", "Deprecated");
-DEFINE_int32_hidden(llama_port, 15000, "Deprecated");
-DEFINE_string_hidden(llama_addresses, "", "Deprecated");
-DEFINE_int64_hidden(llama_registration_timeout_secs, 30, "Deprecated");
-DEFINE_int64_hidden(llama_registration_wait_secs, 3, "Deprecated");
-DEFINE_int64_hidden(llama_max_request_attempts, 5, "Deprecated");
-DEFINE_string_hidden(cgroup_hierarchy_path, "", "Deprecated");
-DEFINE_string_hidden(staging_cgroup, "impala_staging", "Deprecated");
-DEFINE_int32_hidden(resource_broker_cnxn_attempts, 1, "Deprecated");
-DEFINE_int32_hidden(resource_broker_cnxn_retry_interval_ms, 3000, "Deprecated");
-DEFINE_int32_hidden(resource_broker_send_timeout, 0, "Deprecated");
-DEFINE_int32_hidden(resource_broker_recv_timeout, 0, "Deprecated");
-
 // TODO-MT: rename or retire
 DEFINE_int32(coordinator_rpc_threads, 12, "(Advanced) Number of threads available to "
     "start fragments on remote Impala daemons.");

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/scheduling/query-schedule.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/query-schedule.cc b/be/src/scheduling/query-schedule.cc
index f833273..a424f78 100644
--- a/be/src/scheduling/query-schedule.cc
+++ b/be/src/scheduling/query-schedule.cc
@@ -35,11 +35,6 @@ using boost::uuids::random_generator;
 using boost::uuids::uuid;
 using namespace impala;
 
-// TODO: Remove for Impala 3.0.
-DEFINE_bool_hidden(rm_always_use_defaults, false, "Deprecated");
-DEFINE_string_hidden(rm_default_memory, "4G", "Deprecated");
-DEFINE_int32_hidden(rm_default_cpu_vcores, 2, "Deprecated");
-
 namespace impala {
 
 QuerySchedule::QuerySchedule(const TUniqueId& query_id,

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index ee8405b..9898029 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -118,8 +118,6 @@ DEFINE_int32(hs2_port, 21050, "port on which HiveServer2 client requests
are ser
 
 DEFINE_int32(fe_service_threads, 64,
     "number of threads available to serve client requests");
-DEFINE_int32_hidden(be_service_threads, 64,
-    "Deprecated, no longer has any effect. Will be removed in Impala 3.0.");
 DEFINE_string(default_query_options, "", "key=value pair of default query options for"
     " impalad, separated by ','");
 DEFINE_int32(query_log_size, 25, "Number of queries to retain in the query log. If -1, "
@@ -208,9 +206,6 @@ DEFINE_bool(is_executor, true, "If true, this Impala daemon will execute
query "
       "milliseconds. Only used for testing.");
 #endif
 
-// TODO: Remove for Impala 3.0.
-DEFINE_string_hidden(local_nodemanager_url, "", "Deprecated");
-
 DECLARE_bool(compact_catalog_topic);
 
 namespace impala {

http://git-wip-us.apache.org/repos/asf/impala/blob/0047f813/be/src/service/impalad-main.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impalad-main.cc b/be/src/service/impalad-main.cc
index e0049d0..8463467 100644
--- a/be/src/service/impalad-main.cc
+++ b/be/src/service/impalad-main.cc
@@ -54,7 +54,6 @@ using namespace impala;
 DECLARE_int32(beeswax_port);
 DECLARE_int32(hs2_port);
 DECLARE_int32(be_port);
-DECLARE_bool(enable_rm);
 DECLARE_bool(is_coordinator);
 
 int ImpaladMain(int argc, char** argv) {
@@ -70,13 +69,6 @@ int ImpaladMain(int argc, char** argv) {
   ABORT_IF_ERROR(JniCatalogCacheUpdateIterator::InitJNI());
   InitFeSupport();
 
-  if (FLAGS_enable_rm) {
-    // TODO: Remove in Impala 3.0.
-    LOG(WARNING) << "*****************************************************************";
-    LOG(WARNING) << "Llama support has been deprecated. FLAGS_enable_rm has no effect.";
-    LOG(WARNING) << "*****************************************************************";
-  }
-
   ExecEnv exec_env;
   ABORT_IF_ERROR(exec_env.Init());
   CommonMetrics::InitCommonMetrics(exec_env.metrics());


Mime
View raw message