impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sail...@apache.org
Subject [1/9] incubator-impala git commit: IMPALA-3552: Make incremental stats max serialized size configurable
Date Tue, 15 Nov 2016 18:50:29 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master cac02d6b7 -> b3cbc960a


IMPALA-3552: Make incremental stats max serialized size configurable

The fix "IMPALA-2648/IMPALA-2664" introduced a conservative limitation
on the maximum serialized size of incremental stats. As a side effect,
some users with very large tables are experiencing regressions
especially when they upgrade impala and the serialized size goes
beyond 200MB.

To mitigate the issue, the change introduces a new gflag,
'inc_stats_size_limit_bytes' to make the max serialized size
configurable, which allows impala users to specify their own maximum
serialized size. Default value for inc_stats_size_limit_bytes is
200MB.

The change introduces a TBackendGflags class to pass the gflags from
backend to the Frontend and the Catalog via thrift.  This also revamps
existing query options to use the TBackendConfig.

Change-Id: I33684725a61eabc67237503e61178305d37d3cb5
Reviewed-on: http://gerrit.cloudera.org:8080/4867
Reviewed-by: Bharath Vissapragada <bharathv@cloudera.com>
Tested-by: Internal Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3f2f008a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3f2f008a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3f2f008a

Branch: refs/heads/master
Commit: 3f2f008ac4365d7f7c616730d3faeebd776b74d2
Parents: cac02d6
Author: Bharath Vissapragada <bharathv@cloudera.com>
Authored: Fri Nov 4 16:36:39 2016 -0700
Committer: Internal Jenkins <cloudera-hudson@gerrit.cloudera.org>
Committed: Tue Nov 15 03:22:11 2016 +0000

----------------------------------------------------------------------
 be/generated-sources/gen-cpp/CMakeLists.txt     |  2 +
 be/src/catalog/catalog.cc                       | 28 ++------
 be/src/common/global-flags.cc                   |  6 ++
 be/src/service/fe-support.cc                    | 18 -----
 be/src/service/frontend.cc                      | 38 +++-------
 be/src/util/CMakeLists.txt                      |  1 +
 be/src/util/backend-gflag-util.cc               | 76 ++++++++++++++++++++
 be/src/util/backend-gflag-util.h                | 31 ++++++++
 common/thrift/BackendGflags.thrift              | 55 ++++++++++++++
 common/thrift/CMakeLists.txt                    |  1 +
 common/thrift/Frontend.thrift                   |  5 --
 .../apache/impala/analysis/AlterViewStmt.java   |  3 +-
 .../impala/analysis/ComputeStatsStmt.java       | 10 +--
 .../apache/impala/analysis/CreateViewStmt.java  |  3 +-
 .../org/apache/impala/authorization/User.java   | 26 ++-----
 .../org/apache/impala/catalog/HdfsTable.java    | 11 +--
 .../org/apache/impala/catalog/KuduTable.java    |  2 +-
 .../org/apache/impala/common/RuntimeEnv.java    | 12 ----
 .../java/org/apache/impala/planner/Planner.java |  3 +-
 .../apache/impala/service/BackendConfig.java    | 70 ++++++++++--------
 .../org/apache/impala/service/FeSupport.java    | 18 -----
 .../org/apache/impala/service/JniCatalog.java   | 30 ++++----
 .../org/apache/impala/service/JniFrontend.java  | 26 +++----
 .../java/org/apache/impala/util/KuduUtil.java   |  4 +-
 24 files changed, 280 insertions(+), 199 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/generated-sources/gen-cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/generated-sources/gen-cpp/CMakeLists.txt b/be/generated-sources/gen-cpp/CMakeLists.txt
index 35f7d65..ea7735f 100644
--- a/be/generated-sources/gen-cpp/CMakeLists.txt
+++ b/be/generated-sources/gen-cpp/CMakeLists.txt
@@ -39,6 +39,8 @@ set(SRC_FILES
   ImpalaHiveServer2Service.cpp
   beeswax_constants.cpp
   beeswax_types.cpp
+  BackendGflags_constants.cpp
+  BackendGflags_types.cpp
   BeeswaxService.cpp
   TCLIService.cpp
   TCLIService_constants.cpp

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/catalog/catalog.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc
index 7d6fdca..b3690f6 100644
--- a/be/src/catalog/catalog.cc
+++ b/be/src/catalog/catalog.cc
@@ -22,15 +22,12 @@
 
 #include "common/logging.h"
 #include "rpc/jni-thrift-util.h"
-#include "util/logging-support.h"
+#include "util/backend-gflag-util.h"
 
 #include "common/names.h"
 
 using namespace impala;
 
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_string(local_library_dir);
 
 DEFINE_bool(load_catalog_in_background, false,
     "If true, loads catalog metadata in the background. If false, metadata is loaded "
@@ -41,13 +38,9 @@ DEFINE_int32(num_metadata_loading_threads, 16,
 DEFINE_string(sentry_config, "", "Local path to a sentry-site.xml configuration "
     "file. If set, authorization will be enabled.");
 
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_int32(kudu_operation_timeout_ms);
-
 Catalog::Catalog() {
   JniMethodDescriptor methods[] = {
-    {"<init>", "(ZILjava/lang/String;IIZLjava/lang/String;Ljava/lang/String;I)V",
-        &catalog_ctor_},
+    {"<init>", "([B)V", &catalog_ctor_},
     {"updateCatalog", "([B)[B", &update_metastore_id_},
     {"execDdl", "([B)[B", &exec_ddl_id_},
     {"resetMetadata", "([B)[B", &reset_metadata_id_},
@@ -70,19 +63,10 @@ Catalog::Catalog() {
     ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, catalog_class_, &(methods[i])));
   }
 
-  jboolean load_in_background = FLAGS_load_catalog_in_background;
-  jint num_metadata_loading_threads = FLAGS_num_metadata_loading_threads;
-  jstring sentry_config = jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
-  // auth_to_local rules are read if --load_auth_to_local_rules is set to true
-  // and impala is kerberized.
-  jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
-  jstring principal = jni_env->NewStringUTF(FLAGS_principal.c_str());
-  jstring local_library_dir = jni_env->NewStringUTF(FLAGS_local_library_dir.c_str());
-  jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
-  jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_,
-      load_in_background, num_metadata_loading_threads, sentry_config,
-      FlagToTLogLevel(FLAGS_v), FlagToTLogLevel(FLAGS_non_impala_java_vlog),
-      auth_to_local, principal, local_library_dir, kudu_operation_timeout);
+  jbyteArray cfg_bytes;
+  ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+  jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_, cfg_bytes);
   EXIT_IF_EXC(jni_env);
   ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, catalog, &catalog_));
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index f75236c..5a3eff4 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -123,3 +123,9 @@ DEFINE_bool(enable_accept_queue_server, true,
     "TThreadedServer that accepts connections as quickly as possible and hands them off "
     "to a thread pool to finish setup, reducing the chances that connections time out "
     "waiting to be accepted.");
+
+DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
+    "incremental stats the catalog is allowed to serialize per table. "
+    "This limit is set as a safety check, to prevent the JVM from "
+    "hitting a maximum array limit of 1GB (or OOM) while building "
+    "the thrift objects to send to impalads. By default, it's set to 200MB");

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/fe-support.cc
----------------------------------------------------------------------
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index 42ceea2..7147964 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -336,20 +336,6 @@ Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad(
   return result_bytes;
 }
 
-extern "C"
-JNIEXPORT jbyteArray JNICALL
-Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions(JNIEnv* env,
-    jclass caller_class) {
-  TStartupOptions options;
-  ExecEnv* exec_env = ExecEnv::GetInstance();
-  ImpalaServer* impala_server = exec_env->impala_server();
-  options.__set_compute_lineage(impala_server->IsLineageLoggingEnabled());
-  jbyteArray result_bytes = NULL;
-  THROW_IF_ERROR_RET(SerializeThriftMsg(env, &options, &result_bytes), env,
-                     JniUtil::internal_exc_class(), result_bytes);
-  return result_bytes;
-}
-
 namespace impala {
 
 static JNINativeMethod native_methods[] = {
@@ -373,10 +359,6 @@ static JNINativeMethod native_methods[] = {
     (char*)"NativePrioritizeLoad", (char*)"([B)[B",
     (void*)::Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad
   },
-  {
-    (char*)"NativeGetStartupOptions", (char*)"()[B",
-    (void*)::Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions
-  },
 };
 
 void InitFeSupport() {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/frontend.cc
----------------------------------------------------------------------
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index f8cecfb..b83d091 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -22,21 +22,13 @@
 
 #include "common/logging.h"
 #include "rpc/jni-thrift-util.h"
+#include "util/backend-gflag-util.h"
 #include "util/jni-util.h"
-#include "util/logging-support.h"
 
 #include "common/names.h"
 
 using namespace impala;
 
-DECLARE_string(sentry_config);
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_int32(kudu_operation_timeout_ms);
-
-DEFINE_bool(load_catalog_at_startup, false, "if true, load all catalog data at startup");
-
 // Authorization related flags. Must be set to valid values to properly configure
 // authorization.
 DEFINE_string(server_name, "", "The name to use for securing this impalad "
@@ -62,10 +54,10 @@ DEFINE_string(authorized_proxy_user_config_delimiter, ",",
 DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The given "
     "value should be a comma separated list of hostnames or IP addresses; ports are "
     "optional.");
+
 Frontend::Frontend() {
   JniMethodDescriptor methods[] = {
-    {"<init>", "(ZLjava/lang/String;Ljava/lang/String;Ljava/lang/String;"
-        "Ljava/lang/String;IIZLjava/lang/String;I)V", &fe_ctor_},
+    {"<init>", "([B)V", &fe_ctor_},
     {"createExecRequest", "([B)[B", &create_exec_request_id_},
     {"getExplainPlan", "([B)Ljava/lang/String;", &get_explain_plan_id_},
     {"getHadoopConfig", "([B)[B", &get_hadoop_config_id_},
@@ -89,7 +81,7 @@ Frontend::Frontend() {
     {"loadTableData", "([B)[B", &load_table_data_id_},
     {"getTableFiles", "([B)[B", &get_table_files_id_},
     {"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_},
-    {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_},
+    {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_}
 };
 
   JNIEnv* jni_env = getJNIEnv();
@@ -102,24 +94,10 @@ Frontend::Frontend() {
     ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, fe_class_, &(methods[i])));
   };
 
-  jboolean lazy = (FLAGS_load_catalog_at_startup ? false : true);
-  jstring policy_file_path =
-      jni_env->NewStringUTF(FLAGS_authorization_policy_file.c_str());
-  jstring server_name =
-      jni_env->NewStringUTF(FLAGS_server_name.c_str());
-  jstring sentry_config =
-      jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
-  jstring auth_provider_class =
-      jni_env->NewStringUTF(FLAGS_authorization_policy_provider_class.c_str());
-  // auth_to_local rules are read if --load_auth_to_local_rules is set to true
-  // and impala is kerberized.
-  jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
-  jstring kudu_master_hosts = jni_env->NewStringUTF(FLAGS_kudu_master_hosts.c_str());
-  jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
-  jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, lazy, server_name,
-      policy_file_path, sentry_config, auth_provider_class, FlagToTLogLevel(FLAGS_v),
-      FlagToTLogLevel(FLAGS_non_impala_java_vlog), auth_to_local, kudu_master_hosts,
-      kudu_operation_timeout);
+  jbyteArray cfg_bytes;
+  ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+  jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, cfg_bytes);
   EXIT_IF_EXC(jni_env);
   ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, fe, &fe_));
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index ecc222a..6656a38 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -32,6 +32,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/util")
 add_library(Util
   auth-util.cc
   avro-util.cc
+  backend-gflag-util.cc
   benchmark.cc
   bitmap.cc
   bit-util.cc

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
new file mode 100644
index 0000000..f504e57
--- /dev/null
+++ b/be/src/util/backend-gflag-util.cc
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/backend-gflag-util.h"
+
+#include "gen-cpp/BackendGflags_types.h"
+#include "rpc/jni-thrift-util.h"
+#include "util/logging-support.h"
+
+#include <gflags/gflags.h>
+
+// Configs for the Frontend and the Catalog.
+DECLARE_bool(load_catalog_in_background);
+DECLARE_bool(load_auth_to_local_rules);
+DECLARE_int32(non_impala_java_vlog);
+DECLARE_int32(read_size);
+DECLARE_int32(num_metadata_loading_threads);
+DECLARE_int32(kudu_operation_timeout_ms);
+DECLARE_int64(inc_stats_size_limit_bytes);
+DECLARE_string(principal);
+DECLARE_string(lineage_event_log_dir);
+DECLARE_string(principal);
+DECLARE_string(local_library_dir);
+DECLARE_string(server_name);
+DECLARE_string(authorization_policy_file);
+DECLARE_string(authorization_policy_provider_class);
+DECLARE_string(authorized_proxy_user_config);
+DECLARE_string(authorized_proxy_user_config_delimiter);
+DECLARE_string(kudu_master_hosts);
+DECLARE_string(sentry_config);
+DECLARE_string(sentry_config);
+
+namespace impala {
+
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
+  TBackendGflags cfg;
+  cfg.__set_authorization_policy_file(FLAGS_authorization_policy_file);
+  cfg.__set_load_catalog_in_background(FLAGS_load_catalog_in_background);
+  cfg.__set_server_name(FLAGS_server_name);
+  cfg.__set_sentry_config(FLAGS_sentry_config);
+  cfg.__set_authorization_policy_provider_class(
+      FLAGS_authorization_policy_provider_class);
+  cfg.__set_kudu_master_hosts(FLAGS_kudu_master_hosts);
+  cfg.__set_read_size(FLAGS_read_size);
+  cfg.__set_num_metadata_loading_threads(FLAGS_num_metadata_loading_threads);
+  cfg.__set_sentry_config(FLAGS_sentry_config);
+  // auth_to_local rules are read if --load_auth_to_local_rules is set to true
+  // and impala is kerberized.
+  cfg.__set_load_auth_to_local_rules(FLAGS_load_auth_to_local_rules);
+  cfg.__set_principal(FLAGS_principal);
+  cfg.__set_impala_log_lvl(FlagToTLogLevel(FLAGS_v));
+  cfg.__set_non_impala_java_vlog(FlagToTLogLevel(FLAGS_non_impala_java_vlog));
+  cfg.__set_inc_stats_size_limit_bytes(FLAGS_inc_stats_size_limit_bytes);
+  cfg.__set_lineage_event_log_dir(FLAGS_lineage_event_log_dir);
+  cfg.__set_local_library_path(FLAGS_local_library_dir);
+  cfg.__set_kudu_operation_timeout_ms(FLAGS_kudu_operation_timeout_ms);
+  JniLocalFrame jni_frame;
+  RETURN_IF_ERROR(jni_frame.push(jni_env));
+  RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
+  return Status::OK();
+}
+}

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.h b/be/src/util/backend-gflag-util.h
new file mode 100644
index 0000000..796a2d5
--- /dev/null
+++ b/be/src/util/backend-gflag-util.h
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef UTIL_BACKEND_CONFIG_H_
+#define UTIL_BACKEND_CONFIG_H_
+
+#include "common/status.h"
+#include "util/jni-util.h"
+
+namespace impala {
+
+/// Builds the TBackendGflags object to pass to JNI. This is used to pass the gflag
+/// configs to the Frontend and the Catalog.
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes);
+}
+
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/BackendGflags.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
new file mode 100644
index 0000000..ad081a6
--- /dev/null
+++ b/common/thrift/BackendGflags.thrift
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace cpp impala
+namespace java org.apache.impala.thrift
+
+// Used to pass gflags from backend to frontend, JniCatalog and JniFrontend
+// Attributes without comments correspond to gflags
+struct TBackendGflags {
+  1: required string sentry_config
+
+  2: required bool load_auth_to_local_rules
+
+  3: required i32 non_impala_java_vlog
+
+  4: required i32 impala_log_lvl
+
+  5: required i64 inc_stats_size_limit_bytes
+
+  6: required string lineage_event_log_dir
+
+  7: required bool load_catalog_in_background
+
+  8: required i32 num_metadata_loading_threads
+
+  9: required string principal
+
+  10: required string authorization_policy_file
+
+  11: required string server_name
+
+  12: required string authorization_policy_provider_class
+
+  13: required string kudu_master_hosts
+
+  14: required string local_library_path
+
+  15: required i32 read_size
+
+  16: required i32 kudu_operation_timeout_ms
+}

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt
index 10d363d..1512e52 100644
--- a/common/thrift/CMakeLists.txt
+++ b/common/thrift/CMakeLists.txt
@@ -144,6 +144,7 @@ set (EXT_DATA_SRC_FILES
 set (SRC_FILES
   ErrorCodes.thrift
   beeswax.thrift
+  BackendGflags.thrift
   CatalogInternalService.thrift
   CatalogObjects.thrift
   CatalogService.thrift

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index f6ba35a..8c279f5 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -721,11 +721,6 @@ struct TGetAllHadoopConfigsResponse {
   1: optional map<string, string> configs;
 }
 
-// BE startup options
-struct TStartupOptions {
-  1: optional bool compute_lineage
-}
-
 // For creating a test descriptor table. The tuples and their memory layout are computed
 // in the FE.
 struct TBuildTestDescriptorTableParams {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
index e7d3f66..6740f14 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
@@ -22,6 +22,7 @@ import org.apache.impala.catalog.Table;
 import org.apache.impala.catalog.View;
 import org.apache.impala.common.AnalysisException;
 import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.service.BackendConfig;
 
 import com.google.common.base.Preconditions;
 
@@ -52,7 +53,7 @@ public class AlterViewStmt extends CreateOrAlterViewStmtBase {
     }
 
     createColumnAndViewDefs(analyzer);
-    if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+    if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
       computeLineageGraph(analyzer);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 467faca..5ad3cad 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -33,6 +33,7 @@ import org.apache.impala.catalog.Type;
 import org.apache.impala.catalog.View;
 import org.apache.impala.common.AnalysisException;
 import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
 import org.apache.impala.thrift.TComputeStatsParams;
 import org.apache.impala.thrift.TPartitionStats;
 import org.apache.impala.thrift.TTableName;
@@ -284,16 +285,17 @@ public class ComputeStatsStmt extends StatementBase {
           }
       }
       // For incremental stats, estimate the size of intermediate stats and report an
-      // error if the estimate is greater than MAX_INCREMENTAL_STATS_SIZE_BYTES.
+      // error if the estimate is greater than --inc_stats_size_limit_bytes in bytes
       if (isIncremental_) {
+        long incStatMaxSize = BackendConfig.INSTANCE.getIncStatsMaxSize();
         long statsSizeEstimate = hdfsTable.getColumns().size() *
             hdfsTable.getPartitions().size() * HdfsTable.STATS_SIZE_PER_COLUMN_BYTES;
-        if (statsSizeEstimate > HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES) {
+        if (statsSizeEstimate > incStatMaxSize) {
           LOG.error("Incremental stats size estimate for table " + hdfsTable.getName() +
-              " exceeded " + HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES + ", estimate = "
+              " exceeded " + incStatMaxSize + ", estimate = "
               + statsSizeEstimate);
           throw new AnalysisException("Incremental stats size estimate exceeds "
-              + PrintUtils.printBytes(HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES)
+              + PrintUtils.printBytes(incStatMaxSize)
               + ". Please try COMPUTE STATS instead.");
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
index 6e30aeb..6b90083 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
@@ -24,6 +24,7 @@ import org.apache.impala.common.AnalysisException;
 import org.apache.impala.common.RuntimeEnv;
 import org.apache.impala.thrift.TAccessEvent;
 import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.service.BackendConfig;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -60,7 +61,7 @@ public class CreateViewStmt extends CreateOrAlterViewStmtBase {
         TCatalogObjectType.VIEW, Privilege.CREATE.toString()));
 
     createColumnAndViewDefs(analyzer);
-    if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+    if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
       computeLineageGraph(analyzer);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/authorization/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/User.java b/fe/src/main/java/org/apache/impala/authorization/User.java
index 190419b..e29b19f 100644
--- a/fe/src/main/java/org/apache/impala/authorization/User.java
+++ b/fe/src/main/java/org/apache/impala/authorization/User.java
@@ -19,42 +19,24 @@ package org.apache.impala.authorization;
 
 import com.google.common.base.Preconditions;
 import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.impala.common.InternalException;
 import org.apache.impala.common.RuntimeEnv;
 import org.apache.impala.service.BackendConfig;
 
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import org.apache.hadoop.security.authentication.util.KerberosName;
 
 /*
  * Class that represents a User of an Impala session.
  */
 public class User {
 
-  static {
-    // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
-    // from core-site.xml and use it for principal to short name conversion. If it is not,
-    // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
-    // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
-    // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
-    // the "DEFAULT" rule that just extracts the username from any principal in the
-    // cluster's local realm. For more details on principal to short name translation,
-    // refer to org.apache.hadoop.security.KerberosName.
-    final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
-    final Configuration conf = new Configuration();
-    if (BackendConfig.isAuthToLocalEnabled()) {
-      KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
-    } else {
-      // just extract the simple user name
-      KerberosName.setRules(defaultRule);
-    }
-  }
-
   private final String name_;
 
+  // Refer to BackendConfig.initAuthToLocal() for initialization
+  // of static auth_to_local configs in KerberosName class.
   private KerberosName kerberosName_;
 
   public User(String name) {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 97831d7..69b4b4e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -63,6 +63,7 @@ import org.apache.impala.common.AnalysisException;
 import org.apache.impala.common.FileSystemUtil;
 import org.apache.impala.common.Pair;
 import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
 import org.apache.impala.thrift.ImpalaInternalServiceConstants;
 import org.apache.impala.thrift.TAccessLevel;
 import org.apache.impala.thrift.TCatalogObjectType;
@@ -169,12 +170,6 @@ public class HdfsTable extends Table {
   // Store all the partition ids of an HdfsTable.
   private HashSet<Long> partitionIds_ = Sets.newHashSet();
 
-  // Maximum size (in bytes) of incremental stats the catalog is allowed to serialize per
-  // table. This limit is set as a safety check, to prevent the JVM from hitting a
-  // maximum array limit of 1GB (or OOM) while building the thrift objects to send to
-  // impalads.
-  public static final long MAX_INCREMENTAL_STATS_SIZE_BYTES = 200 * 1024 * 1024;
-
   // Estimate (in bytes) of the incremental stats size per column per partition
   public static final long STATS_SIZE_PER_COLUMN_BYTES = 400;
 
@@ -1593,7 +1588,7 @@ public class HdfsTable extends Table {
    * partitions). To prevent the catalog from hitting an OOM error while trying to
    * serialize large partition incremental stats, we estimate the stats size and filter
    * the incremental stats data from partition objects if the estimate exceeds
-   * MAX_INCREMENTAL_STATS_SIZE_BYTES.
+   * --inc_stats_size_limit_bytes
    */
   private THdfsTable getTHdfsTable(boolean includeFileDesc, Set<Long> refPartitions) {
     // includeFileDesc implies all partitions should be included (refPartitions == null).
@@ -1603,7 +1598,7 @@ public class HdfsTable extends Table {
     long statsSizeEstimate =
         numPartitions * getColumns().size() * STATS_SIZE_PER_COLUMN_BYTES;
     boolean includeIncrementalStats =
-        (statsSizeEstimate < MAX_INCREMENTAL_STATS_SIZE_BYTES);
+        (statsSizeEstimate < BackendConfig.INSTANCE.getIncStatsMaxSize());
     Map<Long, THdfsPartition> idToPartition = Maps.newHashMap();
     for (HdfsPartition partition: partitionMap_.values()) {
       long id = partition.getId();

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 6c30077..7b906a7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -340,7 +340,7 @@ public class KuduTable extends Table {
     try (KuduClient client = KuduUtil.createKuduClient(getKuduMasterHosts())) {
       org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName_);
       List<LocatedTablet> tablets =
-          kuduTable.getTabletsLocations(BackendConfig.getKuduClientTimeoutMs());
+          kuduTable.getTabletsLocations(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
       for (LocatedTablet tab: tablets) {
         TResultRowBuilder builder = new TResultRowBuilder();
         builder.add("-1");   // The Kudu client API doesn't expose tablet row counts.

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
index af58b16..d33455a 100644
--- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
+++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
@@ -21,7 +21,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.impala.service.FeSupport;
-import org.apache.impala.thrift.TStartupOptions;
 
 /**
  * Contains runtime-specific parameters such as the number of CPU cores. Currently only
@@ -35,21 +34,11 @@ public class RuntimeEnv {
 
   private int numCores_;
 
-  // Indicates if column lineage information should be computed for each query.
-  private boolean computeLineage_;
-
   // Indicates whether this is an environment for testing.
   private boolean isTestEnv_;
 
   public RuntimeEnv() {
     reset();
-    try {
-      TStartupOptions opts = FeSupport.GetStartupOptions();
-      computeLineage_ = opts.compute_lineage;
-    } catch (InternalException e) {
-      LOG.error("Error retrieving BE startup options. Shutting down JVM");
-      System.exit(1);
-    }
   }
 
   /**
@@ -63,7 +52,6 @@ public class RuntimeEnv {
   public void setNumCores(int numCores) { this.numCores_ = numCores; }
   public void setTestEnv(boolean v) { isTestEnv_ = v; }
   public boolean isTestEnv() { return isTestEnv_; }
-  public boolean computeLineage() { return computeLineage_; }
   public boolean isKuduSupported() {
     return "true".equals(System.getenv("KUDU_IS_SUPPORTED"));
   }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/planner/Planner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index ec4cf7e..52524fb 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -39,6 +39,7 @@ import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.PrintUtils;
 import org.apache.impala.common.RuntimeEnv;
 import org.apache.impala.common.TreeNode;
+import org.apache.impala.service.BackendConfig;
 import org.apache.impala.thrift.TExplainLevel;
 import org.apache.impala.thrift.TQueryCtx;
 import org.apache.impala.thrift.TQueryExecRequest;
@@ -179,7 +180,7 @@ public class Planner {
     ctx_.getRootAnalyzer().getTimeline().markEvent("Distributed plan created");
 
     ColumnLineageGraph graph = ctx_.getRootAnalyzer().getColumnLineageGraph();
-    if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+    if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
       // Compute the column lineage graph
       if (ctx_.isInsertOrCtas()) {
         Table targetTable = ctx_.getAnalysisResult().getInsertStmt().getTargetTable();

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index c90f674..47c987e 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -18,46 +18,60 @@
 package org.apache.impala.service;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.impala.thrift.TBackendGflags;
 
 /**
  * This class is meant to provide the FE with impalad backend configuration parameters,
  * including command line arguments.
- * TODO: Remove this class and either
- * a) Figure out if there's a standard way to access flags from java
- * b) Create a util/gflags.java that let's us access the be flags
  */
 public class BackendConfig {
-  public static BackendConfig INSTANCE = new BackendConfig();
-
-  // Default read block size (in bytes). This is the same as
-  // the default FLAGS_read_size used by the IO manager in the backend.
-  private final long READ_SIZE;
-
-  // Variables below are overriden by JniFrontend/JniCatalog with user set configuration.
-  // TODO: Read from backend instead of using static variables.
-
-  // Determines how principal to short name conversion works. See User.java for more info.
-  private static boolean allowAuthToLocalRules_ = false;
+  public static BackendConfig INSTANCE;
 
-  // Kudu client timeout (ms).
-  private static int kuduOperationTimeoutMs_ = 3 * 60 * 1000;
+  private TBackendGflags backendCfg_;
 
-  private BackendConfig() {
-    // TODO: Populate these by making calls to the backend instead of default constants.
-    READ_SIZE = 8 * 1024 * 1024L;
+  private BackendConfig(TBackendGflags cfg) {
+    backendCfg_ = cfg;
   }
 
-  public long getReadSize() { return READ_SIZE; }
-
-  public static boolean isAuthToLocalEnabled() { return allowAuthToLocalRules_; }
-  public static void setAuthToLocal(boolean authToLocal) {
-    allowAuthToLocalRules_ = authToLocal;
+  public static void create(TBackendGflags cfg) {
+    Preconditions.checkNotNull(cfg);
+    INSTANCE = new BackendConfig(cfg);
+    initAuthToLocal();
   }
 
-  public static int getKuduClientTimeoutMs() { return kuduOperationTimeoutMs_; }
+  public long getReadSize() { return backendCfg_.read_size; }
+  public boolean getComputeLineage() {
+    return !Strings.isNullOrEmpty(backendCfg_.lineage_event_log_dir);
+  }
+  public long getIncStatsMaxSize() { return backendCfg_.inc_stats_size_limit_bytes; }
+  public boolean isAuthToLocalEnabled() {
+    return backendCfg_.load_auth_to_local_rules &&
+        !Strings.isNullOrEmpty(backendCfg_.principal);
+  }
+  public int getKuduClientTimeoutMs() { return backendCfg_.kudu_operation_timeout_ms; }
 
-  public static void setKuduClientTimeoutMs(int kuduOperationTimeoutMs) {
-    Preconditions.checkArgument(kuduOperationTimeoutMs > 0);
-    BackendConfig.kuduOperationTimeoutMs_ = kuduOperationTimeoutMs;
+  // Inits the auth_to_local configuration in the static KerberosName class.
+  private static void initAuthToLocal() {
+    // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
+    // from core-site.xml and use it for principal to short name conversion. If it is not,
+    // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
+    // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
+    // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
+    // the "DEFAULT" rule that just extracts the username from any principal in the
+    // cluster's local realm. For more details on principal to short name translation,
+    // refer to org.apache.hadoop.security.KerberosName.
+    final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
+    final Configuration conf = new Configuration();
+    if (INSTANCE.isAuthToLocalEnabled()) {
+      KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
+    } else {
+      // just extract the simple user name
+      KerberosName.setRules(defaultRule);
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/FeSupport.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/FeSupport.java b/fe/src/main/java/org/apache/impala/service/FeSupport.java
index f982df3..c578726 100644
--- a/fe/src/main/java/org/apache/impala/service/FeSupport.java
+++ b/fe/src/main/java/org/apache/impala/service/FeSupport.java
@@ -45,7 +45,6 @@ import org.apache.impala.thrift.TPrioritizeLoadResponse;
 import org.apache.impala.thrift.TQueryCtx;
 import org.apache.impala.thrift.TResultRow;
 import org.apache.impala.thrift.TStatus;
-import org.apache.impala.thrift.TStartupOptions;
 import org.apache.impala.thrift.TSymbolLookupParams;
 import org.apache.impala.thrift.TSymbolLookupResult;
 import org.apache.impala.thrift.TTable;
@@ -84,9 +83,6 @@ public class FeSupport {
   // using Java Thrift bindings.
   public native static byte[] NativePrioritizeLoad(byte[] thriftReq);
 
-  // Return select BE startup options as a serialized TStartupOptions
-  public native static byte[] NativeGetStartupOptions();
-
   /**
    * Locally caches the jar at the specified HDFS location.
    *
@@ -263,20 +259,6 @@ public class FeSupport {
     }
   }
 
-  public static TStartupOptions GetStartupOptions() throws InternalException {
-    try {
-      byte[] result = NativeGetStartupOptions();
-      Preconditions.checkNotNull(result);
-      TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
-      TStartupOptions options = new TStartupOptions();
-      deserializer.deserialize(options, result);
-      return options;
-    } catch (TException e) {
-      throw new InternalException("Error retrieving startup options: " + e.getMessage(),
-          e);
-    }
-  }
-
   /**
    * This function should only be called explicitly by the FeSupport to ensure that
    * native functions are loaded.

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
index 7993191..db9ba63 100644
--- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
+++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
@@ -47,6 +47,7 @@ import org.apache.impala.thrift.TResetMetadataRequest;
 import org.apache.impala.thrift.TSentryAdminCheckRequest;
 import org.apache.impala.thrift.TUniqueId;
 import org.apache.impala.thrift.TUpdateCatalogRequest;
+import org.apache.impala.thrift.TBackendGflags;
 import org.apache.impala.util.GlogAppender;
 import org.apache.impala.util.PatternMatcher;
 import org.apache.thrift.TException;
@@ -78,29 +79,30 @@ public class JniCatalog {
     return new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
   }
 
-  public JniCatalog(boolean loadInBackground, int numMetadataLoadingThreads,
-      String sentryServiceConfig, int impalaLogLevel, int otherLogLevel,
-      boolean allowAuthToLocal, String kerberosPrincipal, String localLibraryPath,
-      int kuduOperationTimeoutMs) throws InternalException {
-    BackendConfig.setAuthToLocal(allowAuthToLocal);
-    BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
-    Preconditions.checkArgument(numMetadataLoadingThreads > 0);
+  public JniCatalog(byte[] thriftBackendConfig) throws InternalException,
+      ImpalaException, TException {
+    TBackendGflags cfg = new TBackendGflags();
+    JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+    BackendConfig.create(cfg);
+
+    Preconditions.checkArgument(cfg.num_metadata_loading_threads > 0);
     // This trick saves having to pass a TLogLevel enum, which is an object and more
     // complex to pass through JNI.
-    GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
-        TLogLevel.values()[otherLogLevel]);
+    GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+        TLogLevel.values()[cfg.non_impala_java_vlog]);
 
     // Check if the Sentry Service is configured. If so, create a configuration object.
     SentryConfig sentryConfig = null;
-    if (!Strings.isNullOrEmpty(sentryServiceConfig)) {
-      sentryConfig = new SentryConfig(sentryServiceConfig);
+    if (!Strings.isNullOrEmpty(cfg.sentry_config)) {
+      sentryConfig = new SentryConfig(cfg.sentry_config);
       sentryConfig.loadConfig();
     }
     LOG.info(JniUtil.getJavaVersion());
 
-    catalog_ = new CatalogServiceCatalog(loadInBackground,
-        numMetadataLoadingThreads, sentryConfig, getServiceId(), kerberosPrincipal,
-        localLibraryPath);
+    catalog_ = new CatalogServiceCatalog(cfg.load_catalog_in_background,
+        cfg.num_metadata_loading_threads, sentryConfig, getServiceId(), cfg.principal,
+        cfg.local_library_path);
     try {
       catalog_.reset();
     } catch (CatalogException e) {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 3ef89e3..5e41af4 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -46,6 +46,7 @@ import org.apache.impala.common.FileSystemUtil;
 import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.InternalException;
 import org.apache.impala.common.JniUtil;
+import org.apache.impala.service.BackendConfig;
 import org.apache.impala.thrift.TBuildTestDescriptorTableParams;
 import org.apache.impala.thrift.TCatalogObject;
 import org.apache.impala.thrift.TDatabase;
@@ -81,6 +82,7 @@ import org.apache.impala.thrift.TTableName;
 import org.apache.impala.thrift.TUniqueId;
 import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
 import org.apache.impala.thrift.TUpdateMembershipRequest;
+import org.apache.impala.thrift.TBackendGflags;
 import org.apache.impala.util.GlogAppender;
 import org.apache.impala.util.PatternMatcher;
 import org.apache.impala.util.TSessionStateUtil;
@@ -115,20 +117,20 @@ public class JniFrontend {
   /**
    * Create a new instance of the Jni Frontend.
    */
-  public JniFrontend(boolean lazy, String serverName, String authorizationPolicyFile,
-      String sentryConfigFile, String authPolicyProviderClass, int impalaLogLevel,
-      int otherLogLevel, boolean allowAuthToLocal, String defaultKuduMasterHosts,
-      int kuduOperationTimeoutMs)
-      throws InternalException {
-    BackendConfig.setAuthToLocal(allowAuthToLocal);
-    BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
-    GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
-        TLogLevel.values()[otherLogLevel]);
+  public JniFrontend(byte[] thriftBackendConfig) throws ImpalaException, TException {
+    TBackendGflags cfg = new TBackendGflags();
+    JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+    BackendConfig.create(cfg);
+
+    GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+        TLogLevel.values()[cfg.non_impala_java_vlog]);
 
     // Validate the authorization configuration before initializing the Frontend.
     // If there are any configuration problems Impala startup will fail.
-    AuthorizationConfig authConfig = new AuthorizationConfig(serverName,
-        authorizationPolicyFile, sentryConfigFile, authPolicyProviderClass);
+    AuthorizationConfig authConfig = new AuthorizationConfig(cfg.server_name,
+        cfg.authorization_policy_file, cfg.sentry_config,
+        cfg.authorization_policy_provider_class);
     authConfig.validateConfig();
     if (authConfig.isEnabled()) {
       LOG.info(String.format("Authorization is 'ENABLED' using %s",
@@ -139,7 +141,7 @@ public class JniFrontend {
     }
     LOG.info(JniUtil.getJavaVersion());
 
-    frontend_ = new Frontend(authConfig, defaultKuduMasterHosts);
+    frontend_ = new Frontend(authConfig, cfg.kudu_master_hosts);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/KuduUtil.java b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
index c50bf25..65fae74 100644
--- a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
@@ -54,8 +54,8 @@ public class KuduUtil {
    */
   public static KuduClient createKuduClient(String kuduMasters) {
     KuduClientBuilder b = new KuduClient.KuduClientBuilder(kuduMasters);
-    b.defaultAdminOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
-    b.defaultOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
+    b.defaultAdminOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
+    b.defaultOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
     return b.build();
   }
 


Mime
View raw message