kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aw...@apache.org
Subject [3/4] kudu git commit: [sentry] move ParseTableName to table_util
Date Tue, 16 Oct 2018 04:34:46 GMT
[sentry] move ParseTableName to table_util

This commit moves HmsCatalog::ParseTableName to table_util, so that
other modules that need to parse a Kudu table name into a database and
table name can use it without including hms module, e.g sentry.

Change-Id: I7fff5aaff623ac65a949f6cabebd9eca997f5cdc
Reviewed-on: http://gerrit.cloudera.org:8080/11658
Reviewed-by: Dan Burkert <danburkert@apache.org>
Reviewed-by: Andrew Wong <awong@cloudera.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f6eddf28
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f6eddf28
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f6eddf28

Branch: refs/heads/master
Commit: f6eddf2873214147a5f9cfd25a69d558253101ca
Parents: 3570c63
Author: Hao Hao <hao.hao@cloudera.com>
Authored: Wed Oct 10 17:33:19 2018 -0700
Committer: Hao Hao <hao.hao@cloudera.com>
Committed: Tue Oct 16 03:41:19 2018 +0000

----------------------------------------------------------------------
 src/kudu/common/CMakeLists.txt                 |  2 +
 src/kudu/common/table_util-test.cc             | 64 ++++++++++++++++++++
 src/kudu/common/table_util.cc                  | 67 +++++++++++++++++++++
 src/kudu/common/table_util.h                   | 38 ++++++++++++
 src/kudu/hms/hms_catalog-test.cc               | 34 -----------
 src/kudu/hms/hms_catalog.cc                    | 46 ++------------
 src/kudu/hms/hms_catalog.h                     | 10 ---
 src/kudu/integration-tests/master_hms-itest.cc |  4 +-
 8 files changed, 179 insertions(+), 86 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/common/CMakeLists.txt b/src/kudu/common/CMakeLists.txt
index 5d29a00..a789ccb 100644
--- a/src/kudu/common/CMakeLists.txt
+++ b/src/kudu/common/CMakeLists.txt
@@ -56,6 +56,7 @@ set(COMMON_SRCS
   row_operations.cc
   scan_spec.cc
   schema.cc
+  table_util.cc
   timestamp.cc
   types.cc
   wire_protocol.cc)
@@ -90,5 +91,6 @@ ADD_KUDU_TEST(row_changelist-test)
 ADD_KUDU_TEST(row_operations-test)
 ADD_KUDU_TEST(scan_spec-test)
 ADD_KUDU_TEST(schema-test)
+ADD_KUDU_TEST(table_util-test)
 ADD_KUDU_TEST(types-test)
 ADD_KUDU_TEST(wire_protocol-test)

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/common/table_util-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/table_util-test.cc b/src/kudu/common/table_util-test.cc
new file mode 100644
index 0000000..01344e3
--- /dev/null
+++ b/src/kudu/common/table_util-test.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/common/table_util.h"
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include "kudu/util/slice.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+
+namespace kudu {
+
+using std::string;
+
+TEST(TestTableUtil, TestParseHiveTableIdentifier) {
+  Slice db;
+  Slice tbl;
+  string table;
+
+  table = "foo.bar";
+  ASSERT_OK(ParseHiveTableIdentifier(table, &db, &tbl));
+  EXPECT_EQ("foo", db);
+  EXPECT_EQ("bar", tbl);
+
+  table = "99bottles.my_awesome/table/22";
+  ASSERT_OK(ParseHiveTableIdentifier(table, &db, &tbl));
+  EXPECT_EQ("99bottles", db);
+  EXPECT_EQ("my_awesome/table/22", tbl);
+
+  table = "_leading_underscore.trailing_underscore_";
+  ASSERT_OK(ParseHiveTableIdentifier(table, &db, &tbl));
+  EXPECT_EQ("_leading_underscore", db);
+  EXPECT_EQ("trailing_underscore_", tbl);
+
+  EXPECT_TRUE(ParseHiveTableIdentifier(".", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("no-table", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("lots.of.tables", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier(".no_table", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("no_table.", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier(".no_database", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("punctuation?.no", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("white space.no", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier("unicode‚ėÉtables.no", &db, &tbl).IsInvalidArgument());
+  EXPECT_TRUE(ParseHiveTableIdentifier(string("\0.\0", 3), &db, &tbl).IsInvalidArgument());
+}
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/common/table_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/table_util.cc b/src/kudu/common/table_util.cc
new file mode 100644
index 0000000..04d0447
--- /dev/null
+++ b/src/kudu/common/table_util.cc
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/common/table_util.h"
+
+#include <string>
+
+#include <boost/optional/optional.hpp>
+
+#include "kudu/gutil/strings/charset.h"
+#include "kudu/util/slice.h"
+#include "kudu/util/status.h"
+
+using boost::optional;
+using std::string;
+
+namespace kudu {
+
+const char* const kInvalidTableError = "when the Hive Metastore integration "
+    "is enabled, Kudu table names must be a period ('.') separated database and table name
"
+    "identifier pair, each containing only ASCII alphanumeric characters, '_', and '/'";
+
+Status ParseHiveTableIdentifier(const string& table_name,
+                                Slice* hms_database,
+                                Slice* hms_table) {
+  const char kSeparator = '.';
+  strings::CharSet charset("abcdefghijklmnopqrstuvwxyz"
+                           "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                           "0123456789"
+                           "_/");
+
+  optional<int> separator_idx;
+  for (int idx = 0; idx < table_name.size(); idx++) {
+    char c = table_name[idx];
+    if (!charset.Test(c)) {
+      if (c == kSeparator && !separator_idx) {
+        separator_idx = idx;
+      } else {
+        return Status::InvalidArgument(kInvalidTableError, table_name);
+      }
+    }
+  }
+  if (!separator_idx || *separator_idx == 0 || *separator_idx == table_name.size() - 1) {
+    return Status::InvalidArgument(kInvalidTableError, table_name);
+  }
+
+  *hms_database = Slice(table_name.data(), *separator_idx);
+  *hms_table = Slice(table_name.data() + *separator_idx + 1,
+                     table_name.size() - *separator_idx - 1);
+  return Status::OK();
+}
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/common/table_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/table_util.h b/src/kudu/common/table_util.h
new file mode 100644
index 0000000..d68076b
--- /dev/null
+++ b/src/kudu/common/table_util.h
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <string>
+
+#include "kudu/gutil/port.h"
+#include "kudu/util/status.h"
+
+namespace kudu {
+
+class Slice;
+
+extern const char* const kInvalidTableError;
+
+// Parses a Kudu table name of the form '<database>.<table>' into
+// a Hive database and table name. Returns an error if the Kudu
+// table name is not correctly formatted. The returned HMS database
+// and table slices must not outlive 'table_name'.
+Status ParseHiveTableIdentifier(const std::string& table_name,
+                                Slice* hms_database,
+                                Slice* hms_table) WARN_UNUSED_RESULT;
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/hms/hms_catalog-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 83369fb..53bd462 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -39,7 +39,6 @@
 #include "kudu/security/test/mini_kdc.h"
 #include "kudu/thrift/client.h"
 #include "kudu/util/net/net_util.h"
-#include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
@@ -57,39 +56,6 @@ using strings::Substitute;
 namespace kudu {
 namespace hms {
 
-TEST(HmsCatalogStaticTest, TestParseTableName) {
-  Slice db;
-  Slice tbl;
-  string table;
-
-  table = "foo.bar";
-  ASSERT_OK(HmsCatalog::ParseTableName(table, &db, &tbl));
-  EXPECT_EQ("foo", db);
-  EXPECT_EQ("bar", tbl);
-
-  table = "99bottles.my_awesome/table/22";
-  ASSERT_OK(HmsCatalog::ParseTableName(table, &db, &tbl));
-  EXPECT_EQ("99bottles", db);
-  EXPECT_EQ("my_awesome/table/22", tbl);
-
-  table = "_leading_underscore.trailing_underscore_";
-  ASSERT_OK(HmsCatalog::ParseTableName(table, &db, &tbl));
-  EXPECT_EQ("_leading_underscore", db);
-  EXPECT_EQ("trailing_underscore_", tbl);
-
-  EXPECT_TRUE(HmsCatalog::ParseTableName(".", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("no-table", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("lots.of.tables", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("no-table", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("lots.of.tables", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName(".no_table", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName(".no_database", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("punctuation?.no", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("white space.no", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName("unicode‚ėÉtables.no", &db, &tbl).IsInvalidArgument());
-  EXPECT_TRUE(HmsCatalog::ParseTableName(string("\0.\0", 3), &db, &tbl).IsInvalidArgument());
-}
-
 TEST(HmsCatalogStaticTest, TestNormalizeTableName) {
   string table = "foo.bar";
   ASSERT_OK(HmsCatalog::NormalizeTableName(&table));

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/hms/hms_catalog.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index 7620b2e..589fe57 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -31,11 +31,11 @@
 
 #include "kudu/common/common.pb.h"
 #include "kudu/common/schema.h"
+#include "kudu/common/table_util.h"
 #include "kudu/common/types.h"
 #include "kudu/gutil/macros.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/ascii_ctype.h"
-#include "kudu/gutil/strings/charset.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/hms/hive_metastore_types.h"
@@ -115,10 +115,6 @@ TAG_FLAG(hive_metastore_max_message_size, runtime);
 namespace kudu {
 namespace hms {
 
-const char* const HmsCatalog::kInvalidTableError = "when the Hive Metastore integration "
-    "is enabled, Kudu table names must be a period ('.') separated database and table name
"
-    "identifier pair, each containing only ASCII alphanumeric characters, '_', and '/'";
-
 HmsCatalog::HmsCatalog(string master_addresses)
     : master_addresses_(std::move(master_addresses)) {
 }
@@ -171,7 +167,7 @@ Status HmsCatalog::DropLegacyTable(const string& name) {
 Status HmsCatalog::DropTable(const string& name, const hive::EnvironmentContext&
env_ctx) {
   Slice hms_database;
   Slice hms_table;
-  RETURN_NOT_OK(ParseTableName(name, &hms_database, &hms_table));
+  RETURN_NOT_OK(ParseHiveTableIdentifier(name, &hms_database, &hms_table));
   return ha_client_.Execute([&] (HmsClient* client) {
     return client->DropTable(hms_database.ToString(), hms_table.ToString(), env_ctx);
   });
@@ -198,7 +194,7 @@ Status HmsCatalog::UpgradeLegacyImpalaTable(const string& id,
 Status HmsCatalog::DowngradeToLegacyImpalaTable(const string& name) {
   Slice hms_database;
   Slice hms_table;
-  RETURN_NOT_OK(ParseTableName(name, &hms_database, &hms_table));
+  RETURN_NOT_OK(ParseHiveTableIdentifier(name, &hms_database, &hms_table));
 
   return ha_client_.Execute([&] (HmsClient* client) {
     hive::Table table;
@@ -258,7 +254,7 @@ Status HmsCatalog::AlterTable(const string& id,
                               const Schema& schema) {
   Slice hms_database;
   Slice hms_table;
-  RETURN_NOT_OK(ParseTableName(name, &hms_database, &hms_table));
+  RETURN_NOT_OK(ParseHiveTableIdentifier(name, &hms_database, &hms_table));
 
   return ha_client_.Execute([&] (HmsClient* client) {
       // The HMS does not have a way to alter individual fields of a table
@@ -349,7 +345,7 @@ Status HmsCatalog::PopulateTable(const string& id,
                                  hive::Table* table) {
   Slice hms_database_name;
   Slice hms_table_name;
-  RETURN_NOT_OK(ParseTableName(name, &hms_database_name, &hms_table_name));
+  RETURN_NOT_OK(ParseHiveTableIdentifier(name, &hms_database_name, &hms_table_name));
   table->dbName = hms_database_name.ToString();
   table->tableName = hms_table_name.ToString();
   if (owner) {
@@ -387,7 +383,7 @@ Status HmsCatalog::NormalizeTableName(string* table_name) {
   CHECK_NOTNULL(table_name);
   Slice hms_database;
   Slice hms_table;
-  RETURN_NOT_OK(ParseTableName(*table_name, &hms_database, &hms_table));
+  RETURN_NOT_OK(ParseHiveTableIdentifier(*table_name, &hms_database, &hms_table));
 
   ToLowerCase(hms_database);
   ToLowerCase(hms_table);
@@ -395,36 +391,6 @@ Status HmsCatalog::NormalizeTableName(string* table_name) {
   return Status::OK();
 }
 
-Status HmsCatalog::ParseTableName(const string& table_name,
-                                  Slice* hms_database,
-                                  Slice* hms_table) {
-  const char kSeparator = '.';
-  strings::CharSet charset("abcdefghijklmnopqrstuvwxyz"
-                           "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-                           "0123456789"
-                           "_/");
-
-  optional<int> separator_idx;
-  for (int idx = 0; idx < table_name.size(); idx++) {
-    char c = table_name[idx];
-    if (!charset.Test(c)) {
-      if (c == kSeparator && !separator_idx) {
-        separator_idx = idx;
-      } else {
-        return Status::InvalidArgument(kInvalidTableError, table_name);
-      }
-    }
-  }
-  if (!separator_idx || *separator_idx == 0 || *separator_idx == table_name.size() - 1) {
-    return Status::InvalidArgument(kInvalidTableError, table_name);
-  }
-
-  *hms_database = Slice(table_name.data(), *separator_idx);
-  *hms_table = Slice(table_name.data() + *separator_idx + 1,
-                     table_name.size() - *separator_idx - 1);
-  return Status::OK();
-}
-
 Status HmsCatalog::ParseUris(const string& metastore_uris, vector<HostPort>* hostports)
{
   hostports->clear();
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/hms/hms_catalog.h
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_catalog.h b/src/kudu/hms/hms_catalog.h
index 6139e28..d0b2a83 100644
--- a/src/kudu/hms/hms_catalog.h
+++ b/src/kudu/hms/hms_catalog.h
@@ -34,7 +34,6 @@ namespace kudu {
 
 class HostPort;
 class Schema;
-class Slice;
 
 namespace hms {
 
@@ -46,8 +45,6 @@ namespace hms {
 class HmsCatalog {
  public:
 
-  static const char* const kInvalidTableError;
-
   explicit HmsCatalog(std::string master_addresses);
   ~HmsCatalog();
 
@@ -172,13 +169,6 @@ class HmsCatalog {
   Status DropTable(const std::string& name,
                    const hive::EnvironmentContext& env_ctx) WARN_UNUSED_RESULT;
 
-  // Parses a Kudu table name into a Hive database and table name.
-  // Returns an error if the Kudu table name is not correctly formatted.
-  // The returned HMS database and table slices must not outlive 'table_name'.
-  static Status ParseTableName(const std::string& table_name,
-                               Slice* hms_database,
-                               Slice* hms_table) WARN_UNUSED_RESULT;
-
   // Parses a Hive Metastore URI string into a sequence of HostPorts.
   static Status ParseUris(const std::string& metastore_uris, std::vector<HostPort>*
hostports);
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6eddf28/src/kudu/integration-tests/master_hms-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/master_hms-itest.cc b/src/kudu/integration-tests/master_hms-itest.cc
index cd148e7..6ea87a9 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -30,9 +30,9 @@
 #include "kudu/client/schema.h"
 #include "kudu/client/shared_ptr.h"
 #include "kudu/common/common.pb.h"
+#include "kudu/common/table_util.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/hms/hive_metastore_types.h"
-#include "kudu/hms/hms_catalog.h"
 #include "kudu/hms/hms_client.h"
 #include "kudu/hms/mini_hms.h"
 #include "kudu/integration-tests/external_mini_cluster-itest-base.h"
@@ -306,7 +306,7 @@ TEST_F(MasterHmsTest, TestRenameTable) {
   table_alterer.reset(client_->NewTableAlterer("db.a"));
   s = table_alterer->RenameTo("foo")->Alter();
   ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
-  ASSERT_STR_CONTAINS(s.ToString(), hms::HmsCatalog::kInvalidTableError);
+  ASSERT_STR_CONTAINS(s.ToString(), kInvalidTableError);
 
   // Attempt to rename the Kudu table to a non-existent database.
   table_alterer.reset(client_->NewTableAlterer("db.a"));


Mime
View raw message