kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject kudu git commit: KUDU-2281: Add Primary Key Support For INT128 Columns
Date Mon, 05 Feb 2018 20:36:06 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 839e3b3a6 -> 471cd1311


KUDU-2281: Add Primary Key Support For INT128 Columns

Change-Id: Ic6aadca2c5207f397ef6bdde4462a7497fea8ccd
Reviewed-on: http://gerrit.cloudera.org:8080/9199
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke <granthenke@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/471cd131
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/471cd131
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/471cd131

Branch: refs/heads/master
Commit: 471cd13113241ee6880dde38ea4baa0ccd4b655a
Parents: 839e3b3
Author: Grant Henke <granthenke@gmail.com>
Authored: Fri Feb 2 13:35:18 2018 -0600
Committer: Grant Henke <granthenke@gmail.com>
Committed: Mon Feb 5 20:33:51 2018 +0000

----------------------------------------------------------------------
 src/kudu/client/predicate-test.cc             |  4 +-
 src/kudu/client/schema.cc                     |  4 ++
 src/kudu/client/schema.h                      |  3 +
 src/kudu/common/encoded_key-test.cc           | 13 ++++
 src/kudu/common/key_encoder.cc                |  1 +
 src/kudu/common/key_encoder.h                 |  5 +-
 src/kudu/common/key_util-test.cc              | 33 +++++++++
 src/kudu/common/key_util.cc                   |  2 +
 src/kudu/common/partial_row.h                 |  1 +
 src/kudu/integration-tests/all_types-itest.cc | 78 ++++++++++++++++++----
 10 files changed, 128 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/client/predicate-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/predicate-test.cc b/src/kudu/client/predicate-test.cc
index 623d59c..e316d10 100644
--- a/src/kudu/client/predicate-test.cc
+++ b/src/kudu/client/predicate-test.cc
@@ -37,6 +37,7 @@
 #include "kudu/gutil/strings/escaping.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/mini-cluster/internal_mini_cluster.h"
+#include "kudu/util/decimal_util.h"
 #include "kudu/util/int128.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
@@ -980,7 +981,8 @@ TEST_F(PredicateTest, TestDecimalPredicates) {
   {
     KuduSchemaBuilder builder;
     builder.AddColumn("key")->NotNull()->Type(KuduColumnSchema::INT64)->PrimaryKey();
-    builder.AddColumn("value")->Type(KuduColumnSchema::DECIMAL)->Precision(4)->Scale(2);
+    builder.AddColumn("value")->Type(KuduColumnSchema::DECIMAL)
+        ->Precision(kMaxDecimal128Precision)->Scale(2);
     CHECK_OK(builder.Build(&schema));
   }
   unique_ptr<client::KuduTableCreator> table_creator(client_->NewTableCreator());

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/client/schema.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/schema.cc b/src/kudu/client/schema.cc
index f29281a..c8e5323 100644
--- a/src/kudu/client/schema.cc
+++ b/src/kudu/client/schema.cc
@@ -623,6 +623,10 @@ KuduColumnSchema::DataType KuduColumnSchema::type() const {
   return FromInternalDataType(DCHECK_NOTNULL(col_)->type_info()->type());
 }
 
+KuduColumnTypeAttributes KuduColumnSchema::type_attributes() const {
+  ColumnTypeAttributes type_attributes = DCHECK_NOTNULL(col_)->type_attributes();
+  return KuduColumnTypeAttributes(type_attributes.precision, type_attributes.scale);
+}
 
 ////////////////////////////////////////////////////////////
 // KuduSchema

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/client/schema.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/schema.h b/src/kudu/client/schema.h
index 4861cfc..d6084ca 100644
--- a/src/kudu/client/schema.h
+++ b/src/kudu/client/schema.h
@@ -231,6 +231,9 @@ class KUDU_EXPORT KuduColumnSchema {
   bool is_nullable() const;
   ///@}
 
+  /// @return Type attributes of the column schema.
+  KuduColumnTypeAttributes type_attributes() const;
+
  private:
   friend class KuduColumnSpec;
   friend class KuduSchema;

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/encoded_key-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/encoded_key-test.cc b/src/kudu/common/encoded_key-test.cc
index f256de1..dd5f992 100644
--- a/src/kudu/common/encoded_key-test.cc
+++ b/src/kudu/common/encoded_key-test.cc
@@ -28,6 +28,7 @@
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/strings/substitute.h" // IWYU pragma: keep
 #include "kudu/util/faststring.h"
+#include "kudu/util/int128.h"
 #include "kudu/util/memory/arena.h"
 #include "kudu/util/random.h"
 #include "kudu/util/random_util.h"
@@ -173,6 +174,18 @@ TEST_F(EncodedKeyTest, TestDecodeSimpleKeys) {
   }
 
   {
+    int128_t val = INT128_MAX;
+    EXPECT_DECODED_KEY_EQ(INT128, "(int128 key=170141183460469231731687303715884105727)",
+                          "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
&val);
+  }
+
+  {
+    int128_t val = -1234567891011121314;
+    EXPECT_DECODED_KEY_EQ(INT128, "(int128 key=-1234567891011121314)",
+                          "\x7f\xff\xff\xff\xff\xff\xff\xff\xee\xdd\xef\x0b\x4d\x2d\xcf\x5e",
&val);
+  }
+
+  {
     Slice val("aKey");
     EXPECT_DECODED_KEY_EQ(STRING, R"((string key="aKey"))", "aKey", &val);
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/key_encoder.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/key_encoder.cc b/src/kudu/common/key_encoder.cc
index 7bcb4f3..288f43e 100644
--- a/src/kudu/common/key_encoder.cc
+++ b/src/kudu/common/key_encoder.cc
@@ -54,6 +54,7 @@ class EncoderResolver {
     AddMapping<UINT64>();
     AddMapping<INT64>();
     AddMapping<BINARY>();
+    AddMapping<INT128>();
   }
 
   template<DataType Type> void AddMapping() {

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/key_encoder.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/key_encoder.h b/src/kudu/common/key_encoder.h
index 54966d7..0211ee9 100644
--- a/src/kudu/common/key_encoder.h
+++ b/src/kudu/common/key_encoder.h
@@ -73,6 +73,7 @@ struct KeyEncoderTraits<Type,
       case 2: return BigEndian::FromHost16(x);
       case 4: return BigEndian::FromHost32(x);
       case 8: return BigEndian::FromHost64(x);
+      case 16: return BigEndian::FromHost128(x);
       default: LOG(FATAL) << "bad type: " << x;
     }
     return 0;
@@ -89,7 +90,7 @@ struct KeyEncoderTraits<Type,
 
     // To encode signed integers, swap the MSB.
     if (MathLimits<cpp_type>::kIsSigned) {
-      key_unsigned ^= 1UL << (sizeof(key_unsigned) * CHAR_BIT - 1);
+      key_unsigned ^= static_cast<unsigned_cpp_type>(1) << (sizeof(key_unsigned)
* CHAR_BIT - 1);
     }
     key_unsigned = SwapEndian(key_unsigned);
     dst->append(reinterpret_cast<const char*>(&key_unsigned), sizeof(key_unsigned));
@@ -111,7 +112,7 @@ struct KeyEncoderTraits<Type,
     memcpy(&val,  encoded_key->data(), sizeof(cpp_type));
     val = SwapEndian(val);
     if (MathLimits<cpp_type>::kIsSigned) {
-      val ^= 1UL << (sizeof(val) * CHAR_BIT - 1);
+      val ^= static_cast<unsigned_cpp_type>(1) << (sizeof(val) * CHAR_BIT - 1);
     }
     memcpy(cell_ptr, &val, sizeof(val));
     encoded_key->remove_prefix(sizeof(cpp_type));

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/key_util-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/key_util-test.cc b/src/kudu/common/key_util-test.cc
index abbf741..29ded24 100644
--- a/src/kudu/common/key_util-test.cc
+++ b/src/kudu/common/key_util-test.cc
@@ -27,8 +27,10 @@
 #include "kudu/common/partial_row.h"
 #include "kudu/common/row.h"
 #include "kudu/common/schema.h"
+#include "kudu/common/types.h"
 #include "kudu/gutil/mathlimits.h"
 #include "kudu/util/memory/arena.h"
+#include "kudu/util/int128.h"
 #include "kudu/util/slice.h"
 #include "kudu/util/test_util.h"
 #include "kudu/util/test_macros.h"
@@ -66,6 +68,25 @@ TEST_F(KeyUtilTest, TestIncrementNonCompositePrimaryKey) {
   EXPECT_EQ("int32 key=-2147483648", p_row.ToString());
 }
 
+TEST_F(KeyUtilTest, TestIncrementInt128PrimaryKey) {
+  Schema schema({ ColumnSchema("key", INT128),
+                  ColumnSchema("other_col", INT32),
+                  ColumnSchema("other_col2", STRING, true) },
+                1);
+  KuduPartialRow p_row(&schema);
+  ContiguousRow row(&schema, row_data(&p_row));
+
+  // Normal increment.
+  EXPECT_OK(p_row.Set<TypeTraits<INT128>>(0, 1000));
+  EXPECT_TRUE(key_util::IncrementPrimaryKey(&row, &arena_));
+  EXPECT_EQ("int128 key=1001", p_row.ToString());
+
+  // Overflow increment.
+  EXPECT_OK(p_row.Set<TypeTraits<INT128>>(0, INT128_MAX));
+  EXPECT_FALSE(key_util::IncrementPrimaryKey(&row, &arena_));
+  EXPECT_EQ("int128 key=-170141183460469231731687303715884105728", p_row.ToString());
+}
+
 TEST_F(KeyUtilTest, TestIncrementCompositePrimaryKey) {
   Schema schema({ ColumnSchema("k1", INT32),
                   ColumnSchema("k2", INT32),
@@ -163,6 +184,18 @@ TEST_F(KeyUtilTest, TestTryDecrementCell) {
     EXPECT_EQ(orig, std::numeric_limits<int32_t>::min());
   }
   {
+    ColumnSchema col_int128("a", INT128);
+    int128_t orig = 0;
+    EXPECT_EQ(key_util::TryDecrementCell(col_int128, &orig), true);
+    EXPECT_EQ(orig, -1);
+  }
+  {
+    ColumnSchema col_int128("a", INT128);
+    int128_t orig = INT128_MIN;
+    EXPECT_EQ(key_util::TryDecrementCell(col_int128, &orig), false);
+    EXPECT_EQ(orig, INT128_MIN);
+  }
+  {
     ColumnSchema col_bool("a", BOOL);
     bool orig = true;
     EXPECT_EQ(key_util::TryDecrementCell(col_bool, &orig), true);

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/key_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/common/key_util.cc b/src/kudu/common/key_util.cc
index 60368b0..ed8dd86 100644
--- a/src/kudu/common/key_util.cc
+++ b/src/kudu/common/key_util.cc
@@ -358,6 +358,7 @@ bool IncrementCell(const ColumnSchema& col, void* cell_ptr, Arena*
arena) {
     HANDLE_TYPE(INT32);
     HANDLE_TYPE(UNIXTIME_MICROS);
     HANDLE_TYPE(INT64);
+    HANDLE_TYPE(INT128);
     case FLOAT:
       return IncrementFloatingPointCell<FLOAT>(cell_ptr);
     case DOUBLE:
@@ -387,6 +388,7 @@ bool TryDecrementCell(const ColumnSchema &col, void *cell_ptr) {
     HANDLE_TYPE(INT32);
     HANDLE_TYPE(UNIXTIME_MICROS);
     HANDLE_TYPE(INT64);
+    HANDLE_TYPE(INT128);
     case FLOAT:
       return DecrementFloatingPointCell<FLOAT>(cell_ptr);
     case DOUBLE:

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/common/partial_row.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/partial_row.h b/src/kudu/common/partial_row.h
index ce40835..68fd0f4 100644
--- a/src/kudu/common/partial_row.h
+++ b/src/kudu/common/partial_row.h
@@ -493,6 +493,7 @@ class KUDU_EXPORT KuduPartialRow {
   template<typename KeyTypeWrapper> friend struct client::IntKeysTestSetup;
   template<typename KeyTypeWrapper> friend struct tablet::SliceTypeRowOps;
   template<typename KeyTypeWrapper> friend struct tablet::NumTypeRowOps;
+  FRIEND_TEST(KeyUtilTest, TestIncrementInt128PrimaryKey);
   FRIEND_TEST(PartitionPrunerTest, TestIntPartialPrimaryKeyRangePruning);
   FRIEND_TEST(PartitionPrunerTest, TestPartialPrimaryKeyRangePruning);
   FRIEND_TEST(PartitionPrunerTest, TestPrimaryKeyRangePruning);

http://git-wip-us.apache.org/repos/asf/kudu/blob/471cd131/src/kudu/integration-tests/all_types-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/all_types-itest.cc b/src/kudu/integration-tests/all_types-itest.cc
index 3d355f7..a5c635e 100644
--- a/src/kudu/integration-tests/all_types-itest.cc
+++ b/src/kudu/integration-tests/all_types-itest.cc
@@ -45,6 +45,7 @@
 #include "kudu/integration-tests/cluster_verifier.h"
 #include "kudu/mini-cluster/external_mini_cluster.h"
 #include "kudu/util/bitmap.h"
+#include "kudu/util/decimal_util.h"
 #include "kudu/util/int128.h"
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
@@ -77,8 +78,16 @@ struct SliceKeysTestSetup {
   }
 
   void AddKeyColumnsToSchema(KuduSchemaBuilder* builder) const {
-    builder->AddColumn("key")->Type(
-        client::FromInternalDataType(KeyTypeWrapper::kType))->NotNull()->PrimaryKey();
+    auto column_spec = builder->AddColumn("key");
+    column_spec->Type(client::FromInternalDataType(KeyTypeWrapper::kType))
+        ->NotNull()->PrimaryKey();
+    if (KeyTypeWrapper::kType == DECIMAL32) {
+      column_spec->Precision(kMaxDecimal32Precision);
+    } else if (KeyTypeWrapper::kType == DECIMAL64) {
+      column_spec->Precision(kMaxDecimal64Precision);
+    } else if (KeyTypeWrapper::kType == DECIMAL128) {
+      column_spec->Precision(kMaxDecimal128Precision);
+    }
   }
 
   // Split points are calculated by equally partitioning the int64_t key space and then
@@ -166,8 +175,16 @@ struct IntKeysTestSetup {
   }
 
   void AddKeyColumnsToSchema(KuduSchemaBuilder* builder) const {
-    builder->AddColumn("key")->Type(
-        client::FromInternalDataType(KeyTypeWrapper::kType))->NotNull()->PrimaryKey();
+    auto column_spec = builder->AddColumn("key");
+    column_spec->Type(client::FromInternalDataType(KeyTypeWrapper::kType))
+        ->NotNull()->PrimaryKey();
+    if (KeyTypeWrapper::kType == DECIMAL32) {
+      column_spec->Precision(kMaxDecimal32Precision);
+    } else if (KeyTypeWrapper::kType == DECIMAL64) {
+      column_spec->Precision(kMaxDecimal64Precision);
+    } else if (KeyTypeWrapper::kType == DECIMAL128) {
+      column_spec->Precision(kMaxDecimal128Precision);
+    }
   }
 
   vector<const KuduPartialRow*> GenerateSplitRows(const KuduSchema& schema) const
{
@@ -271,7 +288,12 @@ class AllTypesItest : public KuduTest {
     builder.AddColumn("float_val")->Type(KuduColumnSchema::FLOAT);
     builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE);
     builder.AddColumn("binary_val")->Type(KuduColumnSchema::BINARY);
-    builder.AddColumn("decimal_val")->Type(KuduColumnSchema::DECIMAL)->Precision(18)->Scale(8);
+    builder.AddColumn("decimal32_val")->Type(KuduColumnSchema::DECIMAL)
+        ->Precision(kMaxDecimal32Precision);
+    builder.AddColumn("decimal64_val")->Type(KuduColumnSchema::DECIMAL)
+        ->Precision(kMaxDecimal64Precision);
+    builder.AddColumn("decimal128_val")->Type(KuduColumnSchema::DECIMAL)
+        ->Precision(kMaxDecimal128Precision);
     CHECK_OK(builder.Build(&schema_));
   }
 
@@ -339,7 +361,9 @@ class AllTypesItest : public KuduTest {
     RETURN_NOT_OK(row->SetDouble("double_val", double_val));
     RETURN_NOT_OK(row->SetFloat("float_val", double_val));
     RETURN_NOT_OK(row->SetBool("bool_val", int_val % 2));
-    RETURN_NOT_OK(row->SetUnscaledDecimal("decimal_val", int_val));
+    RETURN_NOT_OK(row->SetUnscaledDecimal("decimal32_val", int_val));
+    RETURN_NOT_OK(row->SetUnscaledDecimal("decimal64_val", int_val));
+    RETURN_NOT_OK(row->SetUnscaledDecimal("decimal128_val", int_val));
     VLOG(1) << "Inserting row[" << split_idx << "," << row_idx <<
"]" << insert->ToString();
     RETURN_NOT_OK(session->Apply(insert));
     return Status::OK();
@@ -376,7 +400,9 @@ class AllTypesItest : public KuduTest {
     projection->push_back("double_val");
     projection->push_back("float_val");
     projection->push_back("bool_val");
-    projection->push_back("decimal_val");
+    projection->push_back("decimal32_val");
+    projection->push_back("decimal64_val");
+    projection->push_back("decimal128_val");
   }
 
   ExpectedVals GetExpectedValsForRow(int split_idx, int row_idx) {
@@ -432,9 +458,15 @@ class AllTypesItest : public KuduTest {
     float float_val;
     ASSERT_OK(row.GetFloat("float_val", &float_val));
     ASSERT_EQ(float_val, vals.expected_float_val);
-    int128_t decimal_val;
-    ASSERT_OK(row.GetUnscaledDecimal("decimal_val", &decimal_val));
-    ASSERT_EQ(decimal_val, vals.expected_decimal_val);
+    int128_t decimal32_val;
+    ASSERT_OK(row.GetUnscaledDecimal("decimal32_val", &decimal32_val));
+    ASSERT_EQ(decimal32_val, vals.expected_decimal_val);
+    int128_t decimal64_val;
+    ASSERT_OK(row.GetUnscaledDecimal("decimal64_val", &decimal64_val));
+    ASSERT_EQ(decimal64_val, vals.expected_decimal_val);
+    int128_t decimal128_val;
+    ASSERT_OK(row.GetUnscaledDecimal("decimal128_val", &decimal128_val));
+    ASSERT_EQ(decimal128_val, vals.expected_decimal_val);
   }
 
   typedef std::function<Status (KuduScanner* scanner)> ScannerSetup;
@@ -519,6 +551,9 @@ typedef ::testing::Types<IntKeysTestSetup<KeyTypeWrapper<INT8>
>,
                          IntKeysTestSetup<KeyTypeWrapper<INT16> >,
                          IntKeysTestSetup<KeyTypeWrapper<INT32> >,
                          IntKeysTestSetup<KeyTypeWrapper<INT64> >,
+                         IntKeysTestSetup<KeyTypeWrapper<DECIMAL32> >,
+                         IntKeysTestSetup<KeyTypeWrapper<DECIMAL64> >,
+                         IntKeysTestSetup<KeyTypeWrapper<DECIMAL128> >,
                          IntKeysTestSetup<KeyTypeWrapper<UNIXTIME_MICROS> >,
                          SliceKeysTestSetup<KeyTypeWrapper<STRING> >,
                          SliceKeysTestSetup<KeyTypeWrapper<BINARY> >
@@ -575,7 +610,7 @@ TYPED_TEST(AllTypesItest, TestTimestampPadding) {
           break;
         default:
           int col_size = GetTypeInfo(ToInternalDataType(col_schema.type(),
-                                                        KuduColumnTypeAttributes(18, 8)))->size();
+                                                        col_schema.type_attributes()))->size();
           projection_offsets.push_back(col_size);
           row_stride += col_size;
       }
@@ -594,10 +629,12 @@ TYPED_TEST(AllTypesItest, TestTimestampPadding) {
       for (int j = 0; j < schema->num_columns(); j++) {
         KuduColumnSchema col_schema = schema->Column(j);
 
-        if (schema->Column(j).name() == "key") {
+        if (col_schema.name() == "key") {
           ASSERT_OK(this->setup_.VerifyRowKeyRaw(row_data, num_tablet, *total_rows_in_tablet
+ i));
         } else {
           ExpectedVals vals = this->GetExpectedValsForRow(num_tablet, *total_rows_in_tablet
+ i);
+          DataType internal_type = ToInternalDataType(col_schema.type(),
+                                                      col_schema.type_attributes());
           switch (col_schema.type()) {
             case KuduColumnSchema::INT8:
               ASSERT_EQ(*reinterpret_cast<const int8_t*>(row_data), vals.expected_int8_val);
@@ -630,7 +667,22 @@ TYPED_TEST(AllTypesItest, TestTimestampPadding) {
               ASSERT_EQ(*reinterpret_cast<const double*>(row_data), vals.expected_double_val);
               break;
             case KuduColumnSchema::DECIMAL:
-              ASSERT_EQ(*reinterpret_cast<const int64_t*>(row_data), vals.expected_decimal_val);
+              switch (internal_type) {
+                case DECIMAL32:
+                  ASSERT_EQ(*reinterpret_cast<const int32_t*>(row_data),
+                            vals.expected_decimal_val);
+                  break;
+                case DECIMAL64:
+                  ASSERT_EQ(*reinterpret_cast<const int64_t*>(row_data),
+                            vals.expected_decimal_val);
+                  break;
+                case DECIMAL128:
+                  ASSERT_EQ(*reinterpret_cast<const int128_t*>(row_data),
+                            vals.expected_decimal_val);
+                  break;
+                default:
+                  LOG(FATAL) << "Unexpected internal decimal type: " << internal_type;
+              }
               break;
             default:
               LOG(FATAL) << "Unexpected type: " << col_schema.type();


Mime
View raw message