kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [1/5] incubator-kudu git commit: Integrate ColumnPredicate into client and server
Date Sun, 20 Mar 2016 20:45:34 GMT
Repository: incubator-kudu
Updated Branches:
  refs/heads/master 9c798f45c -> 90be132b9


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53e67e9e/src/kudu/tserver/tablet_service.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_service.cc b/src/kudu/tserver/tablet_service.cc
index 565bd22..5a34927 100644
--- a/src/kudu/tserver/tablet_service.cc
+++ b/src/kudu/tserver/tablet_service.cc
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "kudu/common/iterator.h"
+#include "kudu/common/scan_spec.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/wire_protocol.h"
 #include "kudu/consensus/consensus.h"
@@ -35,13 +36,13 @@
 #include "kudu/rpc/rpc_context.h"
 #include "kudu/rpc/rpc_sidecar.h"
 #include "kudu/server/hybrid_clock.h"
-#include "kudu/tablet/tablet_bootstrap.h"
-#include "kudu/tserver/remote_bootstrap_service.h"
 #include "kudu/tablet/metadata.pb.h"
-#include "kudu/tablet/tablet_peer.h"
+#include "kudu/tablet/tablet_bootstrap.h"
 #include "kudu/tablet/tablet_metrics.h"
+#include "kudu/tablet/tablet_peer.h"
 #include "kudu/tablet/transactions/alter_schema_transaction.h"
 #include "kudu/tablet/transactions/write_transaction.h"
+#include "kudu/tserver/remote_bootstrap_service.h"
 #include "kudu/tserver/scanners.h"
 #include "kudu/tserver/tablet_server.h"
 #include "kudu/tserver/ts_tablet_manager.h"
@@ -1153,6 +1154,10 @@ void TabletServiceImpl::Checksum(const ChecksumRequestPB* req,
   context->RespondSuccess();
 }
 
+bool TabletServiceImpl::SupportsFeature(uint32_t feature) const {
+  return feature == TabletServerFeatures::COLUMN_PREDICATES;
+}
+
 void TabletServiceImpl::Shutdown() {
 }
 
@@ -1243,15 +1248,72 @@ static Status SetupScanSpec(const NewScanRequestPB& scan_pb,
 
   unordered_set<string> missing_col_names;
 
-  // First the column range predicates.
-  for (const ColumnRangePredicatePB& pred_pb : scan_pb.range_predicates()) {
-    if (!pred_pb.has_lower_bound() && !pred_pb.has_upper_bound()) {
+  // First the column predicates.
+  for (const ColumnPredicatePB& pred_pb : scan_pb.column_predicates()) {
+    if (!pred_pb.has_column()) {
+      return Status::InvalidArgument("Column predicate must include a column",
+                                     pred_pb.DebugString());
+    }
+
+    const string& column = pred_pb.column();
+    int32_t idx = tablet_schema.find_column(column);
+    if (idx == Schema::kColumnNotFound) {
+      return Status::InvalidArgument("Unknown column in predicate", pred_pb.DebugString());
+    }
+    const ColumnSchema& col = tablet_schema.column(idx);
+
+    if (projection.find_column(column) == Schema::kColumnNotFound &&
+        !ContainsKey(missing_col_names, column)) {
+      InsertOrDie(&missing_col_names, column);
+      missing_cols->push_back(col);
+    }
+
+    switch (pred_pb.predicate_case()) {
+      case ColumnPredicatePB::kRange: {
+        const auto& range = pred_pb.range();
+        if (!range.has_lower() && !range.has_upper()) {
+          return Status::InvalidArgument("Invalid range predicate on column: no bounds",
+                                         col.name());
+        }
+
+        const void* lower = nullptr;
+        const void* upper = nullptr;
+        if (range.has_lower()) {
+          RETURN_NOT_OK(ExtractPredicateValue(col, range.lower(), scanner->arena(), &lower));
+        }
+        if (range.has_upper()) {
+          RETURN_NOT_OK(ExtractPredicateValue(col, range.upper(), scanner->arena(), &upper));
+        }
+
+        ret->AddPredicate(ColumnPredicate::Range(col, lower, upper));
+        break;
+      };
+      case ColumnPredicatePB::kEquality: {
+        const auto& equality = pred_pb.equality();
+        if (!equality.has_value()) {
+          return Status::InvalidArgument("Invalid equality predicate on column: no value",
+                                         col.name());
+        }
+        const void* value = nullptr;
+        RETURN_NOT_OK(ExtractPredicateValue(col, equality.value(), scanner->arena(), &value));
+        ret->AddPredicate(ColumnPredicate::Equality(col, value));
+        break;
+      };
+      default: return Status::InvalidArgument("Unknown predicate type for column", col.name());
+    }
+  }
+
+  // Then the column range predicates.
+  // TODO: remove this once all clients have moved to ColumnPredicatePB and
+  // backwards compatibility can be broken.
+  for (const ColumnRangePredicatePB& pred_pb : scan_pb.deprecated_range_predicates())
{
+    if (!pred_pb.has_lower_bound() && !pred_pb.has_inclusive_upper_bound()) {
       return Status::InvalidArgument(
         string("Invalid predicate ") + pred_pb.ShortDebugString() +
         ": has no lower or upper bound.");
     }
     ColumnSchema col(ColumnSchemaFromPB(pred_pb.column()));
-    if (projection.find_column(col.name()) == -1 &&
+    if (projection.find_column(col.name()) == Schema::kColumnNotFound &&
         !ContainsKey(missing_col_names, col.name())) {
       missing_cols->push_back(col);
       InsertOrDie(&missing_col_names, col.name());
@@ -1265,24 +1327,23 @@ static Status SetupScanSpec(const NewScanRequestPB& scan_pb,
                                           scanner->arena(),
                                           &val));
       lower_bound = val;
-    } else {
-      lower_bound = nullptr;
     }
-    if (pred_pb.has_upper_bound()) {
+    if (pred_pb.has_inclusive_upper_bound()) {
       const void* val;
-      RETURN_NOT_OK(ExtractPredicateValue(col, pred_pb.upper_bound(),
+      RETURN_NOT_OK(ExtractPredicateValue(col, pred_pb.inclusive_upper_bound(),
                                           scanner->arena(),
                                           &val));
       upper_bound = val;
-    } else {
-      upper_bound = nullptr;
     }
 
-    ColumnRangePredicate pred(col, lower_bound, upper_bound);
-    if (VLOG_IS_ON(3)) {
-      VLOG(3) << "Parsed predicate " << pred.ToString() << " from " <<
scan_pb.ShortDebugString();
+    auto pred = ColumnPredicate::InclusiveRange(col, lower_bound, upper_bound, scanner->arena());
+    if (pred) {
+      if (VLOG_IS_ON(3)) {
+        VLOG(3) << "Parsed predicate " << pred->ToString()
+                << " from " << scan_pb.ShortDebugString();
+      }
+      ret->AddPredicate(*pred);
     }
-    ret->AddPredicate(pred);
   }
 
   // When doing an ordered scan, we need to include the key columns to be able to encode
@@ -1367,6 +1428,16 @@ Status TabletServiceImpl::HandleNewScanRequest(TabletPeer* tablet_peer,
     return s;
   }
 
+  VLOG(3) << "Before optimizing scan spec: " << spec->ToString(tablet_schema);
+  spec->OptimizeScan(tablet_schema, scanner->arena(), scanner->autorelease_pool(),
true);
+  VLOG(3) << "After optimizing scan spec: " << spec->ToString(tablet_schema);
+
+  if (spec->CanShortCircuit()) {
+    VLOG(1) << "short-circuiting without creating a server-side scanner.";
+    *has_more_results = false;
+    return Status::OK();
+  }
+
   // Store the original projection.
   gscoped_ptr<Schema> orig_projection(new Schema(projection));
   scanner->set_client_projection_schema(std::move(orig_projection));

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53e67e9e/src/kudu/tserver/tablet_service.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_service.h b/src/kudu/tserver/tablet_service.h
index 8f5846e..a3fcc59 100644
--- a/src/kudu/tserver/tablet_service.h
+++ b/src/kudu/tserver/tablet_service.h
@@ -71,6 +71,8 @@ class TabletServiceImpl : public TabletServerServiceIf {
                         ChecksumResponsePB* resp,
                         rpc::RpcContext* context) OVERRIDE;
 
+  bool SupportsFeature(uint32_t feature) const override;
+
   virtual void Shutdown() OVERRIDE;
 
  private:

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53e67e9e/src/kudu/tserver/tserver-path-handlers.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver-path-handlers.cc b/src/kudu/tserver/tserver-path-handlers.cc
index 96dc5e9..67afcb6 100644
--- a/src/kudu/tserver/tserver-path-handlers.cc
+++ b/src/kudu/tserver/tserver-path-handlers.cc
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+#include "kudu/common/scan_spec.h"
 #include "kudu/consensus/log_anchor_registry.h"
 #include "kudu/consensus/quorum_util.h"
 #include "kudu/gutil/map-util.h"
@@ -444,8 +445,13 @@ string TabletServerPathHandlers::ScannerToHtml(const Scanner& scanner)
const {
       range_pred_str = EncodedKey::RangeToString(spec.lower_bound_key(),
                                                  spec.exclusive_upper_bound_key());
     }
-    for (const ColumnRangePredicate& pred : scanner.spec().predicates()) {
-      other_preds.push_back(pred.ToString());
+    for (const auto& col_pred : scanner.spec().predicates()) {
+      int32_t col_idx = projection->find_column(col_pred.first);
+      if (col_idx == Schema::kColumnNotFound) {
+        other_preds.emplace_back("unknown column");
+      } else {
+        other_preds.push_back(col_pred.second.ToString());
+      }
     }
     string other_pred_str = JoinStrings(other_preds, "\n");
     html << Substitute("<td>$0</td><td>$1</td></tr>\n",

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/53e67e9e/src/kudu/tserver/tserver.proto
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tserver.proto b/src/kudu/tserver/tserver.proto
index 5d7933e..eee0752 100644
--- a/src/kudu/tserver/tserver.proto
+++ b/src/kudu/tserver/tserver.proto
@@ -168,6 +168,8 @@ message ListTabletsResponsePB {
   repeated StatusAndSchemaPB status_and_schema = 2;
 }
 
+// DEPRECATED: Use ColumnPredicatePB
+//
 // A range predicate on one of the columns in the underlying
 // data.
 message ColumnRangePredicatePB {
@@ -182,8 +184,10 @@ message ColumnRangePredicatePB {
   // NULL is defined to neither be greater than or less than other values
   // for the comparison operator. We will eventually add a special
   // predicate type for null-ness.
+  //
+  // Both bounds are inclusive.
   optional bytes lower_bound = 2;
-  optional bytes upper_bound = 3;
+  optional bytes inclusive_upper_bound = 3;
 }
 
 // List of predicates used by the Java client. Will rapidly evolve into something more reusable
@@ -201,8 +205,13 @@ message NewScanRequestPB {
   // itself after reaching this number of result rows.
   optional uint64 limit = 2;
 
+  // DEPRECATED: use column_predicates field.
+  //
   // Any column range predicates to enforce.
-  repeated ColumnRangePredicatePB range_predicates = 3;
+  repeated ColumnRangePredicatePB DEPRECATED_range_predicates = 3;
+
+  // Column predicates to enforce.
+  repeated ColumnPredicatePB column_predicates = 13;
 
   // Encoded primary key to begin scanning at (inclusive).
   optional bytes start_primary_key = 8;
@@ -329,4 +338,9 @@ message ScannerKeepAliveRequestPB {
 message ScannerKeepAliveResponsePB {
   // The error, if an error occurred with this request.
   optional TabletServerErrorPB error = 1;
-}
\ No newline at end of file
+}
+
+enum TabletServerFeatures {
+  UNKNOWN_FEATURE = 0;
+  COLUMN_PREDICATES = 1;
+}


Mime
View raw message