kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [1/5] incubator-kudu git commit: [c++ client] - Expose the schema being scanned in KuduScanBatch
Date Thu, 04 Feb 2016 05:33:44 GMT
Repository: incubator-kudu
Updated Branches:
  refs/heads/master f00d20c76 -> 1a24338ad


[c++ client] - Expose the schema being scanned in KuduScanBatch

This exposes the schema being scanned in the KuduScanBatch and KuduScanBatch::RowPtr
allowing to pass pointer/references to these around when in use without an accompanying
schema, which was hard to infer in any case since the scanner projection is set with
just column names.

Note that this is just exposing the schema information that was already there, just
in KuduSchema form (vs. the internal Schema) which is more suited for user consumption.

Change-Id: Id4276036d5ba7faf7959b4324fd013b68a08bec5
Reviewed-on: http://gerrit.cloudera.org:8080/2020
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <adar@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/403f9da9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/403f9da9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/403f9da9

Branch: refs/heads/master
Commit: 403f9da9619aba0f84fb6729868edf2a88063cc9
Parents: f00d20c
Author: David Alves <david.alves@cloudera.com>
Authored: Wed Feb 3 13:47:24 2016 -0800
Committer: David Ribeiro Alves <david.alves@cloudera.com>
Committed: Thu Feb 4 03:55:10 2016 +0000

----------------------------------------------------------------------
 src/kudu/client/client.cc           |  4 +++-
 src/kudu/client/scan_batch.cc       |  9 +++++++++
 src/kudu/client/scan_batch.h        | 15 +++++++++++++--
 src/kudu/client/scanner-internal.cc | 12 ++++++++++--
 src/kudu/client/scanner-internal.h  | 12 +++++++++++-
 src/kudu/client/schema.h            |  7 ++++++-
 src/kudu/tools/ts-cli.cc            |  6 +++++-
 7 files changed, 57 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc
index fc47110..2e3b059 100644
--- a/src/kudu/client/client.cc
+++ b/src/kudu/client/client.cc
@@ -876,7 +876,7 @@ Status KuduScanner::SetProjectedColumnIndexes(const vector<int>&
col_indexes) {
 
   gscoped_ptr<Schema> s(new Schema());
   RETURN_NOT_OK(s->Reset(cols, 0));
-  data_->projection_ = data_->pool_.Add(s.release());
+  data_->SetProjectionSchema(data_->pool_.Add(s.release()));
   return Status::OK();
 }
 
@@ -1156,6 +1156,7 @@ Status KuduScanner::NextBatch(KuduScanBatch* result) {
     data_->data_in_open_ = false;
     return result->data_->Reset(&data_->controller_,
                                 data_->projection_,
+                                &data_->client_projection_,
                                 make_gscoped_ptr(data_->last_response_.release_data()));
   } else if (data_->last_response_.has_more_results()) {
     // More data is available in this tablet.
@@ -1197,6 +1198,7 @@ Status KuduScanner::NextBatch(KuduScanBatch* result) {
       data_->scan_attempts_ = 0;
       return result->data_->Reset(&data_->controller_,
                                   data_->projection_,
+                                  &data_->client_projection_,
                                   make_gscoped_ptr(data_->last_response_.release_data()));
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/scan_batch.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scan_batch.cc b/src/kudu/client/scan_batch.cc
index f0a1bd7..fef9c98 100644
--- a/src/kudu/client/scan_batch.cc
+++ b/src/kudu/client/scan_batch.cc
@@ -18,6 +18,7 @@
 #include "kudu/client/row_result.h"
 #include "kudu/client/scan_batch.h"
 #include "kudu/client/scanner-internal.h"
+#include "kudu/client/schema.h"
 
 #include <string>
 
@@ -49,6 +50,10 @@ KuduRowResult KuduScanBatch::Row(int idx) const {
   return data_->row(idx);
 }
 
+const KuduSchema* KuduScanBatch::projection_schema() const {
+  return data_->client_projection_;
+}
+
 ////////////////////////////////////////////////////////////
 // KuduScanBatch::RowPtr
 ////////////////////////////////////////////////////////////
@@ -212,6 +217,10 @@ const void* KuduScanBatch::RowPtr::cell(int col_idx) const {
   return row_data_ + schema_->column_offset(col_idx);
 }
 
+const KuduSchema* KuduScanBatch::RowPtr::row_schema() const {
+  return client_schema_;
+}
+
 //------------------------------------------------------------
 // Template instantiations: We instantiate all possible templates to avoid linker issues.
 // see: https://isocpp.org/wiki/faq/templates#separate-template-fn-defn-from-decl

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/scan_batch.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/scan_batch.h b/src/kudu/client/scan_batch.h
index 3d510d4..073b5e3 100644
--- a/src/kudu/client/scan_batch.h
+++ b/src/kudu/client/scan_batch.h
@@ -37,6 +37,7 @@ class TsAdminClient;
 } // namespace tools
 
 namespace client {
+class KuduSchema;
 
 // A batch of zero or more rows returned from a KuduScanner.
 //
@@ -78,6 +79,10 @@ class KUDU_EXPORT KuduScanBatch {
   const_iterator begin() const;
   const_iterator end() const;
 
+  // Returns the projection schema for this batch.
+  // All KuduScanBatch::RowPtr returned by this batch are guaranteed to have this schema.
+  const KuduSchema* projection_schema() const;
+
  private:
   class KUDU_NO_EXPORT Data;
   friend class KuduScanner;
@@ -138,6 +143,8 @@ class KUDU_EXPORT KuduScanBatch::RowPtr {
   // Raw cell access. Should be avoided unless absolutely necessary.
   const void* cell(int col_idx) const;
 
+  const KuduSchema* row_schema() const;
+
   std::string ToString() const;
 
  private:
@@ -146,9 +153,12 @@ class KUDU_EXPORT KuduScanBatch::RowPtr {
   template<typename KeyTypeWrapper> friend struct IntKeysTestSetup;
 
   // Only invoked by KuduScanner.
-  RowPtr(const Schema* schema, const uint8_t* row_data)
+  RowPtr(const Schema* schema,
+         const KuduSchema* client_projection,
+         const uint8_t* row_data)
       : schema_(schema),
-      row_data_(row_data) {
+        client_schema_(client_projection),
+        row_data_(row_data) {
   }
 
   template<typename T>
@@ -158,6 +168,7 @@ class KUDU_EXPORT KuduScanBatch::RowPtr {
   Status Get(int col_idx, typename T::cpp_type* val) const;
 
   const Schema* schema_;
+  const KuduSchema* client_schema_;
   const uint8_t* row_data_;
 };
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/scanner-internal.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/scanner-internal.cc b/src/kudu/client/scanner-internal.cc
index a25a486..68b1ca8 100644
--- a/src/kudu/client/scanner-internal.cc
+++ b/src/kudu/client/scanner-internal.cc
@@ -60,11 +60,11 @@ KuduScanner::Data::Data(KuduTable* table)
     is_fault_tolerant_(false),
     snapshot_timestamp_(kNoTimestamp),
     table_(DCHECK_NOTNULL(table)),
-    projection_(table->schema().schema_),
     arena_(1024, 1024*1024),
     spec_encoder_(table->schema().schema_, &arena_),
     timeout_(MonoDelta::FromMilliseconds(kScanTimeoutMillis)),
     scan_attempts_(0) {
+  SetProjectionSchema(table->schema().schema_);
 }
 
 KuduScanner::Data::~Data() {
@@ -456,6 +456,12 @@ void KuduScanner::Data::UpdateLastError(const Status& error) {
   }
 }
 
+void KuduScanner::Data::SetProjectionSchema(const Schema* schema) {
+  projection_ = schema;
+  client_projection_ = KuduSchema(*schema);
+}
+
+
 
 ////////////////////////////////////////////////////////////
 // KuduScanBatch
@@ -472,10 +478,12 @@ size_t KuduScanBatch::Data::CalculateProjectedRowSize(const Schema&
proj) {
 
 Status KuduScanBatch::Data::Reset(RpcController* controller,
                                   const Schema* projection,
+                                  const KuduSchema* client_projection,
                                   gscoped_ptr<RowwiseRowBlockPB> data) {
   CHECK(controller->finished());
   controller_.Swap(controller);
   projection_ = projection;
+  client_projection_ = client_projection;
   resp_data_.Swap(data.get());
 
   // First, rewrite the relative addresses into absolute ones.
@@ -520,7 +528,7 @@ void KuduScanBatch::Data::ExtractRows(vector<KuduScanBatch::RowPtr>*
rows) {
   const uint8_t* src = direct_data_.data();
   KuduScanBatch::RowPtr* dst = &(*rows)[0];
   while (n_rows > 0) {
-    *dst = KuduScanBatch::RowPtr(projection_, src);
+    *dst = KuduScanBatch::RowPtr(projection_, client_projection_,src);
     dst++;
     src += projected_row_size_;
     n_rows--;

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/scanner-internal.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/scanner-internal.h b/src/kudu/client/scanner-internal.h
index d25c7ed..0c53005 100644
--- a/src/kudu/client/scanner-internal.h
+++ b/src/kudu/client/scanner-internal.h
@@ -94,6 +94,9 @@ class KuduScanner::Data {
   // non-fatal (i.e. retriable) scan error is encountered.
   void UpdateLastError(const Status& error);
 
+  // Sets the projection schema.
+  void SetProjectionSchema(const Schema* schema);
+
   bool open_;
   bool data_in_open_;
   bool has_batch_size_bytes_;
@@ -129,6 +132,10 @@ class KuduScanner::Data {
   // The projection schema used in the scan.
   const Schema* projection_;
 
+  // 'projection_' after it is converted to KuduSchema, so that users can obtain
+  // the projection without having to include common/schema.h.
+  KuduSchema client_projection_;
+
   Arena arena_;
   AutoReleasePool pool_;
 
@@ -168,6 +175,7 @@ class KuduScanBatch::Data {
 
   Status Reset(rpc::RpcController* controller,
                const Schema* projection,
+               const KuduSchema* client_projection,
                gscoped_ptr<RowwiseRowBlockPB> resp_data);
 
   int num_rows() const {
@@ -178,7 +186,7 @@ class KuduScanBatch::Data {
     DCHECK_GE(idx, 0);
     DCHECK_LT(idx, num_rows());
     int offset = idx * projected_row_size_;
-    return KuduRowResult(projection_, &direct_data_[offset]);
+    return KuduRowResult(projection_, client_projection_, &direct_data_[offset]);
   }
 
   void ExtractRows(vector<KuduScanBatch::RowPtr>* rows);
@@ -202,6 +210,8 @@ class KuduScanBatch::Data {
 
   // The projection being scanned.
   const Schema* projection_;
+  // The KuduSchema version of 'projection_'
+  const KuduSchema* client_projection_;
 
   // The number of bytes of direct data for each row.
   size_t projected_row_size_;

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/client/schema.h
----------------------------------------------------------------------
diff --git a/src/kudu/client/schema.h b/src/kudu/client/schema.h
index 5e18699..6bdeb0c 100644
--- a/src/kudu/client/schema.h
+++ b/src/kudu/client/schema.h
@@ -30,6 +30,11 @@ namespace kudu {
 class ColumnSchema;
 class KuduPartialRow;
 class Schema;
+class TestWorkload;
+
+namespace tools {
+class TsAdminClient;
+}
 
 namespace client {
 
@@ -283,7 +288,6 @@ class KUDU_EXPORT KuduSchemaBuilder {
   Data* data_;
 };
 
-
 class KUDU_EXPORT KuduSchema {
  public:
   KuduSchema();
@@ -326,6 +330,7 @@ class KUDU_EXPORT KuduSchema {
   friend class internal::GetTableSchemaRpc;
   friend class internal::LookupRpc;
   friend class internal::WriteRpc;
+  friend class kudu::tools::TsAdminClient;
 
   friend KuduSchema KuduSchemaFromSchema(const Schema& schema);
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/403f9da9/src/kudu/tools/ts-cli.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ts-cli.cc b/src/kudu/tools/ts-cli.cc
index d0f2eb8..80bcda3 100644
--- a/src/kudu/tools/ts-cli.cc
+++ b/src/kudu/tools/ts-cli.cc
@@ -248,6 +248,7 @@ Status TsAdminClient::DumpTablet(const std::string& tablet_id) {
   RETURN_NOT_OK(GetTabletSchema(tablet_id, &schema_pb));
   Schema schema;
   RETURN_NOT_OK(SchemaFromPB(schema_pb, &schema));
+  kudu::client::KuduSchema client_schema(schema);
 
   ScanRequestPB req;
   ScanResponsePB resp;
@@ -274,7 +275,10 @@ Status TsAdminClient::DumpTablet(const std::string& tablet_id) {
 
     rows.clear();
     KuduScanBatch::Data results;
-    RETURN_NOT_OK(results.Reset(&rpc, &schema, make_gscoped_ptr(resp.release_data())));
+    RETURN_NOT_OK(results.Reset(&rpc,
+                                &schema,
+                                &client_schema,
+                                make_gscoped_ptr(resp.release_data())));
     results.ExtractRows(&rows);
     for (const KuduRowResult& r : rows) {
       std::cout << r.ToString() << std::endl;


Mime
View raw message