kudu-commits mailing list archives

Fix ksck checksum_scan printing

I removed the checksum progress summary line when the ksck_format is
json_pretty or json_compact so that the checksum_scan output is
completely JSON-parsable. I also added a method and refactored the
code to do the checking of non-JSON format.

Example command:
kudu cluster ksck --checksum_scan --checksum_snapshot=false
--ksck_format=json_compact localhost:7052,localhost:7051,
localhost:7053

Before:
Checksum finished in 0s: 0/8 replicas remaining (0B from disk, 2.00k
rows summed)
{"master_summaries"..}

After:
{"master_summaries"..}

Change-Id: I25fe6d0f14d1713b848d0a79f4d92b056924a5a5
Reviewed-on: http://gerrit.cloudera.org:8080/10535
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wdberkeley@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5e705530
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5e705530
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5e705530

Branch: refs/heads/master
Commit: 5e705530cc34bedde1215dd5f50a624cabd658e8
Parents: c008731
Author: fwang29 <fwang@cloudera.com>
Authored: Tue Jun 12 16:33:14 2018 -0700
Committer: Will Berkeley <wdberkeley@gmail.com>
Committed: Thu Jun 14 17:29:32 2018 +0000

----------------------------------------------------------------------
 src/kudu/tools/ksck-test.cc | 10 ++++++++++
 src/kudu/tools/ksck.cc      | 21 +++++++++++++++------
 2 files changed, 25 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/5e705530/src/kudu/tools/ksck-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc
index 22d737f..273be4b 100644
--- a/src/kudu/tools/ksck-test.cc
+++ b/src/kudu/tools/ksck-test.cc
@@ -52,6 +52,7 @@
 
 DECLARE_bool(checksum_scan);
 DECLARE_string(color);
+DECLARE_string(ksck_format);
 DECLARE_uint32(truncate_server_csv_length);
 
 namespace kudu {
@@ -1493,5 +1494,14 @@ TEST_F(KsckTest, TestVersionCheck) {
   CheckJsonStringVsKsckResults(KsckResultsToJsonString(), ksck_->results());
 }
 
+TEST_F(KsckTest, TestChecksumScanJson) {
+  CreateOneTableOneTablet();
+  FLAGS_checksum_scan = true;
+  FLAGS_ksck_format = "json_compact";
+  ASSERT_OK(RunKsck());
+  JsonReader r(err_stream_.str());
+  ASSERT_OK(r.Init());
+}
+
 } // namespace tools
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/5e705530/src/kudu/tools/ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index c895059..eab1ac1 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -26,6 +26,7 @@
 #include <set>
 #include <vector>
 
+#include <boost/algorithm/string/predicate.hpp>
 #include <boost/optional.hpp> // IWYU pragma: keep
 #include <gflags/gflags.h>
 #include <glog/logging.h>
@@ -143,6 +144,12 @@ bool IsNotAuthorizedMethodAccess(const Status& s) {
          s.ToString().find("Not authorized: unauthorized access to method") != string::npos;
 }
 
+// Return whether the format of the ksck results is non-JSON.
+bool IsNonJSONFormat() {
+  return boost::iequals(FLAGS_ksck_format, "plain_full") ||
+         boost::iequals(FLAGS_ksck_format, "plain_concise");
+}
+
 } // anonymous namespace
 
 ChecksumOptions::ChecksumOptions()
@@ -613,12 +620,14 @@ class ChecksumResultReporter : public RefCountedThreadSafe<ChecksumResultReporte
 
       done = responses_.WaitFor(MonoDelta::FromMilliseconds(std::min(rem_ms, 5000)));
       string status = done ? "finished in " : "running for ";
-      int run_time_sec = (MonoTime::Now() - start).ToSeconds();
-      (*out) << "Checksum " << status << run_time_sec << "s: "
-             << responses_.count() << "/" << expected_count_ << "
replicas remaining ("
-             << HumanReadableNumBytes::ToString(disk_bytes_summed_.Load()) <<
" from disk, "
-             << HumanReadableInt::ToString(rows_summed_.Load()) << " rows summed)"
-             << endl;
+      if (IsNonJSONFormat()) {
+        int run_time_sec = (MonoTime::Now() - start).ToSeconds();
+        (*out) << "Checksum " << status << run_time_sec << "s: "
+               << responses_.count() << "/" << expected_count_ <<
" replicas remaining ("
+               << HumanReadableNumBytes::ToString(disk_bytes_summed_.Load()) <<
" from disk, "
+               << HumanReadableInt::ToString(rows_summed_.Load()) << " rows summed)"
+               << endl;
+      }
     }
     return true;
   }