kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wdberke...@apache.org
Subject kudu git commit: KUDU-2035 Enable HTTP compression for all webserver's paths
Date Fri, 18 May 2018 22:18:10 GMT
Repository: kudu
Updated Branches:
  refs/heads/master 185781f6c -> 8f525829d


KUDU-2035 Enable HTTP compression for all webserver's paths

This patch enables HTTP compression when gzip compression is
accepted by the client.

Change-Id: I7c6db24b0fd2cbcca8a554460d310bd39ee5c071
Reviewed-on: http://gerrit.cloudera.org:8080/10332
Reviewed-by: Alexey Serbin <aserbin@cloudera.com>
Tested-by: Will Berkeley <wdberkeley@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8f525829
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8f525829
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8f525829

Branch: refs/heads/master
Commit: 8f525829dcfc49edea9fadbe7514db2ea64f2087
Parents: 185781f
Author: fwang29 <fwang@cloudera.com>
Authored: Thu May 17 13:33:50 2018 -0700
Committer: Will Berkeley <wdberkeley@gmail.com>
Committed: Fri May 18 21:23:08 2018 +0000

----------------------------------------------------------------------
 .../integration-tests/linked_list-test-util.h   |  6 +-
 src/kudu/server/webserver-test.cc               | 62 +++++++++++++++++++-
 src/kudu/server/webserver.cc                    | 22 +++++++
 src/kudu/util/curl_util.cc                      | 21 ++++++-
 src/kudu/util/curl_util.h                       |  9 ++-
 5 files changed, 113 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/8f525829/src/kudu/integration-tests/linked_list-test-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/linked_list-test-util.h b/src/kudu/integration-tests/linked_list-test-util.h
index ba04ea6..252d640 100644
--- a/src/kudu/integration-tests/linked_list-test-util.h
+++ b/src/kudu/integration-tests/linked_list-test-util.h
@@ -374,8 +374,12 @@ class PeriodicWebUIChecker {
     while (is_running_.Load()) {
       // Poll all of the URLs.
       const MonoTime start = MonoTime::Now();
+      bool compression_enabled = true;
       for (const auto& url : urls_) {
-        Status s = curl.FetchURL(url, &dst);
+        // Switch compression back and forth.
+        Status s = compression_enabled ? curl.FetchURL(url, &dst, {"Accept-Encoding:
gzip"})
+                                       : curl.FetchURL(url, &dst);
+        compression_enabled = !compression_enabled;
         if (s.ok()) {
           CHECK_GT(dst.length(), 0);
         }

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f525829/src/kudu/server/webserver-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/server/webserver-test.cc b/src/kudu/server/webserver-test.cc
index 7c53a4b..aa4da52 100644
--- a/src/kudu/server/webserver-test.cc
+++ b/src/kudu/server/webserver-test.cc
@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "kudu/server/webserver.h"
+
+#include <iosfwd>
 #include <memory>
 #include <string>
 #include <vector>
@@ -32,7 +35,6 @@
 #include "kudu/security/test/test_certs.h"
 #include "kudu/security/test/test_pass.h"
 #include "kudu/server/default_path_handlers.h"
-#include "kudu/server/webserver.h"
 #include "kudu/server/webserver_options.h"
 #include "kudu/util/curl_util.h"
 #include "kudu/util/env.h"
@@ -44,6 +46,7 @@
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
+#include "kudu/util/zlib.h"
 
 using std::string;
 using std::vector;
@@ -149,6 +152,63 @@ TEST_F(WebserverTest, TestIndexPage) {
   ASSERT_STR_CONTAINS(buf_.ToString(), "memz");
 }
 
+TEST_F(WebserverTest, TestHttpCompression) {
+  string url = strings::Substitute("http://$0/", addr_.ToString());
+  std::ostringstream oss;
+  string decoded_str;
+
+  // Curl with gzip compression enabled.
+  ASSERT_OK(curl_.FetchURL(url, &buf_, {"Accept-Encoding: deflate, br, gzip"}));
+
+  // If compressed successfully, we should be able to uncompress.
+  ASSERT_OK(zlib::Uncompress(Slice(buf_.ToString()), &oss));
+  decoded_str = oss.str();
+
+  // Should have expected title.
+  ASSERT_STR_CONTAINS(decoded_str, "Kudu");
+
+  // Should have link to default path handlers (e.g memz)
+  ASSERT_STR_CONTAINS(decoded_str, "memz");
+
+  // Should have expected header when compressed with headers returned.
+  curl_.set_return_headers(true);
+  ASSERT_OK(curl_.FetchURL(url, &buf_,
+                          {"Accept-Encoding: deflate, megaturbogzip,  gzip , br"}));
+  ASSERT_STR_CONTAINS(buf_.ToString(), "Content-Encoding: gzip");
+
+
+  // Curl with compression disabled.
+  curl_.set_return_headers(true);
+  ASSERT_OK(curl_.FetchURL(url, &buf_));
+  // Check expected header.
+  ASSERT_STR_CONTAINS(buf_.ToString(), "Content-Type:");
+
+  // Check unexpected header.
+  ASSERT_STR_NOT_CONTAINS(buf_.ToString(), "Content-Encoding: gzip");
+
+  // Should have expected title.
+  ASSERT_STR_CONTAINS(buf_.ToString(), "Kudu");
+
+  // Should have link to default path handlers (e.g memz)
+  ASSERT_STR_CONTAINS(buf_.ToString(), "memz");
+
+
+  // Curl with compression enabled but not accepted by Kudu.
+  curl_.set_return_headers(true);
+  ASSERT_OK(curl_.FetchURL(url, &buf_, {"Accept-Encoding: megaturbogzip, deflate, xz"}));
+  // Check expected header.
+  ASSERT_STR_CONTAINS(buf_.ToString(), "HTTP/1.1 200 OK");
+
+  // Check unexpected header.
+  ASSERT_STR_NOT_CONTAINS(buf_.ToString(), "Content-Encoding: gzip");
+
+  // Should have expected title.
+  ASSERT_STR_CONTAINS(buf_.ToString(), "Kudu");
+
+  // Should have link to default path handlers (e.g memz)
+  ASSERT_STR_CONTAINS(buf_.ToString(), "memz");
+}
+
 TEST_F(SslWebserverTest, TestSSL) {
   // We use a self-signed cert, so we need to disable cert verification in curl.
   curl_.set_verify_peer(false);

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f525829/src/kudu/server/webserver.cc
----------------------------------------------------------------------
diff --git a/src/kudu/server/webserver.cc b/src/kudu/server/webserver.cc
index 3016650..516cd89 100644
--- a/src/kudu/server/webserver.cc
+++ b/src/kudu/server/webserver.cc
@@ -45,6 +45,7 @@
 #include "kudu/gutil/strings/numbers.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/stringpiece.h"
+#include "kudu/gutil/strings/strip.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/security/openssl_util.h"
 #include "kudu/util/easy_json.h"
@@ -56,6 +57,7 @@
 #include "kudu/util/net/sockaddr.h"
 #include "kudu/util/url-coding.h"
 #include "kudu/util/version_info.h"
+#include "kudu/util/zlib.h"
 
 struct sockaddr_in;
 
@@ -477,10 +479,30 @@ int Webserver::RunPathHandler(const PathHandler& handler,
     full_content = content.str();
   }
 
+  // Check if the gzip compression is accepted by the caller. If so, compress the content.
+  const char* accept_encoding_str = sq_get_header(connection, "Accept-Encoding");
+  bool is_compressed = false;
+  vector<string> encodings = strings::Split(accept_encoding_str, ",");
+  for (string& encoding : encodings) {
+    StripWhiteSpace(&encoding);
+    if (encoding == "gzip") {
+      ostringstream oss;
+      Status s = zlib::Compress(Slice(full_content), &oss);
+      if (s.ok()) {
+        full_content = oss.str();
+        is_compressed = true;
+      } else {
+        LOG(WARNING) << "Could not compress output: " << s.ToString();
+      }
+      break;
+    }
+  }
+
   ostringstream headers_stream;
   headers_stream << Substitute("HTTP/1.1 $0\r\n", HttpStatusCodeToString(resp.status_code));
   headers_stream << Substitute("Content-Type: $0\r\n", use_style ? "text/html" : "text/plain");
   headers_stream << Substitute("Content-Length: $0\r\n", full_content.length());
+  if (is_compressed) headers_stream << "Content-Encoding: gzip\r\n";
   headers_stream << Substitute("X-Frame-Options: $0\r\n", FLAGS_webserver_x_frame_options);
   std::unordered_set<string> invalid_headers{"Content-Type", "Content-Length", "X-Frame-Options"};
   for (const auto& entry : resp.response_headers) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f525829/src/kudu/util/curl_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/curl_util.cc b/src/kudu/util/curl_util.cc
index d2aa75b..4eddb64 100644
--- a/src/kudu/util/curl_util.cc
+++ b/src/kudu/util/curl_util.cc
@@ -27,6 +27,7 @@
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/security/openssl_util.h"
 #include "kudu/util/faststring.h"
+#include "kudu/util/scoped_cleanup.h"
 
 namespace kudu {
 
@@ -63,8 +64,9 @@ EasyCurl::~EasyCurl() {
   curl_easy_cleanup(curl_);
 }
 
-Status EasyCurl::FetchURL(const std::string& url, faststring* buf) {
-  return DoRequest(url, nullptr, buf);
+Status EasyCurl::FetchURL(const std::string& url, faststring* dst,
+                          const std::vector<std::string>& headers) {
+  return DoRequest(url, nullptr, dst, headers);
 }
 
 Status EasyCurl::PostToURL(const std::string& url,
@@ -75,7 +77,8 @@ Status EasyCurl::PostToURL(const std::string& url,
 
 Status EasyCurl::DoRequest(const std::string& url,
                            const std::string* post_data,
-                           faststring* dst) {
+                           faststring* dst,
+                           const std::vector<std::string>& headers) {
   CHECK_NOTNULL(dst)->clear();
 
   if (!verify_peer_) {
@@ -84,6 +87,18 @@ Status EasyCurl::DoRequest(const std::string& url,
     RETURN_NOT_OK(TranslateError(curl_easy_setopt(
         curl_, CURLOPT_SSL_VERIFYPEER, 0)));
   }
+
+  // Add headers if specified.
+  struct curl_slist* curl_headers = nullptr;
+  auto clean_up_curl_slist = MakeScopedCleanup([&]() {
+    curl_slist_free_all(curl_headers);
+  });
+
+  for (const auto& header : headers) {
+    curl_headers = CHECK_NOTNULL(curl_slist_append(curl_headers, header.c_str()));
+  }
+  RETURN_NOT_OK(TranslateError(curl_easy_setopt(curl_, CURLOPT_HTTPHEADER, curl_headers)));
+
   RETURN_NOT_OK(TranslateError(curl_easy_setopt(curl_, CURLOPT_URL, url.c_str())));
   if (return_headers_) {
     RETURN_NOT_OK(TranslateError(curl_easy_setopt(curl_, CURLOPT_HEADER, 1)));

http://git-wip-us.apache.org/repos/asf/kudu/blob/8f525829/src/kudu/util/curl_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/curl_util.h b/src/kudu/util/curl_util.h
index 49ba2d4..cccd2db 100644
--- a/src/kudu/util/curl_util.h
+++ b/src/kudu/util/curl_util.h
@@ -18,6 +18,7 @@
 #define KUDU_UTIL_CURL_UTIL_H
 
 #include <string>
+#include <vector>
 
 #include "kudu/gutil/macros.h"
 #include "kudu/util/monotime.h"
@@ -40,8 +41,11 @@ class EasyCurl {
 
   // Fetch the given URL into the provided buffer.
   // Any existing data in the buffer is replaced.
+  // The optional param 'headers' holds additional headers.
+  // e.g. {"Accept-Encoding: gzip"}
   Status FetchURL(const std::string& url,
-                  faststring* dst);
+                  faststring* dst,
+                  const std::vector<std::string>& headers = {});
 
   // Issue an HTTP POST to the given URL with the given data.
   // Returns results in 'dst' as above.
@@ -68,7 +72,8 @@ class EasyCurl {
   // Otherwise, does a GET.
   Status DoRequest(const std::string& url,
                    const std::string* post_data,
-                   faststring* dst);
+                   faststring* dst,
+                   const std::vector<std::string>& headers = {});
   CURL* curl_;
 
   // Whether to verify the server certificate.


Mime
View raw message