impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sail...@apache.org
Subject [5/5] incubator-impala git commit: IMPALA-6128: Spill-to-disk Encryption(AES-CFB + SHA256) is slow
Date Sun, 19 Nov 2017 22:55:58 GMT
IMPALA-6128: Spill-to-disk Encryption(AES-CFB + SHA256) is slow

CFB mode is stream cipher and is secure when use different nonce/IV
for every message. However it would be a performance bottleneck.
CTR mode is also stream cipher and is secure, 4~6x faster
than CFB mode in OpenSSL.

AES-CTR+SHA256 is about 40~70% faster than AES-CFB+SHA256

Testing:
run runtime tmp-file-mgr-test, openssl-util-test, buffer-pool-test and buffered-tuple-stream-test

Change-Id: Ib97939f2334838263364b53ef3413871638bf53e
Reviewed-on: http://gerrit.cloudera.org:8080/8510
Reviewed-by: Sailesh Mukil <sailesh@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fb4c3b01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fb4c3b01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fb4c3b01

Branch: refs/heads/master
Commit: fb4c3b01240d8f65fc2c45bf27b668ae9b1fa5d2
Parents: 5e9b4e2
Author: Xianda Ke <kexianda@gmail.com>
Authored: Tue Nov 7 23:02:46 2017 +0800
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Sun Nov 19 22:21:03 2017 +0000

----------------------------------------------------------------------
 be/src/runtime/tmp-file-mgr.cc |  2 +-
 be/src/util/openssl-util.cc    | 24 ++++++++++++++++--------
 be/src/util/openssl-util.h     |  7 +++++--
 3 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fb4c3b01/be/src/runtime/tmp-file-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index 24217de..ac165a7 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -612,7 +612,7 @@ void TmpFileMgr::WriteHandle::WaitForWrite() {
 Status TmpFileMgr::WriteHandle::EncryptAndHash(MemRange buffer) {
   DCHECK(FLAGS_disk_spill_encryption);
   SCOPED_TIMER(encryption_timer_);
-  // Since we're using AES-CFB mode, we must take care not to reuse a key/IV pair.
+  // Since we're using AES-CTR mode, we must take care not to reuse a key/IV pair.
   // Regenerate a new key and IV for every data buffer we write.
   key_.InitializeRandom();
   RETURN_IF_ERROR(key_.Encrypt(buffer.data(), buffer.len(), buffer.data()));

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fb4c3b01/be/src/util/openssl-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/openssl-util.cc b/be/src/util/openssl-util.cc
index e3b2299..8eef4be 100644
--- a/be/src/util/openssl-util.cc
+++ b/be/src/util/openssl-util.cc
@@ -99,13 +99,15 @@ Status EncryptionKey::EncryptInternal(
   EVP_CIPHER_CTX_init(&ctx);
   EVP_CIPHER_CTX_set_padding(&ctx, 0);
 
-  int success;
-
   // Start encryption/decryption.  We use a 256-bit AES key, and the cipher block mode
-  // is CFB because this gives us a stream cipher, which supports arbitrary
-  // length ciphertexts - it doesn't have to be a multiple of 16 bytes.
-  success = encrypt ? EVP_EncryptInit_ex(&ctx, EVP_aes_256_cfb(), NULL, key_, iv_) :
-                      EVP_DecryptInit_ex(&ctx, EVP_aes_256_cfb(), NULL, key_, iv_);
+  // is CTR. CTR is a stream cipher, which supports
+  //   (1). arbitrary length ciphertexts - it doesn't have to be a multiple of 16 bytes.
+  //   (2). well-optimized(instruction level parallelism) with
+  //        hardware acceleration on x86 or PowerPC
+  const EVP_CIPHER* evpCipher = IsCtrSupported() ? EVP_aes_256_ctr() : EVP_aes_256_cfb();
+  int success = encrypt ? EVP_EncryptInit_ex(&ctx, evpCipher, NULL, key_, iv_) :
+                          EVP_DecryptInit_ex(&ctx, evpCipher, NULL, key_, iv_);
+
   if (success != 1) {
     return OpenSSLErr(encrypt ? "EVP_EncryptInit_ex" : "EVP_DecryptInit_ex");
   }
@@ -122,7 +124,7 @@ Status EncryptionKey::EncryptInternal(
     if (success != 1) {
       return OpenSSLErr(encrypt ? "EVP_EncryptUpdate" : "EVP_DecryptUpdate");
     }
-    // This is safe because we're using CFB mode without padding.
+    // This is safe because we're using CTR mode without padding.
     DCHECK_EQ(in_len, out_len);
     offset += in_len;
   }
@@ -134,8 +136,14 @@ Status EncryptionKey::EncryptInternal(
   if (success != 1) {
     return OpenSSLErr(encrypt ? "EVP_EncryptFinal" : "EVP_DecryptFinal");
   }
-  // Again safe due to CFB with no padding
+  // Again safe due to CTR with no padding
   DCHECK_EQ(final_out_len, 0);
   return Status::OK();
 }
+
+#define OPENSSL_VERSION_1_0_1 0x1000100L
+bool EncryptionKey::IsCtrSupported() const {
+  // aes_256_ctr was supported since v1.0.1
+  return (SSLeay() >= OPENSSL_VERSION_1_0_1);
+}
 }

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fb4c3b01/be/src/util/openssl-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/openssl-util.h b/be/src/util/openssl-util.h
index 4b32db6..cd1ca1b 100644
--- a/be/src/util/openssl-util.h
+++ b/be/src/util/openssl-util.h
@@ -47,7 +47,7 @@ class IntegrityHash {
 /// The key and initialization vector (IV) required to encrypt and decrypt a buffer of
 /// data. This should be regenerated for each buffer of data.
 ///
-/// We use AES with a 256-bit key and CFB cipher block mode, which gives us a stream
+/// We use AES with a 256-bit key and CTR cipher block mode, which gives us a stream
 /// cipher that can support arbitrary-length ciphertexts. The IV is used as an input to
 /// the cipher as the "block to supply before the first block of plaintext". This is
 /// required because all ciphers (except the weak ECB) are built such that each block
@@ -59,7 +59,7 @@ class EncryptionKey {
   EncryptionKey() : initialized_(false) {}
 
   /// Initialize a key for temporary use with randomly generated data. Reinitializes with
-  /// new random values if the key was already initialized. We use AES-CFB mode so key/IV
+  /// new random values if the key was already initialized. We use AES-CTR mode so key/IV
   /// pairs should not be reused. This function automatically reseeds the RNG
   /// periodically, so callers do not need to do it.
   void InitializeRandom();
@@ -84,6 +84,9 @@ class EncryptionKey {
   Status EncryptInternal(bool encrypt, const uint8_t* data, int64_t len,
       uint8_t* out) const WARN_UNUSED_RESULT;
 
+  /// check if CTR mode is supported at runtime
+  bool IsCtrSupported() const;
+
   /// Track whether this key has been initialized, to avoid accidentally using
   /// uninitialized keys.
   bool initialized_;


Mime
View raw message