Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5B1F718384 for ; Sat, 20 Feb 2016 04:02:17 +0000 (UTC) Received: (qmail 61791 invoked by uid 500); 20 Feb 2016 04:02:17 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 61746 invoked by uid 500); 20 Feb 2016 04:02:17 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 61730 invoked by uid 99); 20 Feb 2016 04:02:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 20 Feb 2016 04:02:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 20EB4E0092; Sat, 20 Feb 2016 04:02:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kturner@apache.org To: commits@accumulo.apache.org Date: Sat, 20 Feb 2016 04:02:17 -0000 Message-Id: <7c3ee1e6338e4310ae990241fe08e2f5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/3] accumulo git commit: ACCUMULO-4141 Prevent unnecessary reads of rfile header Repository: accumulo Updated Branches: refs/heads/master 82e25e894 -> 2651ba8e2 ACCUMULO-4141 Prevent unnecessary reads of rfile header Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b033b04f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b033b04f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b033b04f Branch: refs/heads/master Commit: b033b04fd01ebbc991944cf51fa748aa9456d140 Parents: eb0f9b4 Author: Keith Turner Authored: Thu Feb 18 12:31:44 2016 -0500 Committer: Keith Turner Committed: Fri Feb 19 22:04:28 2016 -0500 ---------------------------------------------------------------------- .../accumulo/core/file/rfile/bcfile/BCFile.java | 68 +++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b033b04f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java index ecc0b90..4767d91 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -54,6 +55,8 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; +import com.google.common.base.Charsets; + /** * Block Compressed file, the underlying physical storage layer for TFile. BCFile provides the basic block level compression for the data block and meta blocks. * It is separated from TFile as it may be used for other block-compressed file implementation. @@ -521,6 +524,30 @@ public final class BCFile { } } + // sha256 of some random data + private static final byte[] NO_CPYPTO_KEY = "ce18cf53c4c5077f771249b38033fa14bcb31cca0e5e95a371ee72daa8342ea2".getBytes(Charsets.UTF_8); + + // This class is used as a place holder in the cache for RFiles that have no crypto.... + private static final BCFileCryptoModuleParameters NO_CRYPTO = new BCFileCryptoModuleParameters() { + + @Override + public Map getAllOptions() { + return Collections.emptyMap(); + } + + @Override + public byte[] getEncryptedKey() { + return NO_CPYPTO_KEY; + } + + @Override + public String getOpaqueKeyEncryptionKeyID() { + // NONE + sha256 of random data + return "NONE:a4007e6aefb095a5a47030cd6c850818fb3a685dc6e85ba1ecc5a44ba68b193b"; + } + + }; + private static class BCFileCryptoModuleParameters extends CryptoModuleParameters { public void write(DataOutput out) throws IOException { @@ -888,17 +915,14 @@ public final class BCFile { cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams); } else if (cachedCryptoParams != null) { - cryptoParams = new BCFileCryptoModuleParameters(); - cryptoParams.read(cachedCryptoParams); - - this.cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoParams.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey())); - this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(cryptoParams.getKeyEncryptionStrategyClass()); - - // This call should put the decrypted session key within the cryptoParameters object - // secretKeyEncryptionStrategy.decryptSecretKey(cryptoParameters); - - cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams); - + setupCryptoFromCachedData(cachedCryptoParams); + } else { + // Place something in cache that indicates this file has no crypto metadata. See ACCUMULO-4141 + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + NO_CRYPTO.write(dos); + dos.close(); + cache.cacheMetaBlock(CRYPTO_BLOCK_NAME, baos.toByteArray()); } if (cachedMetaIndex == null) { @@ -931,15 +955,25 @@ public final class BCFile { metaIndex = new MetaIndex(cachedMetaIndex); dataIndex = new DataIndex(cachedDataIndex); - cryptoParams = new BCFileCryptoModuleParameters(); - cryptoParams.read(cachedCryptoParams); + setupCryptoFromCachedData(cachedCryptoParams); + } + } - this.cryptoModule = CryptoModuleFactory.getCryptoModule(cryptoParams.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey())); - this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(cryptoParams.getKeyEncryptionStrategyClass()); + private void setupCryptoFromCachedData(BlockRead cachedCryptoParams) throws IOException { + BCFileCryptoModuleParameters params = new BCFileCryptoModuleParameters(); + params.read(cachedCryptoParams); - // This call should put the decrypted session key within the cryptoParameters object - cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(cryptoParams); + if (Arrays.equals(params.getEncryptedKey(), NO_CRYPTO.getEncryptedKey()) + && NO_CRYPTO.getOpaqueKeyEncryptionKeyID().equals(params.getOpaqueKeyEncryptionKeyID())) { + this.cryptoParams = null; + this.cryptoModule = null; + this.secretKeyEncryptionStrategy = null; + } else { + this.cryptoModule = CryptoModuleFactory.getCryptoModule(params.getAllOptions().get(Property.CRYPTO_MODULE_CLASS.getKey())); + this.secretKeyEncryptionStrategy = CryptoModuleFactory.getSecretKeyEncryptionStrategy(params.getKeyEncryptionStrategyClass()); + // This call should put the decrypted session key within the cryptoParameters object + cryptoParams = (BCFileCryptoModuleParameters) secretKeyEncryptionStrategy.decryptSecretKey(params); } }