Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C5AB811741 for ; Wed, 9 Apr 2014 17:59:34 +0000 (UTC) Received: (qmail 39194 invoked by uid 500); 9 Apr 2014 17:58:29 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 38849 invoked by uid 500); 9 Apr 2014 17:58:19 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 37324 invoked by uid 99); 9 Apr 2014 17:57:38 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Apr 2014 17:57:38 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 86AC394F902; Wed, 9 Apr 2014 17:57:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Wed, 09 Apr 2014 17:58:26 -0000 Message-Id: <8b17aff2c61f4d7b830870fa3123d9ff@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [55/64] [abbrv] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java index 5ae072a,0000000..244a877 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java +++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleParameters.java @@@ -1,643 -1,0 +1,637 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.security.crypto; + +import java.io.FilterOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.SecureRandom; +import java.util.Map; + +import javax.crypto.Cipher; +import javax.crypto.CipherOutputStream; + +/** + * This class defines several parameters needed by by a module providing cryptographic stream support in Accumulo. The following Javadoc details which + * parameters are used for which operations (encryption vs. decryption), which ones return values (i.e. are "out" parameters from the {@link CryptoModule}), and + * which ones are required versus optional in certain situations. + * + * Most of the time, these classes can be constructed using + * {@link CryptoModuleFactory#createParamsObjectFromAccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration)}. + */ +public class CryptoModuleParameters { + + /** + * Gets the name of the symmetric algorithm to use for encryption. + * + * @see CryptoModuleParameters#setAlgorithmName(String) + */ + + public String getAlgorithmName() { + return algorithmName; + } + + /** + * Sets the name of the symmetric algorithm to use for an encryption stream. + *

+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "AES", "RC4", + * "DESede", etc. + *

+ * For encryption, this value is required and is always used. Its value should be prepended or otherwise included with the ciphertext for future + * decryption.
+ * For decryption, this value is often disregarded in favor of the value encoded with the ciphertext. + * + * @param algorithmName + * the name of the cryptographic algorithm to use. + * @see Standard Algorithm Names in JCE + * + */ + + public void setAlgorithmName(String algorithmName) { + this.algorithmName = algorithmName; + } + + /** + * Gets the name of the encryption mode to use for encryption. + * + * @see CryptoModuleParameters#setEncryptionMode(String) + */ + + public String getEncryptionMode() { + return encryptionMode; + } + + /** + * Sets the name of the encryption mode to use for an encryption stream. + *

+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "EBC", "CBC", + * "CFB", etc. + *

+ * For encryption, this value is required and is always used. Its value should be prepended or otherwise included with the ciphertext for future + * decryption.
+ * For decryption, this value is often disregarded in favor of the value encoded with the ciphertext. + * + * @param encryptionMode + * the name of the encryption mode to use. + * @see Standard Mode Names in JCE + * + */ + + public void setEncryptionMode(String encryptionMode) { + this.encryptionMode = encryptionMode; + } + + /** + * Gets the name of the padding type to use for encryption. + * + * @see CryptoModuleParameters#setPadding(String) + */ + + public String getPadding() { + return padding; + } + + /** + * Sets the name of the padding type to use for an encryption stream. + *

+ * Valid names are names recognized by your cryptographic engine provider. For the default Java provider, valid names would include things like "NoPadding", + * "None", etc. + *

+ * For encryption, this value is required and is always used. Its value should be prepended or otherwise included with the ciphertext for future + * decryption.
+ * For decryption, this value is often disregarded in favor of the value encoded with the ciphertext. + * + * @param padding + * the name of the padding type to use. + * @see Standard Padding Names in JCE + * + */ + public void setPadding(String padding) { + this.padding = padding; + } + + /** + * Gets the plaintext secret key. + *

+ * For decryption, this value is often the out parameter of using a secret key encryption strategy to decrypt an encrypted version of this secret key. + * (See {@link CryptoModuleParameters#setKeyEncryptionStrategyClass(String)}.) + * + * + * @see CryptoModuleParameters#setPlaintextKey(byte[]) + */ + public byte[] getPlaintextKey() { + return plaintextKey; + } + + /** + * Sets the plaintext secret key that will be used to encrypt and decrypt bytes. + *

+ * Valid values and lengths for this secret key depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information. + *

+ * For encryption, this value is optional. If it is not provided, it will be automatically generated by the underlying cryptographic module.
+ * For decryption, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see + * {@link CryptoModuleParameters#setEncryptedKey(byte[])}). + * + * @param plaintextKey + * the value of the plaintext secret key + */ + + public void setPlaintextKey(byte[] plaintextKey) { + this.plaintextKey = plaintextKey; + } + + /** + * Gets the length of the secret key. + * + * @see CryptoModuleParameters#setKeyLength(int) + */ + public int getKeyLength() { + return keyLength; + } + + /** + * Sets the length of the secret key that will be used to encrypt and decrypt bytes. + *

+ * Valid lengths depend entirely on the algorithm type. Refer to the documentation about the algorithm for further information. (For example, AES may use + * either 128 or 256 bit keys in the default Java cryptography provider.) + *

+ * For encryption, this value is required if the secret key is not set.
+ * For decryption, this value is often obtained from the underlying cipher stream, or derived from the encrypted version of the key (see + * {@link CryptoModuleParameters#setEncryptedKey(byte[])}). + * + * @param keyLength + * the length of the secret key to be generated + */ + + public void setKeyLength(int keyLength) { + this.keyLength = keyLength; + } + + /** + * Gets the random number generator name. + * + * @see CryptoModuleParameters#setRandomNumberGenerator(String) + */ + + public String getRandomNumberGenerator() { + return randomNumberGenerator; + } + + /** + * Sets the name of the random number generator to use. The default for this for the baseline JCE implementation is "SHA1PRNG". + *

+ * + *

+ * For encryption, this value is required.
+ * For decryption, this value is often obtained from the underlying cipher stream. + * + * @param randomNumberGenerator + * the name of the random number generator to use + */ + + public void setRandomNumberGenerator(String randomNumberGenerator) { + this.randomNumberGenerator = randomNumberGenerator; + } + + /** + * Gets the random number generator provider name. + * + * @see CryptoModuleParameters#setRandomNumberGeneratorProvider(String) + */ + public String getRandomNumberGeneratorProvider() { + return randomNumberGeneratorProvider; + } + + /** + * Sets the name of the random number generator provider to use. The default for this for the baseline JCE implementation is "SUN". + *

+ * The provider, as the name implies, provides the RNG implementation specified by {@link CryptoModuleParameters#getRandomNumberGenerator()}. + *

+ * For encryption, this value is required.
+ * For decryption, this value is often obtained from the underlying cipher stream. + * + * @param randomNumberGeneratorProvider + * the name of the provider to use + */ + + public void setRandomNumberGeneratorProvider(String randomNumberGeneratorProvider) { + this.randomNumberGeneratorProvider = randomNumberGeneratorProvider; + } + + /** + * Gets the key encryption strategy class. + * + * @see CryptoModuleParameters#setKeyEncryptionStrategyClass(String) + */ + + public String getKeyEncryptionStrategyClass() { + return keyEncryptionStrategyClass; + } + + /** + * Sets the class name of the key encryption strategy class. The class obeys the {@link SecretKeyEncryptionStrategy} interface. It instructs the + * {@link DefaultCryptoModule} on how to encrypt the keys it uses to secure the streams. + *

+ * The default implementation of this interface, {@link CachingHDFSSecretKeyEncryptionStrategy}, creates a random key encryption key (KEK) as another symmetric + * key and places the KEK into HDFS. This is not really very secure. Users of the crypto modules are encouraged to either safeguard that KEK carefully + * or to obtain and use another {@link SecretKeyEncryptionStrategy} class. + *

+ * For encryption, this value is optional. If it is not specified, then it assumed that the secret keys used for encrypting files will not be + * encrypted. This is not a secure approach, thus setting this is highly recommended.
+ * For decryption, this value is often obtained from the underlying cipher stream. However, the underlying stream's value can be overridden (at least + * when using {@link DefaultCryptoModule}) by setting the {@link CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean)} to true. + * + * @param keyEncryptionStrategyClass + * the name of the key encryption strategy class to use + */ + public void setKeyEncryptionStrategyClass(String keyEncryptionStrategyClass) { + this.keyEncryptionStrategyClass = keyEncryptionStrategyClass; + } + + /** + * Gets the encrypted version of the plaintext key. This parameter is generally either obtained from an underlying stream or computed in the process of + * employed the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}. + * + * @see CryptoModuleParameters#setEncryptedKey(byte[]) + */ + public byte[] getEncryptedKey() { + return encryptedKey; + } + + /** + * Sets the encrypted version of the plaintext key ({@link CryptoModuleParameters#getPlaintextKey()}). Generally this operation will be done either by: + *

+ *

    + *
  • the code reading an encrypted stream and coming across the encrypted version of one of these keys, OR + *
  • the {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()} that encrypted the plaintext key (see + * {@link CryptoModuleParameters#getPlaintextKey()}). + *
      + *

      + * For encryption, this value is generally not required, but is usually set by the underlying module during encryption.
      + * For decryption, this value is usually required. + * + * + * @param encryptedKey + * the encrypted value of the plaintext key + */ + + public void setEncryptedKey(byte[] encryptedKey) { + this.encryptedKey = encryptedKey; + } + + /** + * Gets the opaque ID associated with the encrypted version of the plaintext key. + * + * @see CryptoModuleParameters#setOpaqueKeyEncryptionKeyID(String) + */ + public String getOpaqueKeyEncryptionKeyID() { + return opaqueKeyEncryptionKeyID; + } + + /** + * Sets an opaque ID assocaited with the encrypted version of the plaintext key. + *

      + * Often, implementors of the {@link SecretKeyEncryptionStrategy} will need to record some information about how they encrypted a particular plaintext key. + * For example, if the strategy employs several keys for its encryption, it will want to record which key it used. The caller should not have to worry about + * the format or contents of this internal ID; thus, the strategy class will encode whatever information it needs into this string. It is then beholden to the + * calling code to record this opqaue string properly to the underlying cryptographically-encoded stream, and then set the opaque ID back into this parameter + * object upon reading. + *

      + * For encryption, this value is generally not required, but will be typically generated and set by the {@link SecretKeyEncryptionStrategy} class (see + * {@link CryptoModuleParameters#getKeyEncryptionStrategyClass()}).
      + * For decryption, this value is required, though it will typically be read from the underlying stream. + * + * @param opaqueKeyEncryptionKeyID + * the opaque ID assoicated with the encrypted version of the plaintext key (see {@link CryptoModuleParameters#getEncryptedKey()}). + */ + + public void setOpaqueKeyEncryptionKeyID(String opaqueKeyEncryptionKeyID) { + this.opaqueKeyEncryptionKeyID = opaqueKeyEncryptionKeyID; + } + + /** + * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code + * to do so. + * + * @see CryptoModuleParameters#setRecordParametersToStream(boolean) + */ + public boolean getRecordParametersToStream() { + return recordParametersToStream; + } + + /** + * Gets the flag that indicates whether or not the module should record its cryptographic parameters to the stream automatically, or rely on the calling code + * to do so. + * + *

      + * + * If this is set to true, then the stream passed to {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)} will be written to by + * the module before it is returned to the caller. There are situations where it is easier to let the crypto module do this writing on behalf of the + * caller, and other times where it is not appropriate (if the format of the underlying stream must be carefully maintained, for instance). + * + * @param recordParametersToStream + * whether or not to require the module to record its parameters to the stream by itself + */ + public void setRecordParametersToStream(boolean recordParametersToStream) { + this.recordParametersToStream = recordParametersToStream; + } + + /** + * Gets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed. + * + * @see CryptoModuleParameters#setCloseUnderylingStreamAfterCryptoStreamClose(boolean) + */ + public boolean getCloseUnderylingStreamAfterCryptoStreamClose() { + return closeUnderylingStreamAfterCryptoStreamClose; + } + + /** + * Sets the flag that indicates whether or not to close the underlying stream when the cipher stream is closed. + * + *

      + * + * {@link CipherOutputStream} will only output its padding bytes when its {@link CipherOutputStream#close()} method is called. However, there are times when a + * caller doesn't want its underlying stream closed at the time that the {@link CipherOutputStream} is closed. This flag indicates that the + * {@link CryptoModule} should wrap the underlying stream in a basic {@link FilterOutputStream} which will swallow any close() calls and prevent them from + * propogating to the underlying stream. + * + * @param closeUnderylingStreamAfterCryptoStreamClose + * the flag that indicates whether or not to close the underlying stream when the cipher stream is closed + */ + public void setCloseUnderylingStreamAfterCryptoStreamClose(boolean closeUnderylingStreamAfterCryptoStreamClose) { + this.closeUnderylingStreamAfterCryptoStreamClose = closeUnderylingStreamAfterCryptoStreamClose; + } + + /** + * Gets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy. + * + * @see CryptoModuleParameters#setOverrideStreamsSecretKeyEncryptionStrategy(boolean) + */ + public boolean getOverrideStreamsSecretKeyEncryptionStrategy() { + return overrideStreamsSecretKeyEncryptionStrategy; + } + + /** + * Sets the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption strategy. + * + *

      + * + * So, why is this important? Say you started out with the default secret key encryption strategy. So, now you have a secret key in HDFS that encrypts all the + * other secret keys. Then you deploy a key management solution. You want to move that secret key up to the key management server. Great! No problem. + * Except, all your encrypted files now contain a setting that says + * "hey I was encrypted by the default strategy, so find decrypt my key using that, not the key management server". This setting signals the + * {@link CryptoModule} that it should ignore the setting in the file and prefer the one from the configuration. + * + * @param overrideStreamsSecretKeyEncryptionStrategy + * the flag that indicates if the underlying stream's key encryption strategy should be overridden by the currently configured key encryption + * strategy + */ + + public void setOverrideStreamsSecretKeyEncryptionStrategy(boolean overrideStreamsSecretKeyEncryptionStrategy) { + this.overrideStreamsSecretKeyEncryptionStrategy = overrideStreamsSecretKeyEncryptionStrategy; + } + + /** + * Gets the plaintext output stream to wrap for encryption. + * + * @see CryptoModuleParameters#setPlaintextOutputStream(OutputStream) + */ + public OutputStream getPlaintextOutputStream() { + return plaintextOutputStream; + } + + /** + * Sets the plaintext output stream to wrap for encryption. + * + *

      + * + * For encryption, this parameter is required.
      + * For decryption, this parameter is ignored. - * - * @param plaintextOutputStream + */ + public void setPlaintextOutputStream(OutputStream plaintextOutputStream) { + this.plaintextOutputStream = plaintextOutputStream; + } + + /** + * Gets the encrypted output stream, which is nearly always a wrapped version of the output stream from + * {@link CryptoModuleParameters#getPlaintextOutputStream()}. + * + *

      + * + * Generally this method is used by {@link CryptoModule} classes as an out parameter from calling + * {@link CryptoModule#getEncryptingOutputStream(CryptoModuleParameters)}. + * + * @see CryptoModuleParameters#setEncryptedOutputStream(OutputStream) + */ + + public OutputStream getEncryptedOutputStream() { + return encryptedOutputStream; + } + + /** + * Sets the encrypted output stream. This method should really only be called by {@link CryptoModule} implementations unless something very unusual is going + * on. + * + * @param encryptedOutputStream + * the encrypted version of the stream from output stream from {@link CryptoModuleParameters#getPlaintextOutputStream()}. + */ + public void setEncryptedOutputStream(OutputStream encryptedOutputStream) { + this.encryptedOutputStream = encryptedOutputStream; + } + + /** + * Gets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}. + * + *

      + * + * Generally this method is used by {@link CryptoModule} classes as an out parameter from calling + * {@link CryptoModule#getDecryptingInputStream(CryptoModuleParameters)}. + * + * + * @see CryptoModuleParameters#setPlaintextInputStream(InputStream) + */ + public InputStream getPlaintextInputStream() { + return plaintextInputStream; + } + + /** + * Sets the plaintext input stream, which is nearly always a wrapped version of the output from {@link CryptoModuleParameters#getEncryptedInputStream()}. + * + *

      + * + * This method should really only be called by {@link CryptoModule} implementations. - * - * @param plaintextInputStream + */ + + public void setPlaintextInputStream(InputStream plaintextInputStream) { + this.plaintextInputStream = plaintextInputStream; + } + + /** + * Gets the encrypted input stream to wrap for decryption. + * + * @see CryptoModuleParameters#setEncryptedInputStream(InputStream) + */ + public InputStream getEncryptedInputStream() { + return encryptedInputStream; + } + + /** + * Sets the encrypted input stream to wrap for decryption. - * - * @param encryptedInputStream + */ + + public void setEncryptedInputStream(InputStream encryptedInputStream) { + this.encryptedInputStream = encryptedInputStream; + } + + /** + * Gets the initialized cipher object. + * + * + * @see CryptoModuleParameters#setCipher(Cipher) + */ + public Cipher getCipher() { + return cipher; + } + + /** + * Sets the initialized cipher object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the cipher + * object is created outside of the module (to determine IV lengths, for one). If it is created and you want the module to use the cipher you already + * initialized, set it here. + * + * @param cipher + * the cipher object + */ + public void setCipher(Cipher cipher) { + this.cipher = cipher; + } + + /** + * Gets the initialized secure random object. + * + * @see CryptoModuleParameters#setSecureRandom(SecureRandom) + */ + public SecureRandom getSecureRandom() { + return secureRandom; + } + + /** + * Sets the initialized secure random object. Generally speaking, callers do not have to create and set this object. There may be circumstances where the + * random object is created outside of the module (for instance, to create a random secret key). If it is created outside the module and you want the module + * to use the random object you already created, set it here. + * + * @param secureRandom + * the {@link SecureRandom} object + */ + + public void setSecureRandom(SecureRandom secureRandom) { + this.secureRandom = secureRandom; + } + + /** + * Gets the initialization vector to use for this crypto module. + * + * @see CryptoModuleParameters#setInitializationVector(byte[]) + */ + public byte[] getInitializationVector() { + return initializationVector; + } + + /** + * Sets the initialization vector to use for this crypto module. + * + *

      + * + * For encryption, this parameter is optional. If the initialization vector is created by the caller, for whatever reasons, it can be set here + * and the crypto module will use it.
      + * + * For decryption, this parameter is required. It should be read from the underlying stream that contains the encrypted data. + * + * @param initializationVector + * the initialization vector to use for this crypto operation. + */ + public void setInitializationVector(byte[] initializationVector) { + this.initializationVector = initializationVector; + } + + /** + * Gets the size of the buffering stream that sits above the cipher stream + */ + public int getBlockStreamSize() { + return blockStreamSize; + } + + /** + * Sets the size of the buffering stream that sits above the cipher stream + */ + public void setBlockStreamSize(int blockStreamSize) { + this.blockStreamSize = blockStreamSize; + } + + /** + * Gets the overall set of options for the {@link CryptoModule}. + * + * @see CryptoModuleParameters#setAllOptions(Map) + */ + public Map getAllOptions() { + return allOptions; + } + + /** + * Sets the overall set of options for the {@link CryptoModule}. + * + *

      + * + * Often, options for the cryptographic modules will be encoded as key/value pairs in a configuration file. This map represents those values. It may include + * some of the parameters already called out as members of this class. It may contain any number of additional parameters which may be required by different + * module or key encryption strategy implementations. + * + * @param allOptions + * the set of key/value pairs that confiure a module, based on a configuration file + */ + public void setAllOptions(Map allOptions) { + this.allOptions = allOptions; + } + + private String algorithmName = null; + private String encryptionMode = null; + private String padding = null; + private byte[] plaintextKey; + private int keyLength = 0; + private String randomNumberGenerator = null; + private String randomNumberGeneratorProvider = null; + + private String keyEncryptionStrategyClass; + private byte[] encryptedKey; + private String opaqueKeyEncryptionKeyID; + + private boolean recordParametersToStream = true; + private boolean closeUnderylingStreamAfterCryptoStreamClose = true; + private boolean overrideStreamsSecretKeyEncryptionStrategy = false; + + private OutputStream plaintextOutputStream; + private OutputStream encryptedOutputStream; + private InputStream plaintextInputStream; + private InputStream encryptedInputStream; + + private Cipher cipher; + private SecureRandom secureRandom; + private byte[] initializationVector; + + private Map allOptions; + private int blockStreamSize; +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java index cc363c7,0000000..e6bfca8 mode 100644,000000..100644 --- a/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/lexicoder/ReverseLexicoderTest.java @@@ -1,62 -1,0 +1,60 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.lexicoder; + +import java.io.UnsupportedEncodingException; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; + +import org.junit.Test; + +public class ReverseLexicoderTest extends LexicoderTest { + public void testSortOrder() { + Comparator comp = Collections.reverseOrder(); + assertSortOrder(new ReverseLexicoder(new LongLexicoder()), comp, Long.MIN_VALUE, 0xff1234567890abcdl, 0xffff1234567890abl, 0xffffff567890abcdl, + 0xffffffff7890abcdl, 0xffffffffff90abcdl, 0xffffffffffffabcdl, 0xffffffffffffffcdl, -1l, 0l, 0x01l, 0x1234l, 0x123456l, 0x12345678l, 0x1234567890l, + 0x1234567890abl, 0x1234567890abcdl, 0x1234567890abcdefl, Long.MAX_VALUE); + + Comparator comp2 = Collections.reverseOrder(); + assertSortOrder(new ReverseLexicoder(new StringLexicoder()), comp2, "a", "aa", "ab", "b", "aab"); + + } + + /** + * Just a simple test verifying reverse indexed dates - * - * @throws UnsupportedEncodingException + */ + @Test + public void testReverseSortDates() throws UnsupportedEncodingException { + + ReverseLexicoder revLex = new ReverseLexicoder(new DateLexicoder()); + + Date date1 = new Date(); + Date date2 = new Date(System.currentTimeMillis() + 10000); + Date date3 = new Date(System.currentTimeMillis() + 500); + + Comparator comparator = Collections.reverseOrder(); + assertSortOrder(revLex, comparator, date1, date2, date3); + + // truncate date to hours + long time = System.currentTimeMillis() - (System.currentTimeMillis() % 3600000); + Date date = new Date(time); + + System.out.println(date); + + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java index e83453e,3ec9bb1..13490e0 --- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java @@@ -57,23 -55,9 +57,21 @@@ public class AccumuloInputFormatTest private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName(); private static final String INSTANCE_NAME = PREFIX + "_mapred_instance"; private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1"; - + + private JobConf job; + + @BeforeClass + public static void setupClass() { + System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp"); + } + + @Before + public void createJob() { + job = new JobConf(); + } + /** * Check that the iterator configuration is getting stored in the Job conf correctly. - * - * @throws IOException */ @Test public void testSetIterator() throws IOException { @@@ -152,11 -141,9 +150,9 @@@ assertEquals(list.get(1).getOptions().get(key), value); assertEquals(list.get(1).getOptions().get(key + "2"), value); } - + /** * Test getting iterator settings for multiple iterators set - * - * @throws IOException */ @Test public void testGetIteratorSettings() throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java index 54bd127,ae5e395..2500972 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java @@@ -64,9 -64,45 +64,7 @@@ public class AccumuloInputFormatTest private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName(); /** - * Test basic setting & getting of max versions. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Deprecated - @Test - public void testMaxVersions() throws IOException { - Job job = new Job(); - AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1); - int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration()); - assertEquals(1, version); - } - - /** - * Test max versions with an invalid value. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Deprecated - @Test(expected = IOException.class) - public void testMaxVersionsLessThan1() throws IOException { - Job job = new Job(); - AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0); - } - - /** - * Test no max version configured. - */ - @Deprecated - @Test - public void testNoMaxVersion() throws IOException { - Job job = new Job(); - assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration())); - } - - /** * Check that the iterator configuration is getting stored in the Job conf correctly. - * - * @throws IOException */ @Test public void testSetIterator() throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java index 009be17,0000000..c06df51 mode 100644,000000..100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java @@@ -1,317 -1,0 +1,309 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.client.mock; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Random; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.NamespaceNotEmptyException; +import org.apache.accumulo.core.client.NamespaceNotFoundException; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.NamespaceOperations; +import org.apache.accumulo.core.client.impl.Namespaces; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.security.Authorizations; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class MockNamespacesTest { + + Random random = new Random(); + public static TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); + + /** + * This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace. - * - * @throws Exception + */ + @Test + public void testDefaultNamespace() throws Exception { + String tableName = "test"; + Instance instance = new MockInstance("default"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + assertTrue(c.namespaceOperations().exists(Namespaces.DEFAULT_NAMESPACE)); + c.tableOperations().create(tableName); + assertTrue(c.tableOperations().exists(tableName)); + } + + /** + * This test creates a new namespace "testing" and a table "testing.table1" which puts "table1" into the "testing" namespace. Then we create "testing.table2" + * which creates "table2" and puts it into "testing" as well. Then we make sure that you can't delete a namespace with tables in it, and then we delete the + * tables and delete the namespace. - * - * @throws Exception + */ + @Test + public void testCreateAndDeleteNamespace() throws Exception { + String namespace = "testing"; + String tableName1 = namespace + ".table1"; + String tableName2 = namespace + ".table2"; + + Instance instance = new MockInstance("createdelete"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + c.namespaceOperations().create(namespace); + assertTrue(c.namespaceOperations().exists(namespace)); + + c.tableOperations().create(tableName1); + assertTrue(c.tableOperations().exists(tableName1)); + + c.tableOperations().create(tableName2); + assertTrue(c.tableOperations().exists(tableName2)); + + // deleting + try { + // can't delete a namespace with tables in it + c.namespaceOperations().delete(namespace); + fail(); + } catch (NamespaceNotEmptyException e) { + // ignore, supposed to happen + } + assertTrue(c.namespaceOperations().exists(namespace)); + assertTrue(c.tableOperations().exists(tableName1)); + assertTrue(c.tableOperations().exists(tableName2)); + + c.tableOperations().delete(tableName2); + assertTrue(!c.tableOperations().exists(tableName2)); + assertTrue(c.namespaceOperations().exists(namespace)); + + c.tableOperations().delete(tableName1); + assertTrue(!c.tableOperations().exists(tableName1)); + c.namespaceOperations().delete(namespace); + assertTrue(!c.namespaceOperations().exists(namespace)); + } + + /** + * This test creates a namespace, modifies it's properties, and checks to make sure that those properties are applied to its tables. To do something on a + * namespace-wide level, use {@link NamespaceOperations}. + * + * Checks to make sure namespace-level properties are overridden by table-level properties. + * + * Checks to see if the default namespace's properties work as well. - * - * @throws Exception + */ + + @Test + public void testNamespaceProperties() throws Exception { + String namespace = "propchange"; + String tableName1 = namespace + ".table1"; + String tableName2 = namespace + ".table2"; + + String propKey = Property.TABLE_SCAN_MAXMEM.getKey(); + String propVal = "42K"; + + Instance instance = new MockInstance("props"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + c.namespaceOperations().create(namespace); + c.tableOperations().create(tableName1); + c.namespaceOperations().setProperty(namespace, propKey, propVal); + + // check the namespace has the property + assertTrue(checkNamespaceHasProp(c, namespace, propKey, propVal)); + + // check that the table gets it from the namespace + assertTrue(checkTableHasProp(c, tableName1, propKey, propVal)); + + // test a second table to be sure the first wasn't magical + // (also, changed the order, the namespace has the property already) + c.tableOperations().create(tableName2); + assertTrue(checkTableHasProp(c, tableName2, propKey, propVal)); + + // test that table properties override namespace properties + String propKey2 = Property.TABLE_FILE_MAX.getKey(); + String propVal2 = "42"; + String tablePropVal = "13"; + + c.tableOperations().setProperty(tableName2, propKey2, tablePropVal); + c.namespaceOperations().setProperty("propchange", propKey2, propVal2); + + assertTrue(checkTableHasProp(c, tableName2, propKey2, tablePropVal)); + + // now check that you can change the default namespace's properties + propVal = "13K"; + String tableName = "some_table"; + c.tableOperations().create(tableName); + c.namespaceOperations().setProperty(Namespaces.DEFAULT_NAMESPACE, propKey, propVal); + + assertTrue(checkTableHasProp(c, tableName, propKey, propVal)); + + // test the properties server-side by configuring an iterator. + // should not show anything with column-family = 'a' + String tableName3 = namespace + ".table3"; + c.tableOperations().create(tableName3); + + IteratorSetting setting = new IteratorSetting(250, "thing", SimpleFilter.class.getName()); + c.namespaceOperations().attachIterator(namespace, setting); + + BatchWriter bw = c.createBatchWriter(tableName3, new BatchWriterConfig()); + Mutation m = new Mutation("r"); + m.put("a", "b", new Value("abcde".getBytes())); + bw.addMutation(m); + bw.flush(); + bw.close(); + + // Scanner s = c.createScanner(tableName3, Authorizations.EMPTY); + // do scanners work correctly in mock? + // assertTrue(!s.iterator().hasNext()); + } + + /** + * This test renames and clones two separate table into different namespaces. different namespace. - * - * @throws Exception + */ + @Test + public void testRenameAndCloneTableToNewNamespace() throws Exception { + String namespace1 = "renamed"; + String namespace2 = "cloned"; + String tableName = "table"; + String tableName1 = "renamed.table1"; + // String tableName2 = "cloned.table2"; + + Instance instance = new MockInstance("renameclone"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + c.tableOperations().create(tableName); + c.namespaceOperations().create(namespace1); + c.namespaceOperations().create(namespace2); + + c.tableOperations().rename(tableName, tableName1); + + assertTrue(c.tableOperations().exists(tableName1)); + assertTrue(!c.tableOperations().exists(tableName)); + + // TODO implement clone in mock + /* + * c.tableOperations().clone(tableName1, tableName2, false, null, null); + * + * assertTrue(c.tableOperations().exists(tableName1)); assertTrue(c.tableOperations().exists(tableName2)); + */ + return; + } + + /** + * This test renames a namespace and ensures that its tables are still correct + */ + @Test + public void testNamespaceRename() throws Exception { + String namespace1 = "n1"; + String namespace2 = "n2"; + String table = "t"; + + Instance instance = new MockInstance("rename"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + c.namespaceOperations().create(namespace1); + c.tableOperations().create(namespace1 + "." + table); + + c.namespaceOperations().rename(namespace1, namespace2); + + assertTrue(!c.namespaceOperations().exists(namespace1)); + assertTrue(c.namespaceOperations().exists(namespace2)); + assertTrue(!c.tableOperations().exists(namespace1 + "." + table)); + assertTrue(c.tableOperations().exists(namespace2 + "." + table)); + } + + /** + * This tests adding iterators to a namespace, listing them, and removing them + */ + @Test + public void testNamespaceIterators() throws Exception { + Instance instance = new MockInstance("Iterators"); + Connector c = instance.getConnector("user", new PasswordToken("pass")); + + String namespace = "iterator"; + String tableName = namespace + ".table"; + String iter = "thing"; + + c.namespaceOperations().create(namespace); + c.tableOperations().create(tableName); + + IteratorSetting setting = new IteratorSetting(250, iter, SimpleFilter.class.getName()); + HashSet scope = new HashSet(); + scope.add(IteratorScope.scan); + c.namespaceOperations().attachIterator(namespace, setting, EnumSet.copyOf(scope)); + + BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); + Mutation m = new Mutation("r"); + m.put("a", "b", new Value("abcde".getBytes(Constants.UTF8))); + bw.addMutation(m); + bw.flush(); + + Scanner s = c.createScanner(tableName, Authorizations.EMPTY); + System.out.println(s.iterator().next()); + // do scanners work correctly in mock? + // assertTrue(!s.iterator().hasNext()); + + assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iter)); + c.namespaceOperations().removeIterator(namespace, iter, EnumSet.copyOf(scope)); + } + + private boolean checkTableHasProp(Connector c, String t, String propKey, String propVal) throws AccumuloException, TableNotFoundException { + for (Entry e : c.tableOperations().getProperties(t)) { + if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) { + return true; + } + } + return false; + } + + private boolean checkNamespaceHasProp(Connector c, String n, String propKey, String propVal) throws AccumuloException, NamespaceNotFoundException, + AccumuloSecurityException { + for (Entry e : c.namespaceOperations().getProperties(n)) { + if (e.getKey().equals(propKey) && e.getValue().equals(propVal)) { + return true; + } + } + return false; + } + + public static class SimpleFilter extends Filter { + @Override + public boolean accept(Key k, Value v) { + if (k.getColumnFamily().toString().equals("a")) + return false; + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java index de6ca21,0000000..d31a788 mode 100644,000000..100644 --- a/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java +++ b/core/src/test/java/org/apache/accumulo/core/security/VisibilityConstraintTest.java @@@ -1,106 -1,0 +1,103 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.security; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; +import java.util.List; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.constraints.Constraint.Environment; +import org.apache.accumulo.core.data.ArrayByteSequence; +import org.apache.accumulo.core.data.Mutation; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +public class VisibilityConstraintTest { + + VisibilityConstraint vc; + Environment env; + Mutation mutation; + + static final ColumnVisibility good = new ColumnVisibility("good"); + static final ColumnVisibility bad = new ColumnVisibility("bad"); + + static final String D = "don't care"; + + static final List ENOAUTH = Arrays.asList((short) 2); + - /** - * @throws java.lang.Exception - */ + @Before + public void setUp() throws Exception { + vc = new VisibilityConstraint(); + mutation = new Mutation("r"); + + ArrayByteSequence bs = new ArrayByteSequence("good".getBytes(Constants.UTF8)); + + AuthorizationContainer ac = createNiceMock(AuthorizationContainer.class); + expect(ac.contains(bs)).andReturn(true); + replay(ac); + + env = createMock(Environment.class); + expect(env.getAuthorizationsContainer()).andReturn(ac); + replay(env); + } + + @Test + public void testNoVisibility() { + mutation.put(D, D, D); + assertNull("authorized", vc.check(env, mutation)); + } + + @Test + public void testVisibilityNoAuth() { + mutation.put(D, D, bad, D); + assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation)); + } + + @Test + public void testGoodVisibilityAuth() { + mutation.put(D, D, good, D); + assertNull("authorized", vc.check(env, mutation)); + } + + @Test + public void testCachedVisibilities() { + mutation.put(D, D, good, "v"); + mutation.put(D, D, good, "v2"); + assertNull("authorized", vc.check(env, mutation)); + } + + @Test + public void testMixedVisibilities() { + mutation.put(D, D, bad, D); + mutation.put(D, D, good, D); + assertEquals("unauthorized", ENOAUTH, vc.check(env, mutation)); + } + + @Test + @Ignore + public void testMalformedVisibility() { + // TODO: ACCUMULO-1006 Should test for returning error code 1, but not sure how since ColumnVisibility won't let us construct a bad one in the first place + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java ---------------------------------------------------------------------- diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java index 4607fdb,e76352a..44947d1 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java @@@ -89,40 -89,29 +89,36 @@@ public class RandomBatchWriter // create a random value that is a function of the // row id for verification purposes byte value[] = createValue(rowid, dataSize); - + m.put(new Text("foo"), new Text("1"), visibility, new Value(value)); - + return m; } - + static class Opts extends ClientOnRequiredTable { - @Parameter(names="--num", required=true) + @Parameter(names = "--num", required = true) int num = 0; - @Parameter(names="--min") + @Parameter(names = "--min") long min = 0; - @Parameter(names="--max") + @Parameter(names = "--max") long max = Long.MAX_VALUE; - @Parameter(names="--size", required=true, description="size of the value to write") + @Parameter(names = "--size", required = true, description = "size of the value to write") int size = 0; - @Parameter(names="--vis", converter=VisibilityConverter.class) + @Parameter(names = "--vis", converter = VisibilityConverter.class) ColumnVisibility visiblity = new ColumnVisibility(""); - @Parameter(names="--seed", description="seed for pseudo-random number generator") + @Parameter(names = "--seed", description = "seed for pseudo-random number generator") Long seed = null; } - + + public static long abs(long l) { + l = Math.abs(l); // abs(Long.MIN_VALUE) == Long.MIN_VALUE... + if (l < 0) + return 0; + return l; + } + /** * Writes a specified number of entries to Accumulo using a {@link BatchWriter}. - * - * @throws AccumuloException - * @throws AccumuloSecurityException - * @throws TableNotFoundException */ public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { Opts opts = new Opts(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/QueryUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/NGramIngest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java ---------------------------------------------------------------------- diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java index 3a211e2,669c76d..30ebd06 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java @@@ -121,9 -120,10 +121,8 @@@ public class TableToFile extends Config * * @param args * instanceName zookeepers username password table columns outputpath - * @throws Exception */ public static void main(String[] args) throws Exception { - int res = ToolRunner.run(CachedConfiguration.getInstance(), new TableToFile(), args); - if (res != 0) - System.exit(res); + ToolRunner.run(new Configuration(), new TableToFile(), args); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java ---------------------------------------------------------------------- diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java index add4c4c,d98d78b..aa12c71 --- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java +++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/shard/Query.java @@@ -59,26 -66,10 +59,23 @@@ public class Query IntersectingIterator.setColumnFamilies(ii, columns); bs.addScanIterator(ii); bs.setRanges(Collections.singleton(new Range())); + List result = new ArrayList(); for (Entry entry : bs) { - System.out.println(" " + entry.getKey().getColumnQualifier()); + result.add(entry.getKey().getColumnQualifier().toString()); } - + return result; + } + - /** - * @param args - */ + public static void main(String[] args) throws Exception { + Opts opts = new Opts(); + BatchScannerOpts bsOpts = new BatchScannerOpts(); + opts.parseArgs(Query.class.getName(), args, bsOpts); + Connector conn = opts.getConnector(); + BatchScanner bs = conn.createBatchScanner(opts.tableName, opts.auths, bsOpts.scanThreads); + bs.setTimeout(bsOpts.scanTimeout, TimeUnit.MILLISECONDS); + + for (String entry : query(bs, opts.terms)) + System.out.println(" " + entry); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java ---------------------------------------------------------------------- diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java index f7070dc,0000000..ab84d37 mode 100644,000000..100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java @@@ -1,264 -1,0 +1,262 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.minicluster; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.ServerSocket; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; + +import org.apache.accumulo.core.cli.Help; +import org.apache.accumulo.core.util.Pair; +import org.apache.commons.io.FileUtils; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.Parameter; +import com.google.common.io.Files; + +/** + * A runner for starting up a {@link MiniAccumuloCluster} from the command line using an optional configuration properties file. An example property file looks + * like the following: + * + * 

       + * rootPassword=secret
 + * instanceName=testInstance
 + * numTServers=1
 + * zooKeeperPort=3191
 + * jdwpEnabled=true
 + * zooKeeperMemory=128M
 + * tserverMemory=256M
 + * masterMemory=128M
 + * defaultMemory=256M
 + * shutdownPort=4446
 + * site.instance.secret=HUSH
 + *
      + * + * All items in the properties file above are optional and a default value will be provided in their absence. Any site configuration properties (typically found + * in the accumulo-site.xml file) should be prefixed with "site." in the properties file. + * + * @since 1.6.0 + */ +public class MiniAccumuloRunner { + private static final String ROOT_PASSWORD_PROP = "rootPassword"; + private static final String SHUTDOWN_PORT_PROP = "shutdownPort"; + private static final String DEFAULT_MEMORY_PROP = "defaultMemory"; + private static final String MASTER_MEMORY_PROP = "masterMemory"; + private static final String TSERVER_MEMORY_PROP = "tserverMemory"; + private static final String ZOO_KEEPER_MEMORY_PROP = "zooKeeperMemory"; + private static final String JDWP_ENABLED_PROP = "jdwpEnabled"; + private static final String ZOO_KEEPER_PORT_PROP = "zooKeeperPort"; + private static final String NUM_T_SERVERS_PROP = "numTServers"; + private static final String DIRECTORY_PROP = "directory"; + private static final String INSTANCE_NAME_PROP = "instanceName"; + + private static void printProperties() { + System.out.println("#mini Accumulo cluster runner properties."); + System.out.println("#"); + System.out.println("#uncomment following propeties to use, propeties not set will use default or random value"); + System.out.println(); + System.out.println("#" + INSTANCE_NAME_PROP + "=devTest"); + System.out.println("#" + DIRECTORY_PROP + "=/tmp/mac1"); + System.out.println("#" + ROOT_PASSWORD_PROP + "=secret"); + System.out.println("#" + NUM_T_SERVERS_PROP + "=2"); + System.out.println("#" + ZOO_KEEPER_PORT_PROP + "=40404"); + System.out.println("#" + SHUTDOWN_PORT_PROP + "=41414"); + System.out.println("#" + DEFAULT_MEMORY_PROP + "=128M"); + System.out.println("#" + MASTER_MEMORY_PROP + "=128M"); + System.out.println("#" + TSERVER_MEMORY_PROP + "=128M"); + System.out.println("#" + ZOO_KEEPER_MEMORY_PROP + "=128M"); + System.out.println("#" + JDWP_ENABLED_PROP + "=false"); + + System.out.println(); + System.out.println("# Configuration normally placed in accumulo-site.xml can be added using a site. prefix."); + System.out.println("# For example the following line will set tserver.compaction.major.concurrent.max"); + System.out.println(); + System.out.println("#site.tserver.compaction.major.concurrent.max=4"); + + } + + public static class PropertiesConverter implements IStringConverter { + @Override + public Properties convert(String fileName) { + Properties prop = new Properties(); + InputStream is; + try { + is = new FileInputStream(fileName); + try { + prop.load(is); + } finally { + is.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return prop; + } + } + + private static final String FORMAT_STRING = " %-21s %s"; + + public static class Opts extends Help { + @Parameter(names = "-p", required = false, description = "properties file name", converter = PropertiesConverter.class) + Properties prop = new Properties(); + + @Parameter(names = {"-c", "--printProperties"}, required = false, description = "prints an example propeties file, redirect to file to use") + boolean printProps = false; + } + + /** + * Runs the {@link MiniAccumuloCluster} given a -p argument with a property file. Establishes a shutdown port for asynchronous operation. + * + * @param args + * An optional -p argument can be specified with the path to a valid properties file. - * - * @throws Exception + */ + public static void main(String[] args) throws Exception { + Opts opts = new Opts(); + opts.parseArgs(MiniAccumuloRunner.class.getName(), args); + + if (opts.printProps) { + printProperties(); + System.exit(0); + } + + int shutdownPort = 4445; + + final File miniDir; + + if (opts.prop.containsKey(DIRECTORY_PROP)) + miniDir = new File(opts.prop.getProperty(DIRECTORY_PROP)); + else + miniDir = Files.createTempDir(); + + String rootPass = opts.prop.containsKey(ROOT_PASSWORD_PROP) ? opts.prop.getProperty(ROOT_PASSWORD_PROP) : "secret"; + + MiniAccumuloConfig config = new MiniAccumuloConfig(miniDir, rootPass); + + if (opts.prop.containsKey(INSTANCE_NAME_PROP)) + config.setInstanceName(opts.prop.getProperty(INSTANCE_NAME_PROP)); + if (opts.prop.containsKey(NUM_T_SERVERS_PROP)) + config.setNumTservers(Integer.parseInt(opts.prop.getProperty(NUM_T_SERVERS_PROP))); + if (opts.prop.containsKey(ZOO_KEEPER_PORT_PROP)) + config.setZooKeeperPort(Integer.parseInt(opts.prop.getProperty(ZOO_KEEPER_PORT_PROP))); + if (opts.prop.containsKey(JDWP_ENABLED_PROP)) + config.setJDWPEnabled(Boolean.parseBoolean(opts.prop.getProperty(JDWP_ENABLED_PROP))); + if (opts.prop.containsKey(ZOO_KEEPER_MEMORY_PROP)) + setMemoryOnConfig(config, opts.prop.getProperty(ZOO_KEEPER_MEMORY_PROP), ServerType.ZOOKEEPER); + if (opts.prop.containsKey(TSERVER_MEMORY_PROP)) + setMemoryOnConfig(config, opts.prop.getProperty(TSERVER_MEMORY_PROP), ServerType.TABLET_SERVER); + if (opts.prop.containsKey(MASTER_MEMORY_PROP)) + setMemoryOnConfig(config, opts.prop.getProperty(MASTER_MEMORY_PROP), ServerType.MASTER); + if (opts.prop.containsKey(DEFAULT_MEMORY_PROP)) + setMemoryOnConfig(config, opts.prop.getProperty(DEFAULT_MEMORY_PROP)); + if (opts.prop.containsKey(SHUTDOWN_PORT_PROP)) + shutdownPort = Integer.parseInt(opts.prop.getProperty(SHUTDOWN_PORT_PROP)); + + Map siteConfig = new HashMap(); + for (Map.Entry entry : opts.prop.entrySet()) { + String key = (String) entry.getKey(); + if (key.startsWith("site.")) + siteConfig.put(key.replaceFirst("site.", ""), (String) entry.getValue()); + } + + config.setSiteConfig(siteConfig); + + final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(config); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + try { + accumulo.stop(); + FileUtils.deleteDirectory(miniDir); + System.out.println("\nShut down gracefully on " + new Date()); + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + + accumulo.start(); + + printInfo(accumulo, shutdownPort); + + // start a socket on the shutdown port and block- anything connected to this port will activate the shutdown + ServerSocket shutdownServer = new ServerSocket(shutdownPort); + shutdownServer.accept(); + + System.exit(0); + } + + private static boolean validateMemoryString(String memoryString) { + String unitsRegex = "["; + MemoryUnit[] units = MemoryUnit.values(); + for (int i = 0; i < units.length; i++) { + unitsRegex += units[i].suffix(); + if (i < units.length - 1) + unitsRegex += "|"; + } + unitsRegex += "]"; + Pattern p = Pattern.compile("\\d+" + unitsRegex); + return p.matcher(memoryString).matches(); + } + + private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString) { + setMemoryOnConfig(config, memoryString, null); + } + + private static void setMemoryOnConfig(MiniAccumuloConfig config, String memoryString, ServerType serverType) { + if (!validateMemoryString(memoryString)) + throw new IllegalArgumentException(memoryString + " is not a valid memory string"); + + long memSize = Long.parseLong(memoryString.substring(0, memoryString.length() - 1)); + MemoryUnit memUnit = MemoryUnit.fromSuffix(memoryString.substring(memoryString.length() - 1)); + + if (serverType != null) + config.setMemory(serverType, memSize, memUnit); + else + config.setDefaultMemory(memSize, memUnit); + } + + private static void printInfo(MiniAccumuloCluster accumulo, int shutdownPort) { + System.out.println("Mini Accumulo Cluster\n"); + System.out.println(String.format(FORMAT_STRING, "Directory:", accumulo.getConfig().getDir().getAbsoluteFile())); + System.out.println(String.format(FORMAT_STRING, "Logs:", accumulo.getConfig().getImpl().getLogDir().getAbsoluteFile())); + System.out.println(String.format(FORMAT_STRING, "Instance Name:", accumulo.getConfig().getInstanceName())); + System.out.println(String.format(FORMAT_STRING, "Root Password:", accumulo.getConfig().getRootPassword())); + System.out.println(String.format(FORMAT_STRING, "ZooKeeper:", accumulo.getZooKeepers())); + + for (Pair pair : accumulo.getDebugPorts()) { + System.out.println(String.format(FORMAT_STRING, pair.getFirst().prettyPrint() + " JDWP Host:", "localhost:" + pair.getSecond())); + } + + System.out.println(String.format(FORMAT_STRING, "Shutdown Port:", shutdownPort)); + + System.out.println(); + System.out.println(" To connect with shell, use the following command : "); + System.out.println(" accumulo shell -zh " + accumulo.getZooKeepers() + " -zi " + accumulo.getConfig().getInstanceName() + " -u root "); + + System.out.println("\n\nSuccessfully started on " + new Date()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java index 442294f,0000000..05806ca mode 100644,000000..100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java @@@ -1,30 -1,0 +1,27 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.conf; + +import org.apache.accumulo.server.client.HdfsZooInstance; + +public class ConfigSanityCheck { + - /** - * @param args - */ + public static void main(String[] args) { + new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration(); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java index fd76ce2,0000000..5bd1632 mode 100644,000000..100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java @@@ -1,151 -1,0 +1,149 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.master.balancer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.master.thrift.TabletServerStatus; +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client; +import org.apache.accumulo.core.tabletserver.thrift.TabletStats; +import org.apache.accumulo.core.util.ThriftUtil; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.master.state.TServerInstance; +import org.apache.accumulo.server.master.state.TabletMigration; +import org.apache.accumulo.server.security.SystemCredentials; +import org.apache.accumulo.trace.instrument.Tracer; +import org.apache.log4j.Logger; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; + +public abstract class TabletBalancer { + + private static final Logger log = Logger.getLogger(TabletBalancer.class); + + protected ServerConfiguration configuration; + + /** + * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration. + */ + public void init(ServerConfiguration conf) { + configuration = conf; + } + + /** + * Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned. + * + * @param current + * The current table-summary state of all the online tablet servers. Read-only. The TabletServerStatus for each server may be null if the tablet + * server has not yet responded to a recent request for status. + * @param unassigned + * A map from unassigned tablet to the last known tablet server. Read-only. + * @param assignments + * A map from tablet to assigned server. Write-only. + */ + abstract public void getAssignments(SortedMap current, Map unassigned, + Map assignments); + + /** + * Ask the balancer if any migrations are necessary. + * + * @param current + * The current table-summary state of all the online tablet servers. Read-only. + * @param migrations + * the current set of migrations. Read-only. + * @param migrationsOut + * new migrations to perform; should not contain tablets in the current set of migrations. Write-only. + * @return the time, in milliseconds, to wait before re-balancing. + * + * This method will not be called when there are unassigned tablets. + */ + public abstract long balance(SortedMap current, Set migrations, List migrationsOut); + + /** + * Fetch the tablets for the given table by asking the tablet server. Useful if your balance strategy needs details at the tablet level to decide what tablets + * to move. + * + * @param tserver + * The tablet server to ask. + * @param tableId + * The table id + * @return a list of tablet statistics + * @throws ThriftSecurityException + * tablet server disapproves of your internal System password. + * @throws TException + * any other problem + */ + public List getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException { + log.debug("Scanning tablet server " + tserver + " for table " + tableId); + Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration()); + try { + List onlineTabletsForTable = client.getTabletStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(configuration.getInstance()), + tableId); + return onlineTabletsForTable; + } catch (TTransportException e) { + log.error("Unable to connect to " + tserver + ": " + e); + } finally { + ThriftUtil.returnClient(client); + } + return null; + } + + /** + * Utility to ensure that the migrations from balance() are consistent: + *
        + *
      • Tablet objects are not null + *
      • Source and destination tablet servers are not null and current + *
      + * - * @param current - * @param migrations + * @return A list of TabletMigration object that passed sanity checks. + */ + public static List checkMigrationSanity(Set current, List migrations) { + List result = new ArrayList(migrations.size()); + for (TabletMigration m : migrations) { + if (m.tablet == null) { + log.warn("Balancer gave back a null tablet " + m); + continue; + } + if (m.newServer == null) { + log.warn("Balancer did not set the destination " + m); + continue; + } + if (m.oldServer == null) { + log.warn("Balancer did not set the source " + m); + continue; + } + if (!current.contains(m.oldServer)) { + log.warn("Balancer wants to move a tablet from a server that is not current: " + m); + continue; + } + if (!current.contains(m.newServer)) { + log.warn("Balancer wants to move a tablet to a server that is not current: " + m); + continue; + } + result.add(m); + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/716ea0ee/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java index e898109,0000000..7c75454 mode 100644,000000..100644 --- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java @@@ -1,91 -1,0 +1,84 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.master.state; + +import java.util.Collection; +import java.util.Collections; + +/** + * Interface for storing information about tablet assignments. There are three implementations: + * + * ZooTabletStateStore: information about the root tablet is stored in ZooKeeper MetaDataStateStore: information about the other tablets are stored in the + * metadata table + * + */ +public abstract class TabletStateStore implements Iterable { + + /** + * Identifying name for this tablet state store. + */ + abstract public String name(); + + /** + * Scan the information about the tablets covered by this store + */ + @Override + abstract public ClosableIterator iterator(); + + /** + * Store the assigned locations in the data store. - * - * @param assignments - * @throws DistributedStoreException + */ + abstract public void setFutureLocations(Collection assignments) throws DistributedStoreException; + + /** + * Tablet servers will update the data store with the location when they bring the tablet online - * - * @param assignments - * @throws DistributedStoreException + */ + abstract public void setLocations(Collection assignments) throws DistributedStoreException; + + /** + * Mark the tablets as having no known or future location. + * + * @param tablets + * the tablets' current information - * @throws DistributedStoreException + */ + abstract public void unassign(Collection tablets) throws DistributedStoreException; + + public static void unassign(TabletLocationState tls) throws DistributedStoreException { + TabletStateStore store; + if (tls.extent.isRootTablet()) { + store = new ZooTabletStateStore(); + } else if (tls.extent.isMeta()) { + store = new RootTabletStateStore(); + } else { + store = new MetaDataStateStore(); + } + store.unassign(Collections.singletonList(tls)); + } + + public static void setLocation(Assignment assignment) throws DistributedStoreException { + TabletStateStore store; + if (assignment.tablet.isRootTablet()) { + store = new ZooTabletStateStore(); + } else if (assignment.tablet.isMeta()) { + store = new RootTabletStateStore(); + } else { + store = new MetaDataStateStore(); + } + store.setLocations(Collections.singletonList(assignment)); + } + +}