From dev-return-1813-archive-asf-public=cust-asf.ponee.io@orc.apache.org Wed Feb 7 18:31:44 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id AC9D118065B for ; Wed, 7 Feb 2018 18:31:44 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 9C7E8160C5B; Wed, 7 Feb 2018 17:31:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 93C63160C3A for ; Wed, 7 Feb 2018 18:31:43 +0100 (CET) Received: (qmail 51863 invoked by uid 500); 7 Feb 2018 17:31:42 -0000 Mailing-List: contact dev-help@orc.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@orc.apache.org Delivered-To: mailing list dev@orc.apache.org Received: (qmail 51850 invoked by uid 99); 7 Feb 2018 17:31:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Feb 2018 17:31:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0977EDFC25; Wed, 7 Feb 2018 17:31:42 +0000 (UTC) From: omalley To: dev@orc.apache.org Reply-To: dev@orc.apache.org References: In-Reply-To: Subject: [GitHub] orc pull request #213: ORC-278 - Create in memory KeyProvider class Content-Type: text/plain Message-Id: <20180207173142.0977EDFC25@git1-us-west.apache.org> Date: Wed, 7 Feb 2018 17:31:42 +0000 (UTC) Github user omalley commented on a diff in the pull request: https://github.com/apache/orc/pull/213#discussion_r166687574 --- Diff: java/core/src/java/org/apache/orc/InMemoryKeystore.java --- @@ -0,0 +1,403 @@ +package org.apache.orc; + +import org.apache.commons.lang.StringUtils; +import org.apache.orc.impl.HadoopShims; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is an in-memory implementation of {@link HadoopShims.KeyProvider}. + * The primary use of this class is to ease testing. + */ +public class InMemoryKeystore implements HadoopShims.KeyProvider { + + /** + * Support AES 256 ? + */ + public static final boolean SUPPORTS_AES_256; + + static { + try { + SUPPORTS_AES_256 = Cipher.getMaxAllowedKeyLength("AES") > 128; + } catch (final NoSuchAlgorithmException e) { + throw new IllegalArgumentException("Unknown algorithm", e); + } + } + + /** + * A map that stores the 'keyName@version' + * and 'metadata + material' mapping. + */ + private final Map keys; + + /** + * A map that store the keyName (without version) + * and metadata associated with it. + */ + private final Map meta = new HashMap<>(); + + private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(true); + + /* Create an instance */ + public InMemoryKeystore() { + this(new HashMap()); + } + + /** + * Create an instance populating the given keys. + * + * @param keys Supplied map of keys + */ + public InMemoryKeystore(final Map keys) { + super(); + this.keys = keys; + } + + /** + * Build a version string from a basename and version number. Converts + * "/aaa/bbb" and 3 to "/aaa/bbb@3". + * + * @param name the basename of the key + * @param version the version of the key + * @return the versionName of the key. + */ + protected static String buildVersionName(final String name, + final int version) { + return name + "@" + version; + } + + /** + * Get the list of key names from the key provider. + * + * @return a list of key names + * @throws IOException + */ + @Override + public List getKeyNames() throws IOException { + return new ArrayList<>(meta.keySet()); + } + + /** + * Get the current metadata for a given key. This is used when encrypting + * new data. + * + * @param keyName the name of a key + * @return metadata for the current version of the key + */ + @Override + public HadoopShims.KeyMetadata getCurrentKeyVersion(final String keyName) { + + /* prevent the material from leaking */ + final HadoopShims.KeyMetadata key = meta.get(keyName); + + if (key == null) { + return null; + } + + return new HadoopShims.KeyMetadata() { + + @Override + public String getKeyName() { + return key.getKeyName(); + } + + @Override + public EncryptionAlgorithm getAlgorithm() { + return key.getAlgorithm(); + } + + @Override + public int getVersion() { + return key.getVersion(); + } + }; + + } + + /** + * Create a metadata object while reading. + * + * @param keyName the name of the key + * @param version the version of the key to use + * @param algorithm the algorithm for that version of the key + * @return the metadata for the key version + */ + @Override + public HadoopShims.KeyMetadata getKeyVersion(final String keyName, + final int version, final EncryptionAlgorithm algorithm) { + return new HadoopShims.KeyMetadata() { + + @Override + public String getKeyName() { + return keyName; + } + + @Override + public EncryptionAlgorithm getAlgorithm() { + return algorithm; + } + + @Override + public int getVersion() { + return version; + } + }; + } + + /** + * Create a local key for the given key version and initialization vector. + * Given a probabilistically unique iv, it will generate a unique key + * with the master key at the specified version. This allows the encryption + * to use this local key for the encryption and decryption without ever + * having access to the master key. + *

+ * This uses KeyProviderCryptoExtension.decryptEncryptedKey with a fixed key + * of the appropriate length. + * + * @param key the master key version + * @param iv the unique initialization vector + * @return the local key's material + */ + @Override + public Key getLocalKey(final HadoopShims.KeyMetadata key, final byte[] iv) { + + rwl.writeLock().lock(); + try { + final KeyVersion secret = keys + .get(buildVersionName(key.getKeyName(), key.getVersion())); + final Cipher cipher = secret.getAlgorithm().createCipher(); + + try { + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(secret.getMaterial(), + secret.getAlgorithm().getAlgorithm()), new IvParameterSpec(iv)); + } catch (final InvalidKeyException e) { + throw new IllegalStateException( + "ORC bad encryption key for " + key.getKeyName(), e); + } catch (final InvalidAlgorithmParameterException e) { + throw new IllegalStateException( + "ORC bad encryption parameter for " + key.getKeyName(), e); + } + + final byte[] buffer = new byte[secret.getAlgorithm().keyLength()]; + + try { + cipher.doFinal(buffer); + } catch (final IllegalBlockSizeException e) { + throw new IllegalStateException( + "ORC bad block size for " + key.getKeyName(), e); + } catch (final BadPaddingException e) { + throw new IllegalStateException( + "ORC bad padding for " + key.getKeyName(), e); + } + + return new SecretKeySpec(buffer, secret.getAlgorithm().getAlgorithm()); + } finally { + rwl.writeLock().unlock(); + } + + } + + /** + * Function to add keys to the in-memory KeyStore. + * + * @param keyName Name of the key to be added + * @param algorithm Algorithm used + * @param version Key Version + * @param masterKey Master key + */ + public void addKey(String keyName, EncryptionAlgorithm algorithm, int version, + byte[] masterKey) throws IOException { + + if (!keyName.equals(StringUtils.lowerCase(keyName))) { + throw new IOException("Key " + keyName + " must be lower case."); + } + + rwl.writeLock().lock(); + try { + + if (meta.containsKey(keyName)) { + throw new IOException("Key " + keyName + " already exists."); + } + + internalAddKey(keyName, algorithm, version, masterKey); + + } finally { + rwl.writeLock().unlock(); + } + + } + + /** + * Function that takes care of adding the key. + * NOTE: Here, if the key already exists with same name then this function overwrites it, + * this is by design and is used by {@link #rollNewVersion(String, byte[])} function. + * Also note, this function is not thread safe, thread safety is the + * responsibility of the calling function. + * + * @param keyName Name of the key to be added + * @param algorithm Algorithm used + * @param version Key Version + * @param masterKey Master key + * @return The new key + */ + private KeyVersion internalAddKey(String keyName, + EncryptionAlgorithm algorithm, int version, byte[] masterKey) { + + /* Test weather platform supports the algorithm */ + if (!SUPPORTS_AES_256 && (algorithm != EncryptionAlgorithm.AES_128)) { + algorithm = EncryptionAlgorithm.AES_128; + } + + final byte[] buffer = new byte[algorithm.keyLength()]; + if (algorithm.keyLength() > masterKey.length) { + + System.arraycopy(masterKey, 0, buffer, 0, masterKey.length); + /* fill with zeros */ + Arrays.fill(buffer, masterKey.length, buffer.length - 1, (byte) 0); + + } else { + System.arraycopy(masterKey, 0, buffer, 0, algorithm.keyLength()); + } + + final KeyVersion key = new KeyVersion(keyName, version, algorithm, buffer); + + keys.put(buildVersionName(keyName, version), key); + /* our metadata also contains key material, but it should be fine for testing */ + meta.put(keyName, key); + + return key; + + } + + /** + * Function to add keys to the in-memory KeyStore. + * With default version 0 + * + * @param keyName Name of the key to be added + * @param algorithm Algorithm used + * @param masterKey Master key + */ + public void addKey(String keyName, EncryptionAlgorithm algorithm, + byte[] masterKey) throws IOException { + + /* default version = 0 */ + this.addKey(keyName, algorithm, 0, masterKey); + } + + /** + * Roll new version for the key i.e. + * increments the version number by 1 and + * uses the provided secret material. + * + * @param name name of the key to be rolled + * @param material the new material to be used + * @return KeyVersion The new rolled key + * @throws IOException + */ + public KeyVersion rollNewVersion(final String name, final byte[] material) + throws IOException { + + rwl.writeLock().lock(); + try { + final HadoopShims.KeyMetadata metadata = meta.get(name); + final int newVersion = metadata.getVersion() + 1; + return internalAddKey(name, metadata.getAlgorithm(), newVersion, + material); + + } finally { + rwl.writeLock().unlock(); + } + } + +} + +/** + * This class contains the meta-data and the material for the key. + */ +class KeyVersion implements HadoopShims.KeyMetadata { --- End diff -- You probably want this to be a static inner class. ---