parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ziva...@apache.org
Subject [parquet-format] branch master updated: PARQUET-1227: Thrift crypto metadata structures (#94)
Date Mon, 23 Jul 2018 12:48:08 GMT
This is an automated email from the ASF dual-hosted git repository.

zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-format.git


The following commit(s) were added to refs/heads/master by this push:
     new 518e206  PARQUET-1227: Thrift crypto metadata structures (#94)
518e206 is described below

commit 518e206c3e6586b76e8315d5f62a8666ed62fa90
Author: ggershinsky <ggershinsky@users.noreply.github.com>
AuthorDate: Mon Jul 23 15:48:06 2018 +0300

    PARQUET-1227: Thrift crypto metadata structures (#94)
    
    New Thrift structures for Parquet modular encryption.
---
 src/main/thrift/parquet.thrift | 53 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/src/main/thrift/parquet.thrift b/src/main/thrift/parquet.thrift
index 3b15cfe..3a26579 100644
--- a/src/main/thrift/parquet.thrift
+++ b/src/main/thrift/parquet.thrift
@@ -660,6 +660,22 @@ struct ColumnMetaData {
   13: optional list<PageEncodingStats> encoding_stats;
 }
 
+struct EncryptionWithFooterKey {
+}
+
+struct EncryptionWithColumnKey {
+  /** Column path in schema **/
+  1: required list<string> path_in_schema
+  
+  /** Retrieval metadata of the column-specific key **/
+  2: optional binary column_key_metadata
+}
+
+union ColumnCryptoMetaData {
+  1: EncryptionWithFooterKey ENCRYPTION_WITH_FOOTER_KEY
+  2: EncryptionWithColumnKey ENCRYPTION_WITH_COLUMN_KEY
+}
+
 struct ColumnChunk {
   /** File where column data is stored.  If not set, assumed to be same file as
     * metadata.  This path is relative to the current file.
@@ -686,6 +702,9 @@ struct ColumnChunk {
 
   /** Size of ColumnChunk's ColumnIndex, in bytes **/
   7: optional i32 column_index_length
+  
+  /** Crypto metadata of encrypted columns **/
+  8: optional ColumnCryptoMetaData crypto_meta_data
 }
 
 struct RowGroup {
@@ -877,3 +896,37 @@ struct FileMetaData {
   7: optional list<ColumnOrder> column_orders;
 }
 
+struct AesGcmV1 {
+  /** Retrieval metadata of AAD used for encryption of pages and structures **/
+  1: optional binary aad_metadata
+}
+
+struct AesGcmCtrV1 {
+  /** Retrieval metadata of AAD used for encryption of structures **/
+  1: optional binary aad_metadata
+}
+
+union EncryptionAlgorithm {
+  1: AesGcmV1 AES_GCM_V1
+  2: AesGcmCtrV1 AES_GCM_CTR_V1
+}
+
+struct FileCryptoMetaData {
+  1: required EncryptionAlgorithm encryption_algorithm
+  
+  /** Parquet footer can be encrypted, or left as plaintext **/
+  2: required bool encrypted_footer
+    
+  /** Retrieval metadata of key used for encryption of footer, 
+   *  and (possibly) columns **/
+  3: optional binary footer_key_metadata
+
+  /** Offset of Parquet footer (encrypted, or plaintext) **/
+  4: required i64 footer_offset
+  
+  /** If file IVs are comprised of a fixed part,
+   *  and variable parts (random or counter), keep the fixed
+   *  part here **/
+  5: optional binary iv_prefix
+}
+


Mime
View raw message