Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5E572200B5E for ; Tue, 26 Jul 2016 14:58:32 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 578AF160A6F; Tue, 26 Jul 2016 12:58:32 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BD9DE160A75 for ; Tue, 26 Jul 2016 14:58:29 +0200 (CEST) Received: (qmail 34511 invoked by uid 500); 26 Jul 2016 12:58:29 -0000 Mailing-List: contact commits-help@pirk.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pirk.incubator.apache.org Delivered-To: mailing list commits@pirk.incubator.apache.org Received: (qmail 34502 invoked by uid 99); 26 Jul 2016 12:58:28 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Jul 2016 12:58:28 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 64F12187B04 for ; Tue, 26 Jul 2016 12:58:28 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.646 X-Spam-Level: X-Spam-Status: No, score=-4.646 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.426] autolearn=disabled Received: from mx2-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id TX-mM2IEehIv for ; Tue, 26 Jul 2016 12:58:14 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-us.apache.org (ASF Mail Server at mx2-lw-us.apache.org) with SMTP id 76DAC61222 for ; Tue, 26 Jul 2016 12:58:13 +0000 (UTC) Received: (qmail 33632 invoked by uid 99); 26 Jul 2016 12:58:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Jul 2016 12:58:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B24C6E03C0; Tue, 26 Jul 2016 12:58:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: eawilliams@apache.org To: commits@pirk.incubator.apache.org Date: Tue, 26 Jul 2016 12:58:13 -0000 Message-Id: <329d1cfe57a743999631fac682b4d3ab@git.apache.org> In-Reply-To: <76dbc0ca4e78456d84fb57a8ff1e19ad@git.apache.org> References: <76dbc0ca4e78456d84fb57a8ff1e19ad@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/7] incubator-pirk git commit: [PIRK-19] Make DataSchema/QuerySchema Agnostic of Persistent Representation, also updated javadocs and formatted source - closes apache/incubator-pirk#26 archived-at: Tue, 26 Jul 2016 12:58:32 -0000 http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/class-use/QueryParserUtils.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/class-use/QueryParserUtils.html b/docs/org/apache/pirk/utils/class-use/QueryParserUtils.html index bf049ca..a45bb2d 100644 --- a/docs/org/apache/pirk/utils/class-use/QueryParserUtils.html +++ b/docs/org/apache/pirk/utils/class-use/QueryParserUtils.html @@ -2,9 +2,9 @@ - + Uses of Class org.apache.pirk.utils.QueryParserUtils - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/class-use/StopListUtils.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/class-use/StopListUtils.html b/docs/org/apache/pirk/utils/class-use/StopListUtils.html index 74558ca..6a34ce7 100644 --- a/docs/org/apache/pirk/utils/class-use/StopListUtils.html +++ b/docs/org/apache/pirk/utils/class-use/StopListUtils.html @@ -2,9 +2,9 @@ - + Uses of Class org.apache.pirk.utils.StopListUtils - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/class-use/StringUtils.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/class-use/StringUtils.html b/docs/org/apache/pirk/utils/class-use/StringUtils.html index d3e1f6e..9bab0e5 100644 --- a/docs/org/apache/pirk/utils/class-use/StringUtils.html +++ b/docs/org/apache/pirk/utils/class-use/StringUtils.html @@ -2,9 +2,9 @@ - + Uses of Class org.apache.pirk.utils.StringUtils - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/class-use/SystemConfiguration.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/class-use/SystemConfiguration.html b/docs/org/apache/pirk/utils/class-use/SystemConfiguration.html index 2adb2b7..20130a7 100644 --- a/docs/org/apache/pirk/utils/class-use/SystemConfiguration.html +++ b/docs/org/apache/pirk/utils/class-use/SystemConfiguration.html @@ -2,9 +2,9 @@ - + Uses of Class org.apache.pirk.utils.SystemConfiguration - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/package-frame.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/package-frame.html b/docs/org/apache/pirk/utils/package-frame.html index 683163f..6475e80 100644 --- a/docs/org/apache/pirk/utils/package-frame.html +++ b/docs/org/apache/pirk/utils/package-frame.html @@ -2,9 +2,9 @@ - + org.apache.pirk.utils - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/package-summary.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/package-summary.html b/docs/org/apache/pirk/utils/package-summary.html index f99270c..3430e79 100644 --- a/docs/org/apache/pirk/utils/package-summary.html +++ b/docs/org/apache/pirk/utils/package-summary.html @@ -2,9 +2,9 @@ - + org.apache.pirk.utils - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/package-tree.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/package-tree.html b/docs/org/apache/pirk/utils/package-tree.html index 6f31a20..fa25474 100644 --- a/docs/org/apache/pirk/utils/package-tree.html +++ b/docs/org/apache/pirk/utils/package-tree.html @@ -2,9 +2,9 @@ - + org.apache.pirk.utils Class Hierarchy - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/org/apache/pirk/utils/package-use.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/utils/package-use.html b/docs/org/apache/pirk/utils/package-use.html index 258c4de..8a51046 100644 --- a/docs/org/apache/pirk/utils/package-use.html +++ b/docs/org/apache/pirk/utils/package-use.html @@ -2,9 +2,9 @@ - + Uses of Package org.apache.pirk.utils - + @@ -96,10 +96,14 @@   -org.apache.pirk.test.utils +org.apache.pirk.schema.data   +org.apache.pirk.test.utils +  + + org.apache.pirk.utils   @@ -191,6 +195,23 @@ +
  • + + + + + + + + + + + + +
    Classes in org.apache.pirk.utils used by org.apache.pirk.schema.data 
    Class and Description
    PIRException +
    Custom exception for the PIR utilities
    +
    +
  • http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/overview-frame.html ---------------------------------------------------------------------- diff --git a/docs/overview-frame.html b/docs/overview-frame.html index c8e152a..dd9f53e 100644 --- a/docs/overview-frame.html +++ b/docs/overview-frame.html @@ -2,9 +2,9 @@ - + Overview List - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/overview-summary.html ---------------------------------------------------------------------- diff --git a/docs/overview-summary.html b/docs/overview-summary.html index 3db2b5e..5c132a4 100644 --- a/docs/overview-summary.html +++ b/docs/overview-summary.html @@ -2,9 +2,9 @@ - + Overview - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/overview-tree.html ---------------------------------------------------------------------- diff --git a/docs/overview-tree.html b/docs/overview-tree.html index b759753..eefb6ad 100644 --- a/docs/overview-tree.html +++ b/docs/overview-tree.html @@ -2,9 +2,9 @@ - + Class Hierarchy - + @@ -121,6 +121,8 @@
  • org.apache.pirk.utils.CSVOutputUtils
  • org.apache.pirk.schema.data.DataSchema (implements java.io.Serializable)
    • +
  • org.apache.pirk.schema.data.DataSchemaLoader
    • +
  • org.apache.pirk.schema.data.DataSchemaRegistry
  • org.apache.pirk.querier.wideskies.decrypt.DecryptResponse
  • org.apache.pirk.querier.wideskies.decrypt.DecryptResponseRunnable (implements java.lang.Runnable)
  • org.apache.pirk.test.distributed.DistributedTestCLI
    • @@ -166,7 +168,6 @@
  • org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner (implements org.apache.pirk.schema.data.partitioner.DataPartitioner)
  • org.apache.pirk.utils.KeyedHash
  • test.general.KeyedHashTest
    • -
  • org.apache.pirk.schema.data.LoadDataSchemas
  • test.schema.data.LoadDataSchemaTest
  • org.apache.pirk.schema.query.LoadQuerySchemas
  • test.schema.query.LoadQuerySchemaTest
    • http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/serialized-form.html ---------------------------------------------------------------------- diff --git a/docs/serialized-form.html b/docs/serialized-form.html index 3680132..b7c5a9b 100644 --- a/docs/serialized-form.html +++ b/docs/serialized-form.html @@ -2,9 +2,9 @@ - + Serialized Form - + @@ -689,20 +689,16 @@ 
    java.lang.String schemaName
  • -

    primitiveTypePartitionerName

    -
    java.lang.String primitiveTypePartitionerName
    -
    • -

  • typeMap

    -
    java.util.HashMap<K,V> typeMap
    +
    java.util.Map<K,V> typeMap
  • -

    partitionerMap

    -
    java.util.HashMap<K,V> partitionerMap
    +

    partitionerTypeMap

    +
    java.util.Map<K,V> partitionerTypeMap
  • -

    listRep

    -
    java.util.HashSet<E> listRep
    +

    arrayElements

    +
    java.util.Set<E> arrayElements
    • http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/ISO8601DateParserTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/ISO8601DateParserTest.html b/docs/test/general/ISO8601DateParserTest.html index fa77dd9..a417e44 100644 --- a/docs/test/general/ISO8601DateParserTest.html +++ b/docs/test/general/ISO8601DateParserTest.html @@ -2,9 +2,9 @@ - + ISO8601DateParserTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/KeyedHashTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/KeyedHashTest.html b/docs/test/general/KeyedHashTest.html index f9c8d66..e46f861 100644 --- a/docs/test/general/KeyedHashTest.html +++ b/docs/test/general/KeyedHashTest.html @@ -2,9 +2,9 @@ - + KeyedHashTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/PaillierTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/PaillierTest.html b/docs/test/general/PaillierTest.html index 588b543..6d57ef0 100644 --- a/docs/test/general/PaillierTest.html +++ b/docs/test/general/PaillierTest.html @@ -2,9 +2,9 @@ - + PaillierTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/PartitionUtilsTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/PartitionUtilsTest.html b/docs/test/general/PartitionUtilsTest.html index ffcb626..ded49e9 100644 --- a/docs/test/general/PartitionUtilsTest.html +++ b/docs/test/general/PartitionUtilsTest.html @@ -2,9 +2,9 @@ - + PartitionUtilsTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/QueryParserUtilsTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/QueryParserUtilsTest.html b/docs/test/general/QueryParserUtilsTest.html index fd3aaad..5b31834 100644 --- a/docs/test/general/QueryParserUtilsTest.html +++ b/docs/test/general/QueryParserUtilsTest.html @@ -2,9 +2,9 @@ - + QueryParserUtilsTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/class-use/ISO8601DateParserTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/class-use/ISO8601DateParserTest.html b/docs/test/general/class-use/ISO8601DateParserTest.html index 4a29a10..d2e5772 100644 --- a/docs/test/general/class-use/ISO8601DateParserTest.html +++ b/docs/test/general/class-use/ISO8601DateParserTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.general.ISO8601DateParserTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/class-use/KeyedHashTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/class-use/KeyedHashTest.html b/docs/test/general/class-use/KeyedHashTest.html index b581eaf..1663492 100644 --- a/docs/test/general/class-use/KeyedHashTest.html +++ b/docs/test/general/class-use/KeyedHashTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.general.KeyedHashTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/class-use/PaillierTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/class-use/PaillierTest.html b/docs/test/general/class-use/PaillierTest.html index e0645d0..f9261d3 100644 --- a/docs/test/general/class-use/PaillierTest.html +++ b/docs/test/general/class-use/PaillierTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.general.PaillierTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/class-use/PartitionUtilsTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/class-use/PartitionUtilsTest.html b/docs/test/general/class-use/PartitionUtilsTest.html index 41e5e29..687f58a 100644 --- a/docs/test/general/class-use/PartitionUtilsTest.html +++ b/docs/test/general/class-use/PartitionUtilsTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.general.PartitionUtilsTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/class-use/QueryParserUtilsTest.html ---------------------------------------------------------------------- diff --git a/docs/test/general/class-use/QueryParserUtilsTest.html b/docs/test/general/class-use/QueryParserUtilsTest.html index eeb04d3..dbf93a4 100644 --- a/docs/test/general/class-use/QueryParserUtilsTest.html +++ b/docs/test/general/class-use/QueryParserUtilsTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.general.QueryParserUtilsTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/package-frame.html ---------------------------------------------------------------------- diff --git a/docs/test/general/package-frame.html b/docs/test/general/package-frame.html index 6116544..0398c7d 100644 --- a/docs/test/general/package-frame.html +++ b/docs/test/general/package-frame.html @@ -2,9 +2,9 @@ - + test.general - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/package-summary.html ---------------------------------------------------------------------- diff --git a/docs/test/general/package-summary.html b/docs/test/general/package-summary.html index 0d0308e..752d26e 100644 --- a/docs/test/general/package-summary.html +++ b/docs/test/general/package-summary.html @@ -2,9 +2,9 @@ - + test.general - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/package-tree.html ---------------------------------------------------------------------- diff --git a/docs/test/general/package-tree.html b/docs/test/general/package-tree.html index c796c9b..e6e042f 100644 --- a/docs/test/general/package-tree.html +++ b/docs/test/general/package-tree.html @@ -2,9 +2,9 @@ - + test.general Class Hierarchy - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/general/package-use.html ---------------------------------------------------------------------- diff --git a/docs/test/general/package-use.html b/docs/test/general/package-use.html index b1ed8bd..04dffae 100644 --- a/docs/test/general/package-use.html +++ b/docs/test/general/package-use.html @@ -2,9 +2,9 @@ - + Uses of Package test.general - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/LoadDataSchemaTest.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/LoadDataSchemaTest.html b/docs/test/schema/data/LoadDataSchemaTest.html index 8fb0bd5..35c003d 100644 --- a/docs/test/schema/data/LoadDataSchemaTest.html +++ b/docs/test/schema/data/LoadDataSchemaTest.html @@ -2,9 +2,9 @@ - + LoadDataSchemaTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/class-use/LoadDataSchemaTest.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/class-use/LoadDataSchemaTest.html b/docs/test/schema/data/class-use/LoadDataSchemaTest.html index 2562414..2bb1ca5 100644 --- a/docs/test/schema/data/class-use/LoadDataSchemaTest.html +++ b/docs/test/schema/data/class-use/LoadDataSchemaTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.schema.data.LoadDataSchemaTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/package-frame.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/package-frame.html b/docs/test/schema/data/package-frame.html index e3babdc..581cb62 100644 --- a/docs/test/schema/data/package-frame.html +++ b/docs/test/schema/data/package-frame.html @@ -2,9 +2,9 @@ - + test.schema.data - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/package-summary.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/package-summary.html b/docs/test/schema/data/package-summary.html index dd0fbf2..916519c 100644 --- a/docs/test/schema/data/package-summary.html +++ b/docs/test/schema/data/package-summary.html @@ -2,9 +2,9 @@ - + test.schema.data - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/package-tree.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/package-tree.html b/docs/test/schema/data/package-tree.html index 6a2e2a6..38d5f72 100644 --- a/docs/test/schema/data/package-tree.html +++ b/docs/test/schema/data/package-tree.html @@ -2,9 +2,9 @@ - + test.schema.data Class Hierarchy - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/data/package-use.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/data/package-use.html b/docs/test/schema/data/package-use.html index 7e6bca7..f84fcdb 100644 --- a/docs/test/schema/data/package-use.html +++ b/docs/test/schema/data/package-use.html @@ -2,9 +2,9 @@ - + Uses of Package test.schema.data - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/LoadQuerySchemaTest.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/LoadQuerySchemaTest.html b/docs/test/schema/query/LoadQuerySchemaTest.html index 278ef00..8b95753 100644 --- a/docs/test/schema/query/LoadQuerySchemaTest.html +++ b/docs/test/schema/query/LoadQuerySchemaTest.html @@ -2,9 +2,9 @@ - + LoadQuerySchemaTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/class-use/LoadQuerySchemaTest.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/class-use/LoadQuerySchemaTest.html b/docs/test/schema/query/class-use/LoadQuerySchemaTest.html index 199e3dc..95bf610 100644 --- a/docs/test/schema/query/class-use/LoadQuerySchemaTest.html +++ b/docs/test/schema/query/class-use/LoadQuerySchemaTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.schema.query.LoadQuerySchemaTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/package-frame.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/package-frame.html b/docs/test/schema/query/package-frame.html index 8269a7f..6837b2c 100644 --- a/docs/test/schema/query/package-frame.html +++ b/docs/test/schema/query/package-frame.html @@ -2,9 +2,9 @@ - + test.schema.query - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/package-summary.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/package-summary.html b/docs/test/schema/query/package-summary.html index d57cd2f..9cdc2b4 100644 --- a/docs/test/schema/query/package-summary.html +++ b/docs/test/schema/query/package-summary.html @@ -2,9 +2,9 @@ - + test.schema.query - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/package-tree.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/package-tree.html b/docs/test/schema/query/package-tree.html index cf2e993..aa8c08c 100644 --- a/docs/test/schema/query/package-tree.html +++ b/docs/test/schema/query/package-tree.html @@ -2,9 +2,9 @@ - + test.schema.query Class Hierarchy - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/schema/query/package-use.html ---------------------------------------------------------------------- diff --git a/docs/test/schema/query/package-use.html b/docs/test/schema/query/package-use.html index 65b49f7..cc965d3 100644 --- a/docs/test/schema/query/package-use.html +++ b/docs/test/schema/query/package-use.html @@ -2,9 +2,9 @@ - + Uses of Package test.schema.query - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/StandaloneTest.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/StandaloneTest.html b/docs/test/wideskies/standalone/StandaloneTest.html index 71b91e5..9639e4f 100644 --- a/docs/test/wideskies/standalone/StandaloneTest.html +++ b/docs/test/wideskies/standalone/StandaloneTest.html @@ -2,9 +2,9 @@ - + StandaloneTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/class-use/StandaloneTest.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/class-use/StandaloneTest.html b/docs/test/wideskies/standalone/class-use/StandaloneTest.html index 6b1f7e5..78541c1 100644 --- a/docs/test/wideskies/standalone/class-use/StandaloneTest.html +++ b/docs/test/wideskies/standalone/class-use/StandaloneTest.html @@ -2,9 +2,9 @@ - + Uses of Class test.wideskies.standalone.StandaloneTest - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/package-frame.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/package-frame.html b/docs/test/wideskies/standalone/package-frame.html index 9c38b32..b77fe32 100644 --- a/docs/test/wideskies/standalone/package-frame.html +++ b/docs/test/wideskies/standalone/package-frame.html @@ -2,9 +2,9 @@ - + test.wideskies.standalone - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/package-summary.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/package-summary.html b/docs/test/wideskies/standalone/package-summary.html index 26f300f..0e06d57 100644 --- a/docs/test/wideskies/standalone/package-summary.html +++ b/docs/test/wideskies/standalone/package-summary.html @@ -2,9 +2,9 @@ - + test.wideskies.standalone - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/package-tree.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/package-tree.html b/docs/test/wideskies/standalone/package-tree.html index 3458651..4dd8995 100644 --- a/docs/test/wideskies/standalone/package-tree.html +++ b/docs/test/wideskies/standalone/package-tree.html @@ -2,9 +2,9 @@ - + test.wideskies.standalone Class Hierarchy - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/docs/test/wideskies/standalone/package-use.html ---------------------------------------------------------------------- diff --git a/docs/test/wideskies/standalone/package-use.html b/docs/test/wideskies/standalone/package-use.html index 55f55c5..c04b4c8 100644 --- a/docs/test/wideskies/standalone/package-use.html +++ b/docs/test/wideskies/standalone/package-use.html @@ -2,9 +2,9 @@ - + Uses of Package test.wideskies.standalone - + http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/findbugs-exclude.xml b/findbugs-exclude.xml new file mode 100644 index 0000000..e30d682 --- /dev/null +++ b/findbugs-exclude.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/pom-with-benchmarks.xml ---------------------------------------------------------------------- diff --git a/pom-with-benchmarks.xml b/pom-with-benchmarks.xml index 053898e..9c85639 100644 --- a/pom-with-benchmarks.xml +++ b/pom-with-benchmarks.xml @@ -70,12 +70,7 @@ always - - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - - + conjars.org http://conjars.org/repo @@ -104,29 +99,29 @@ commons-math3 3.3 - + org.apache.hadoop - hadoop-yarn-client - 2.6.0-cdh5.6.0 + hadoop-common + 2.7.2 - org.apache.hbase - hbase-client - 1.0.0-cdh5.6.0 + org.apache.hadoop + hadoop-mapreduce-client-core + 2.7.2 - org.apache.hbase - hbase-it - 1.0.0-cdh5.6.0 + org.apache.hadoop + hadoop-client + 2.7.2 org.apache.spark - spark-core_2.10 - 1.5.0-cdh5.6.0 + spark-core_2.11 + 1.6.1 @@ -141,12 +136,6 @@ 3.3 - - org.apache.spark - spark-mllib_2.10 - 1.3.0 - - org.elasticsearch elasticsearch-hadoop http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3d4c1b5..1435862 100644 --- a/pom.xml +++ b/pom.xml @@ -301,6 +301,7 @@ .travis.yml appveyor.yml + findbugs-exclude.xml http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/inputformat/hadoop/json/JSONRecordReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/inputformat/hadoop/json/JSONRecordReader.java b/src/main/java/org/apache/pirk/inputformat/hadoop/json/JSONRecordReader.java index 80fd64f..06e395f 100644 --- a/src/main/java/org/apache/pirk/inputformat/hadoop/json/JSONRecordReader.java +++ b/src/main/java/org/apache/pirk/inputformat/hadoop/json/JSONRecordReader.java @@ -30,7 +30,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; import org.apache.pirk.inputformat.hadoop.TextArrayWritable; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.utils.QueryParserUtils; import org.apache.pirk.utils.StringUtils; import org.apache.pirk.utils.SystemConfiguration; @@ -72,13 +73,13 @@ public class JSONRecordReader extends RecordReader try { SystemConfiguration.setProperty("data.schemas", context.getConfiguration().get("data.schemas")); - LoadDataSchemas.initialize(true, fs); + DataSchemaLoader.initialize(true, fs); } catch (Exception e) { e.printStackTrace(); } String dataSchemaName = context.getConfiguration().get("dataSchemaName"); - dataSchema = LoadDataSchemas.getSchema(dataSchemaName); + dataSchema = DataSchemaRegistry.get(dataSchemaName); } @Override @@ -176,7 +177,7 @@ public class JSONRecordReader extends RecordReader Text mapValue = new Text(); if (jsonObj.get(key) != null) { - if (dataSchema.hasListRep(key.toString())) + if (dataSchema.isArrayElement(key.toString())) { String[] elements = StringUtils.jsonArrayStringToList(jsonObj.get(key).toString()); TextArrayWritable aw = new TextArrayWritable(elements); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java index 193617a..879f028 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java @@ -24,7 +24,7 @@ import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; @@ -315,7 +315,7 @@ public class QuerierDriverCLI + SystemConfiguration.getProperty("query.schemas")); try { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); LoadQuerySchemas.initialize(); } catch (Exception e) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java index b3c9be9..df38851 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java @@ -31,7 +31,7 @@ import org.apache.pirk.query.wideskies.Query; import org.apache.pirk.query.wideskies.QueryInfo; import org.apache.pirk.query.wideskies.QueryUtils; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.schema.query.QuerySchema; import org.apache.pirk.utils.KeyedHash; @@ -174,7 +174,7 @@ public class EncryptQuery private void populateEmbeddedSelectorMap() { QuerySchema qSchema = LoadQuerySchemas.getSchema(queryInfo.getQueryType()); - DataSchema dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + DataSchema dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); String type = dSchema.getElementType(qSchema.getSelectorName()); int sNum = 0; for (String selector : selectors) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java b/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java index 2c67866..8344630 100644 --- a/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java +++ b/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java @@ -29,7 +29,7 @@ import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.data.partitioner.DataPartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; import org.apache.pirk.schema.query.QuerySchema; @@ -56,7 +56,7 @@ public class QueryUtils { QueryResponseJSON qrJSON = new QueryResponseJSON(queryInfo); - DataSchema dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + DataSchema dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); int numArrayElementsToReturn = Integer.parseInt(SystemConfiguration.getProperty("pir.numReturnArrayElements", "1")); @@ -79,7 +79,7 @@ public class QueryUtils for (String fieldName : dataFieldsToExtract) { int numElements = 1; - if (dSchema.hasListRep(fieldName)) + if (dSchema.isArrayElement(fieldName)) { numElements = numArrayElementsToReturn; } @@ -107,7 +107,7 @@ public class QueryUtils public static ArrayList partitionDataElement(QuerySchema qSchema, JSONObject jsonData, boolean embedSelector) throws Exception { ArrayList parts = new ArrayList<>(); - DataSchema dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + DataSchema dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); // Add the embedded selector to the parts if (embedSelector) @@ -131,7 +131,7 @@ public class QueryUtils dataElement = jsonData.get(fieldName); } - if (dSchema.hasListRep(fieldName)) + if (dSchema.isArrayElement(fieldName)) { List elementArray; if (dataElement == null) @@ -189,12 +189,12 @@ public class QueryUtils for (String fieldName : dataFieldsToExtract) { Object dataElement = null; - if (dataMap.containsKey(dSchema.getTextElement(fieldName))) + if (dataMap.containsKey(dSchema.getTextName(fieldName))) { - dataElement = dataMap.get(dSchema.getTextElement(fieldName)); + dataElement = dataMap.get(dSchema.getTextName(fieldName)); } - if (dSchema.hasListRep(fieldName)) + if (dSchema.isArrayElement(fieldName)) { List elementArray = null; if (dataElement == null) @@ -305,22 +305,22 @@ public class QueryUtils String selector; String fieldName = qSchema.getSelectorName(); - if (dSchema.hasListRep(fieldName)) + if (dSchema.isArrayElement(fieldName)) { - if (dataMap.get(dSchema.getTextElement(fieldName)) instanceof WritableArrayWritable) + if (dataMap.get(dSchema.getTextName(fieldName)) instanceof WritableArrayWritable) { - String[] selectorArray = ((WritableArrayWritable) dataMap.get(dSchema.getTextElement(fieldName))).toStrings(); + String[] selectorArray = ((WritableArrayWritable) dataMap.get(dSchema.getTextName(fieldName))).toStrings(); selector = selectorArray[0]; } else { - String[] elementArray = ((ArrayWritable) (dataMap.get(dSchema.getTextElement(fieldName)))).toStrings(); + String[] elementArray = ((ArrayWritable) (dataMap.get(dSchema.getTextName(fieldName)))).toStrings(); selector = elementArray[0]; } } else { - selector = dataMap.get(dSchema.getTextElement(fieldName)).toString(); + selector = dataMap.get(dSchema.getTextName(fieldName)).toString(); } return selector; @@ -335,10 +335,10 @@ public class QueryUtils { String selector; - DataSchema dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + DataSchema dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); String fieldName = qSchema.getSelectorName(); - if (dSchema.hasListRep(fieldName)) + if (dSchema.isArrayElement(fieldName)) { ArrayList elementArray = StringUtils.jsonArrayStringToArrayList(dataMap.get(fieldName).toString()); selector = elementArray.get(0); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/ResponderCLI.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/ResponderCLI.java b/src/main/java/org/apache/pirk/responder/wideskies/ResponderCLI.java index e60c262..1a2fd32 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/ResponderCLI.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/ResponderCLI.java @@ -25,7 +25,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.pirk.inputformat.hadoop.InputFormatConst; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; @@ -356,7 +356,7 @@ public class ResponderCLI // Load the new local query and data schemas try { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); LoadQuerySchemas.initialize(); } catch (Exception e) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java b/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java index edba66a..fe8e4aa 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java @@ -26,9 +26,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.concurrent.ExecutionException; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.pirk.encryption.ModPowAbstraction; @@ -36,9 +33,14 @@ import org.apache.pirk.inputformat.hadoop.BytesArrayWritable; import org.apache.pirk.query.wideskies.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; import scala.Tuple3; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + /** * Class to compute the encrypted row elements for a query from extracted data partitions * http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/common/HashSelectorAndPartitionData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/common/HashSelectorAndPartitionData.java b/src/main/java/org/apache/pirk/responder/wideskies/common/HashSelectorAndPartitionData.java index b429377..7a652df 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/common/HashSelectorAndPartitionData.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/common/HashSelectorAndPartitionData.java @@ -31,6 +31,7 @@ import org.apache.pirk.utils.KeyedHash; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java index 0d04eab..f124bc2 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java @@ -46,7 +46,7 @@ import org.apache.pirk.inputformat.hadoop.BytesArrayWritable; import org.apache.pirk.inputformat.hadoop.InputFormatConst; import org.apache.pirk.query.wideskies.Query; import org.apache.pirk.query.wideskies.QueryInfo; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.schema.query.QuerySchema; import org.apache.pirk.serialization.HadoopFileSystemStore; @@ -125,7 +125,7 @@ public class ComputeResponseTool extends Configured implements Tool fs = FileSystem.get(conf); // Load the schemas - LoadDataSchemas.initialize(true, fs); + DataSchemaLoader.initialize(true, fs); LoadQuerySchemas.initialize(true, fs); query = new HadoopFileSystemStore(fs).recall(queryInputDir, Query.class); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java index a244a40..73bc4cd 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java @@ -31,7 +31,8 @@ import org.apache.pirk.query.wideskies.Query; import org.apache.pirk.query.wideskies.QueryInfo; import org.apache.pirk.responder.wideskies.common.HashSelectorAndPartitionData; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.schema.query.QuerySchema; import org.apache.pirk.schema.query.filter.DataFilter; @@ -40,6 +41,7 @@ import org.apache.pirk.utils.StringUtils; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** @@ -85,7 +87,7 @@ public class HashSelectorsAndPartitionDataMapper extends Mapper QueryInfo queryInfo = bbVarsIn.getQueryInfo(); QuerySchema qSchema = LoadQuerySchemas.getSchema(queryInfo.getQueryType()); - dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); filter = qSchema.getFilterInstance(); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java index 90fef67..087031e 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java @@ -29,6 +29,7 @@ import org.apache.pirk.schema.query.QuerySchema; import org.apache.spark.api.java.function.PairFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/data/DataSchema.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/DataSchema.java b/src/main/java/org/apache/pirk/schema/data/DataSchema.java index e0512bb..d363a0c 100644 --- a/src/main/java/org/apache/pirk/schema/data/DataSchema.java +++ b/src/main/java/org/apache/pirk/schema/data/DataSchema.java @@ -21,195 +21,261 @@ package org.apache.pirk.schema.data; import java.io.Serializable; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.io.Text; import org.apache.pirk.schema.data.partitioner.DataPartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.pirk.utils.PIRException; /** - * Class to hold a data schema + * A data schema describes the target data being referenced by a Querier and a Responder. + *

    + * The schema comprises a number of elements, each of which has a name, type, and a partitioner. Elements may be declared as arrays of types. + *

    + * Schemas are typically loaded from XML descriptors. + * + * @see DataSchemaLoader */ public class DataSchema implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger logger = LoggerFactory.getLogger(DataSchema.class); + // This schema's name. + private final String schemaName; - private String schemaName = null; + // Maps element name -> Java type name + private final Map typeMap = new HashMap<>(); - private String primitiveTypePartitionerName = null; + // Maps element name -> partitioner class name. + private final Map partitionerTypeMap = new HashMap<>(); - private transient HashMap textRep = null; // string element name -> Text representation + // Element names that are declared as array types. + private final Set arrayElements = new HashSet<>(); - private transient HashMap partitionerInstances = null; // partitioner class name -> Text representation + // Lazily maps partitioner class name -> an instance of the partitioner. + private transient Map partitionerInstances = new HashMap<>(); - private HashMap typeMap = null; // string element name -> java type + // Lazily maps element name -> Hadoop Text representation. + private transient Map textRep = new HashMap<>(); - private HashMap partitionerMap = null; // string element name -> partitioner class name - - private HashSet listRep = null; // elements that are list/array types - - public DataSchema(String schemaNameInput, HashMap textRepInput, HashSet listRepInput, HashMap typeMapInput, - HashMap partitionerMapInput) + /* + * Creates an empty, named data schema. + */ + DataSchema(String schemaName) { - schemaName = schemaNameInput; - textRep = textRepInput; - listRep = listRepInput; - typeMap = typeMapInput; - partitionerMap = partitionerMapInput; - primitiveTypePartitionerName = PrimitiveTypePartitioner.class.getName(); + this.schemaName = schemaName; } - public String getSchemaName() + /** + * Returns true if the data schema contains an element with the given name. + * + * @param elementName + * The element name to check. + * @return true if the schema does define an element with that name, of false otherwise. + */ + public boolean containsElement(String elementName) { - return schemaName; + return typeMap.containsKey(elementName); } - public HashMap getTextRep() + /** + * Returns the set of element names defined by this schema. + * + * @return The possibly empty set of element names. + */ + public Set getElementNames() { - if (textRep == null) - { - constructTextRep(); - } - return textRep; + return typeMap.keySet(); } - private void constructTextRep() + /** + * Returns the name of the Java type associated with the given element name. + *

    + * The Java type is either a primitive type name, as defined in the {@link PrimitiveTypePartitioner}, or a full canonical class name representing the element + * type. + * + * @see PrimitiveTypePartitioner + * @param elementName + * The element name whose type is requested. + * @return The type of the element, or null if the schema does not define the given element name. + */ + public String getElementType(String elementName) { - textRep = new HashMap<>(); - for (String name : typeMap.keySet()) - { - textRep.put(name, new Text(name)); - } + return typeMap.get(elementName); } /** - * Method to get the partitionerInstances HashMap of partitionerName -> partitionerInstance - *

    - * Will create it if it doesn't already exist + * Returns the element names that are declared as arrays. + * + * @return The set of names that are arrays, or an empty set if none. */ - public HashMap getPartitionerInstances() throws Exception + public Set getArrayElements() { - if (partitionerInstances == null) - { - constructPartitionerInstances(); - } - return partitionerInstances; + return arrayElements; } - private void constructPartitionerInstances() throws Exception + /** + * Returns the element names that are declared to not be arrays. + * + * @return The set of names that are not arrays, or an empty set if none. + */ + public Set getNonArrayElements() { - partitionerInstances = new HashMap<>(); - for (String partitionerName : partitionerMap.values()) - { - if (!partitionerInstances.containsKey(partitionerName)) - { - if (partitionerName.equals(primitiveTypePartitionerName)) - { - partitionerInstances.put(primitiveTypePartitionerName, new PrimitiveTypePartitioner()); - } - else - // If we have a non-primitive partitioner - { - Class c = Class.forName(partitionerName); - Object obj = c.newInstance(); - - // Interface check again just in case the class is used independently of the LoadDataSchemas load functionality - if (!(obj instanceof DataPartitioner)) - { - throw new Exception("partitionerName = " + partitionerName + " DOES NOT implement the DataPartitioner interface"); - } - partitionerInstances.put(partitionerName, obj); - } - } - } + Set elements = new HashSet<>(); + elements.addAll(typeMap.keySet()); + elements.removeAll(getArrayElements()); + return elements; + } /** - * Method to set the partitionerInstances HashMap of partitionerName -> partitionerInstance + * Returns the partitioner instance for the given element name. + *

    + * A partitioner for the named type is created on first request, and the same partitioner is returned on subsequent calls. + * + * @param elementName + * the name of the element whose partitioner is required. + * @return the data partitioner, or null if the element does not exist. + * @throws PIRExcpetion + * if the partitioner cannot be instantiated. + * @see DataSchema#getPartitionerInstance(String) */ - public void setPartitionerInstances(HashMap partitionerInstancesInput) + public DataPartitioner getPartitionerForElement(String elementName) throws PIRException { - partitionerInstances = partitionerInstancesInput; + String partitionerType = partitionerTypeMap.get(elementName); + return partitionerType == null ? null : getPartitionerInstance(partitionerType); } /** - * Method to get the partitioner class instance corresponding to the given partitioner class name + * Returns the partitioner corresponding to the given partitioner class name. *

    - * Will construct the partitionerInstances HashMap if it doesn't exist + * A partitioner for the named type is created on first request, and the same partitioner is returned on subsequent calls to this method. + * + * @param partitionerTypeName + * The class name for a partitioner type. + * @return The partitioner instance of the requested type. + * @throws PIRException + * If a problem occurs instantiating a new partitioner of the requested type. */ - public Object getPartitionerInstance(String partitionerName) throws Exception + public DataPartitioner getPartitionerInstance(String partitionerTypeName) throws PIRException { - if (partitionerInstances == null) + DataPartitioner partitioner = partitionerInstances.get(partitionerTypeName); + if (partitioner == null) { - constructPartitionerInstances(); + boolean isPrimitivePartitioner = partitionerTypeName.equals(PrimitiveTypePartitioner.class.getName()); + partitioner = isPrimitivePartitioner ? new PrimitiveTypePartitioner() : instantiatePartitioner(partitionerTypeName); + partitionerInstances.put(partitionerTypeName, partitioner); } - return partitionerInstances.get(partitionerName); + return partitioner; } /** - * Method to get the partitioner instance given an element name + * Returns the partitioner type name for a given element name. *

    - * Will construct the partitionerInstances HashMap if it doesn't exist + * The partitioner type name is either that of the primitive partitioner, where the element name is a primitive type. For non-primitives it is the fully + * qualified name of a Java class that implements the {@link DataPartitioner} interface. + * + * @param elementName + * The element name whose partitioner type is requested. + * @return The type name of the element's partitioner, or null if there is no element of that name. */ - public Object getPartitionerForElement(String element) throws Exception + public String getPartitionerTypeName(String elementName) { - return getPartitionerInstance(partitionerMap.get(element)); + return partitionerTypeMap.get(elementName); } /** - * Method to get the partitioner class name given an element name + * Returns the name of this schema. + * + * @return The schema name. */ - public String getPartitionerName(String element) + public String getSchemaName() { - return partitionerMap.get(element); + return schemaName; } /** - * Get the representation of a given element name + * Returns the Hadoop text representation of a given element name. + * + * @param elementName + * The name of the element whose text representation is requested. + * @returns The text representation, or null if the element name does not exist in this schema. */ - public Text getTextElement(String element) + public Text getTextName(String elementName) { - if (textRep == null) + Text text = textRep.get(elementName); + if (text == null && containsElement(elementName)) { - constructTextRep(); + text = new Text(elementName); + textRep.put(elementName, text); } - - return textRep.get(element); + return text; } - public HashMap getTypeMap() + /** + * Returns true if the given element name is an array type. + *

    + * The method returns false if the element is not an array type or the schema does not define an element of this type. + * + * @param element + * The name of the element to test. + * @return true if the element is an array type, and false otherwise. + */ + public boolean isArrayElement(String element) { - return typeMap; + return arrayElements.contains(element); } - public String getElementType(String element) + /* + * Returns the map of partitionerTypeName -> partitionerInstance + */ + Map getPartitionerInstances() { - return typeMap.get(element); + return partitionerInstances; } - public boolean containsElement(String element) + /* + * Returns the mapping from element name to partitioner type name. + */ + Map getPartitionerTypeMap() { - return textRep.keySet().contains(element); + return partitionerTypeMap; } - public HashSet getListRep() + /* + * Returns the Hadoop text map. + */ + Map getTextRep() { - return listRep; + return textRep; } - public HashSet getNonListRep() + /* + * Returns the mapping from element name to element's Java type. + */ + Map getTypeMap() { - HashSet elements = new HashSet<>(); - elements.addAll(textRep.keySet()); - elements.removeAll(listRep); - return elements; + return typeMap; } - public boolean hasListRep(String element) + /* + * Creates a new instance of the partitioner with the given type name, or throws a PIRExcpetion describing the problem. + */ + DataPartitioner instantiatePartitioner(String partitionerTypeName) throws PIRException { - return listRep.contains(element); + Object obj; + try + { + @SuppressWarnings("unchecked") + Class c = (Class) Class.forName(partitionerTypeName); + obj = c.newInstance(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | ClassCastException e) + { + throw new PIRException("partitioner = " + partitionerTypeName + " cannot be instantiated or does not implement DataParitioner.", e); + } + + return (DataPartitioner) obj; } }