pirk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smar...@apache.org
Subject [3/6] incubator-pirk git commit: Move all tests under src/test directory, this closes apache/incubator-pirk#66
Date Thu, 18 Aug 2016 15:31:54 GMT
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
new file mode 100644
index 0000000..58f835c
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.distributed.testsuite;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.inputformat.hadoop.InputFormatConst;
+import org.apache.pirk.inputformat.hadoop.json.JSONInputFormatBase;
+import org.apache.pirk.querier.wideskies.Querier;
+import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
+import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery;
+import org.apache.pirk.query.wideskies.Query;
+import org.apache.pirk.query.wideskies.QueryInfo;
+import org.apache.pirk.responder.wideskies.ResponderProps;
+import org.apache.pirk.responder.wideskies.mapreduce.ComputeResponseTool;
+import org.apache.pirk.response.wideskies.Response;
+import org.apache.pirk.schema.response.QueryResponseJSON;
+import org.apache.pirk.serialization.HadoopFileSystemStore;
+import org.apache.pirk.test.distributed.DistributedTestDriver;
+import org.apache.pirk.test.utils.BaseTests;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.apache.spark.launcher.SparkLauncher;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Distributed test class for PIR
+ * 
+ */
+public class DistTestSuite
+{
+  private static final Logger logger = LoggerFactory.getLogger(DistTestSuite.class);
+
+  // This method also tests all non-query specific configuration options/properties
+  // for the MapReduce version of PIR
+  public static void testJSONInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
+  {
+    logger.info("Starting testJSONInputMR");
+
+    // Pull original data and query schema properties
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "100");
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+    // Set up base configs
+    SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
+    SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
+    SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+    // Run tests
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+    BaseTests.testSRCIPQueryNoFilter(dataElements, fs, false, true, 2);
+
+    // Test hit limits per selector
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 3);
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+    // Test the local cache for modular exponentiation
+    SystemConfiguration.setProperty("pir.useLocalCache", "true");
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+    BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
+    SystemConfiguration.setProperty("pir.useLocalCache", "false");
+
+    // Change query for NXDOMAIN
+    SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
+    SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+    // Test the expTable cases
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+
+    // In memory table
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "true");
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+    // Create exp table in hdfs
+    SystemConfiguration.setProperty("mapreduce.map.memory.mb", "10000");
+    SystemConfiguration.setProperty("mapreduce.reduce.memory.mb", "10000");
+    SystemConfiguration.setProperty("mapreduce.map.java.opts", "-Xmx9000m");
+    SystemConfiguration.setProperty("mapreduce.reduce.java.opts", "-Xmx9000m");
+
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+    SystemConfiguration.setProperty("pir.expCreationSplits", "50");
+    SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+    // Reset exp properties
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+    // Reset property
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+
+    // Test embedded QuerySchema
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+    logger.info("Completed testJSONInputMR");
+  }
+
+  public static void testESInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
+  {
+    logger.info("Starting testESInputMR");
+
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+    // Set up ES configs
+    SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
+    SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
+    SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
+
+    // Run tests
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+    BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
+    BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+    // Change query for NXDOMAIN
+    SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
+
+    logger.info("Completed testESInputMR");
+  }
+
+  public static void testJSONInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
+  {
+    logger.info("Starting testJSONInputSpark");
+
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+    SystemConfiguration.setProperty("pir.useModExpJoin", "false");
+
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+    SystemConfiguration.setProperty("pir.numColMultPartitions", "20");
+    SystemConfiguration.setProperty("pir.colMultReduceByKey", "false");
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+    // Set up JSON configs
+    SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
+    SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
+    SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+    // Run tests
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+    BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
+
+    BaseTests.testSRCIPQueryNoFilter(dataElements, fs, true, true, 2);
+
+    // Test embedded QuerySchema
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+    // Test pad columns
+    SystemConfiguration.setProperty("pir.padEmptyColumns", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+    SystemConfiguration.setProperty("pir.padEmptyColumns", "false");
+
+    // Test hit limits per selector
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 3);
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+    // Test the local cache for modular exponentiation
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    SystemConfiguration.setProperty("pir.useLocalCache", "true");
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
+
+    // Test the join functionality for the modular exponentiation table
+    SystemConfiguration.setProperty("pir.useModExpJoin", "true");
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
+    SystemConfiguration.setProperty("pir.useModExpJoin", "false");
+
+    // Test file based exp lookup table for modular exponentiation
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
+    SystemConfiguration.setProperty("pir.expCreationSplits", "500");
+    SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+
+    // Change query for NXDOMAIN
+    SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+    // Test with reduceByKey for column mult
+    SystemConfiguration.setProperty("pir.colMultReduceByKey", "true");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+    logger.info("Completed testJSONInputSpark");
+  }
+
+  public static void testESInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
+  {
+    logger.info("Starting testESInputSpark");
+
+    SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+    SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+    SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+    SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+    SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+    SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+    // Set up ES configs
+    SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
+    SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
+    SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
+
+    // Run tests
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
+    BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
+    BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+
+    // Change query for NXDOMAIN
+    SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
+
+    SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+    SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+    BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+    logger.info("Completed testESInputSpark");
+  }
+
+  // Base method to perform query
+  public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads)
+      throws Exception
+  {
+    logger.info("performQuery: ");
+
+    String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR);
+    String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY);
+    fs.delete(new Path(outputFile), true); // Ensure old output does not exist.
+
+    SystemConfiguration.setProperty("pir.queryInput", queryInputDir);
+    SystemConfiguration.setProperty("pir.outputFile", outputFile);
+    SystemConfiguration.setProperty("pir.numReduceTasks", "1");
+    SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE));
+
+    // Create the temp result file
+    File fileFinalResults = File.createTempFile("finalResultsFile", ".txt");
+    fileFinalResults.deleteOnExit();
+    logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath());
+
+    boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
+    boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
+    boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
+
+    // Set the necessary objects
+    QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
+        queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
+
+    Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
+
+    // Perform the encryption
+    logger.info("Performing encryption of the selectors - forming encrypted query vectors:");
+    EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier);
+    encryptQuery.encrypt(numThreads);
+    logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:");
+
+    // Grab the necessary objects
+    Querier querier = encryptQuery.getQuerier();
+    Query query = encryptQuery.getQuery();
+
+    // Write the Querier object to a file
+    Path queryInputDirPath = new Path(queryInputDir);
+    new HadoopFileSystemStore(fs).store(queryInputDirPath, query);
+    fs.deleteOnExit(queryInputDirPath);
+
+    // Grab the original data and query schema properties to reset upon completion
+    String dataSchemaProp = SystemConfiguration.getProperty("data.schemas");
+    String querySchemaProp = SystemConfiguration.getProperty("query.schemas");
+
+    // Get the correct input format class name
+    JSONInputFormatBase jFormat = new JSONInputFormatBase();
+    String jsonBaseInputFormatString = jFormat.getClass().getName();
+    SystemConfiguration.setProperty("pir.baseInputFormat", jsonBaseInputFormatString);
+
+    // Submitting the tool for encrypted query
+    logger.info("Performing encrypted query:");
+    if (isSpark)
+    {
+      // Build args
+      String inputFormat = SystemConfiguration.getProperty("pir.dataInputFormat");
+      logger.info("inputFormat = " + inputFormat);
+      ArrayList<String> args = new ArrayList<>();
+      args.add("-" + ResponderProps.PLATFORM + "=spark");
+      args.add("-" + ResponderProps.DATAINPUTFORMAT + "=" + inputFormat);
+      args.add("-" + ResponderProps.QUERYINPUT + "=" + SystemConfiguration.getProperty("pir.queryInput"));
+      args.add("-" + ResponderProps.OUTPUTFILE + "=" + SystemConfiguration.getProperty("pir.outputFile"));
+      args.add("-" + ResponderProps.STOPLISTFILE + "=" + SystemConfiguration.getProperty("pir.stopListFile"));
+      args.add("-" + ResponderProps.USELOCALCACHE + "=" + SystemConfiguration.getProperty("pir.useLocalCache", "true"));
+      args.add("-" + ResponderProps.LIMITHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.limitHitsPerSelector", "false"));
+      args.add("-" + ResponderProps.MAXHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.maxHitsPerSelector", "1000"));
+      args.add("-" + ResponderProps.QUERYSCHEMAS + "=" + Inputs.HDFS_QUERY_FILES);
+      args.add("-" + ResponderProps.DATASCHEMAS + "=" + Inputs.DATA_SCHEMA_FILE_HDFS);
+      args.add("-" + ResponderProps.NUMEXPLOOKUPPARTS + "=" + SystemConfiguration.getProperty("pir.numExpLookupPartitions", "100"));
+      args.add("-" + ResponderProps.USEMODEXPJOIN + "=" + SystemConfiguration.getProperty("pir.useModExpJoin", "false"));
+      args.add("-" + ResponderProps.NUMCOLMULTPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numColMultPartitions", "20"));
+      args.add("-" + ResponderProps.COLMULTREDUCEBYKEY + "=" + SystemConfiguration.getProperty("pir.colMultReduceByKey", "false"));
+      if (inputFormat.equals(InputFormatConst.BASE_FORMAT))
+      {
+        args.add("-" + ResponderProps.INPUTDATA + "=" + SystemConfiguration.getProperty("pir.inputData"));
+        args.add("-" + ResponderProps.BASEQUERY + "=" + SystemConfiguration.getProperty("pir.baseQuery"));
+        args.add("-" + ResponderProps.BASEINPUTFORMAT + "=" + SystemConfiguration.getProperty("pir.baseInputFormat"));
+      }
+      else if (inputFormat.equals(InputFormatConst.ES))
+      {
+        args.add("-" + ResponderProps.ESQUERY + "=" + SystemConfiguration.getProperty("pir.esQuery"));
+        args.add("-" + ResponderProps.ESRESOURCE + "=" + SystemConfiguration.getProperty("pir.esResource"));
+      }
+
+      for (String arg : args)
+      {
+        logger.info("arg = " + arg);
+      }
+
+      // Run spark application
+      Process sLauncher = new SparkLauncher().setAppResource(SystemConfiguration.getProperty("jarFile"))
+          .setSparkHome(SystemConfiguration.getProperty("spark.home")).setMainClass("org.apache.pirk.responder.wideskies.ResponderDriver")
+          .addAppArgs(args.toArray(new String[args.size()])).setMaster("yarn-cluster").setConf(SparkLauncher.EXECUTOR_MEMORY, "2g")
+          .setConf(SparkLauncher.DRIVER_MEMORY, "2g").setConf(SparkLauncher.EXECUTOR_CORES, "1").launch();
+      sLauncher.waitFor();
+    }
+    else
+    {
+      SystemConfiguration.setProperty("data.schemas", Inputs.DATA_SCHEMA_FILE_HDFS);
+      SystemConfiguration.setProperty("query.schemas", Inputs.HDFS_QUERY_FILES);
+
+      ComputeResponseTool responseTool = new ComputeResponseTool();
+      ToolRunner.run(responseTool, new String[] {});
+    }
+    logger.info("Completed encrypted query");
+
+    // Perform decryption
+    // Reconstruct the necessary objects from the files
+    logger.info("Performing decryption; writing final results file");
+    Response response = new HadoopFileSystemStore(fs).recall(outputFile, Response.class);
+
+    // Perform decryption and output the result file
+    DecryptResponse decryptResponse = new DecryptResponse(response, querier);
+    decryptResponse.decrypt(numThreads);
+    decryptResponse.writeResultFile(fileFinalResults);
+    logger.info("Completed performing decryption and writing final results file");
+
+    // Read in results
+    logger.info("Reading in and checking results");
+    List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
+
+    // Reset data and query schema properties
+    SystemConfiguration.setProperty("data.schemas", dataSchemaProp);
+    SystemConfiguration.setProperty("query.schemas", querySchemaProp);
+
+    // Clean up output dir in hdfs
+    fs.delete(new Path(outputFile), true);
+
+    return results;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java b/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
new file mode 100644
index 0000000..9821db3
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+
+import java.text.ParseException;
+
+import org.apache.pirk.utils.ISO8601DateParser;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class to test basic functionality of ISO8601DateParser class
+ */
+public class ISO8601DateParserTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(ISO8601DateParserTest.class);
+
+  @Test
+  public void testDateParsing() throws ParseException
+  {
+    logger.info("Starting testDateParsing: ");
+
+    String date = "2016-02-20T23:29:05.000Z";
+    long longDate = Long.parseLong("1456010945000"); // date in UTC
+
+    assertEquals(longDate, ISO8601DateParser.getLongDate(date));
+    assertEquals(date, ISO8601DateParser.fromLongDate(longDate));
+
+    logger.info("Successfully completed testDateParsing");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java b/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
new file mode 100644
index 0000000..e53cebf
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.pirk.utils.KeyedHash;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic functional tests for KeyedHash
+ * 
+ */
+public class KeyedHashTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class);
+
+  @Test
+  public void testKeyedHash()
+  {
+    logger.info("Starting testKeyedHash: ");
+
+    int hash1 = KeyedHash.hash("someKey", 12, "someInput");
+    logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+    int hash2 = KeyedHash.hash("someKey", 32, "someInput");
+    logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+    int hash3 = KeyedHash.hash("someKey", 34, "someInput");
+    logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+    assertEquals(hash2, hash3);
+    assertEquals(hash1, hash2 & 0xFFF);
+
+    logger.info("Successfully completed testKeyedHash");
+  }
+
+  @Test
+  public void testKeyedHashWithType()
+  {
+    testKeyedHashType("MD5");
+    testKeyedHashType("SHA-1");
+    testKeyedHashType("SHA-256");
+    testKeyedHashType("FAKE-HASH-TYPE");
+  }
+
+  private void testKeyedHashType(String type)
+  {
+    logger.info("Starting testKeyedHashType with type: " + type);
+
+    int hash1 = KeyedHash.hash("someKey", 12, "someInput", type);
+    logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+    int hash2 = KeyedHash.hash("someKey", 32, "someInput", type);
+    logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+    int hash3 = KeyedHash.hash("someKey", 34, "someInput", type);
+    logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+    assertEquals(hash2, hash3);
+    assertEquals(hash1, hash2 & 0xFFF);
+
+    logger.info("Successfully completed testKeyedHashType with type: " + type);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/PaillierTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/PaillierTest.java b/src/test/java/org/apache/pirk/test/general/PaillierTest.java
new file mode 100644
index 0000000..6aa1396
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/PaillierTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.math.BigInteger;
+import java.util.Random;
+
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic test functionality for Paillier library
+ * 
+ */
+public class PaillierTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(PaillierTest.class);
+
+  private static BigInteger p = null; // large prime
+  private static BigInteger q = null; // large prime
+  private static BigInteger N = null; // N=pq, RSA modulus
+  private static BigInteger NSquared = null; // N^2
+  private static BigInteger lambdaN = null; // lambda(N) = lcm(p-1,q-1)
+
+  private static int bitLength = 0; // bit length of the modulus N
+  private static int certainty = 64; // prob that new BigInteger values represents primes will exceed (1 - (1/2)^certainty)
+
+  private static BigInteger r1 = null; // random number in (Z/NZ)*
+  private static BigInteger r2 = null; // random number in (Z/NZ)*
+
+  private static BigInteger m1 = null; // message to encrypt
+  private static BigInteger m2 = null; // message to encrypt
+
+  @BeforeClass
+  public static void setup()
+  {
+    p = BigInteger.valueOf(7);
+    q = BigInteger.valueOf(17);
+    N = p.multiply(q);
+    NSquared = N.multiply(N);
+
+    lambdaN = BigInteger.valueOf(48);
+
+    r1 = BigInteger.valueOf(3);
+    r2 = BigInteger.valueOf(4);
+
+    m1 = BigInteger.valueOf(5);
+    m2 = BigInteger.valueOf(2);
+
+    bitLength = 201;// bitLength = 384;
+    certainty = 128;
+
+    logger.info("p = " + p.intValue() + " q = " + q.intValue() + " N = " + N.intValue() + " bitLength = " + N.bitLength() + " lambdaN = " + lambdaN + " m1 = "
+        + m1.intValue() + " m2 = " + m2.intValue() + " r1 = " + r1.intValue() + " r2 = " + r2.intValue());
+  }
+
+  @Test
+  public void testPIRExceptions()
+  {
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(2), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for p,q < 3");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(3), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for p < 3");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(3), BigInteger.valueOf(2), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for q < 3");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(7), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for p = q");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(8), BigInteger.valueOf(7), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for p not prime");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(10), 128);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for q not prime");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 128);
+      Paillier paillier = new Paillier(3072, systemPrimeCertainty - 10);
+      assertNotNull(paillier);
+      fail("Paillier constructor did not throw PIRException for certainty less than system default of " + systemPrimeCertainty);
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier pailler = new Paillier(p, q, bitLength);
+      BigInteger encM1 = pailler.encrypt(N);
+      assertNotNull(encM1);
+      fail("Paillier encryption did not throw PIRException for message m = N");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier pailler = new Paillier(p, q, bitLength);
+      BigInteger encM1 = pailler.encrypt(N.add(BigInteger.TEN));
+      assertNotNull(encM1);
+      fail("Paillier encryption did not throw PIRException for message m > N");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier pailler = new Paillier(bitLength, 128, bitLength);
+      assertNotNull(pailler);
+      fail("Paillier constructor did not throw PIRException for ensureBitSet = bitLength");
+    } catch (PIRException ignore)
+    {}
+
+    try
+    {
+      Paillier pailler = new Paillier(bitLength, 128, bitLength + 1);
+      assertNotNull(pailler);
+      fail("Paillier constructor did not throw PIRException for ensureBitSet > bitLength");
+    } catch (PIRException ignore)
+    {}
+  }
+
+  @Test
+  public void testPaillierGivenAllParameters() throws Exception
+  {
+    logger.info("Starting testPaillierGivenAllParameters: ");
+
+    Paillier pailler = new Paillier(p, q, bitLength);
+
+    assertEquals(pailler.getN(), N);
+    assertEquals(pailler.getLambdaN(), lambdaN);
+
+    // Check encryption
+    BigInteger encM1 = pailler.encrypt(m1, r1);
+    BigInteger encM2 = pailler.encrypt(m2, r2);
+    logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+    assertEquals(encM1, BigInteger.valueOf(14019));
+    assertEquals(encM2, BigInteger.valueOf(8836));
+
+    // Check decryption
+    BigInteger decM1 = pailler.decrypt(encM1);
+    BigInteger decM2 = pailler.decrypt(encM2);
+    logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+    assertEquals(decM1, m1);
+    assertEquals(decM2, m2);
+
+    // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+    BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(NSquared);
+    BigInteger encM1plusM2 = pailler.encrypt((m1.add(m2)).mod(N), r1.multiply(r2));
+    logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " encM1plusM2 = " + encM1plusM2.intValue());
+
+    assertEquals(encM1_times_encM2, BigInteger.valueOf(5617));
+    assertEquals(encM1plusM2, BigInteger.valueOf(5617));
+
+    logger.info("Successfully completed testPaillierGivenAllParameters: ");
+  }
+
+  @Test
+  public void testPaillierWithKeyGeneration() throws Exception
+  {
+    logger.info("Starting testPaillierWithKeyGeneration: ");
+
+    // Test with and without gmp optimization for modPow
+    SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "true");
+    SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+    SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+    testPaillerWithKeyGenerationGeneral();
+
+    SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "false");
+
+    SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+    SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+    testPaillerWithKeyGenerationGeneral();
+
+    SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+    SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+    testPaillerWithKeyGenerationGeneral();
+
+    SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
+    SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+    testPaillerWithKeyGenerationGeneral();
+
+    // Reset the properties
+    SystemConfiguration.initialize();
+
+    logger.info("Ending testPaillierWithKeyGeneration: ");
+  }
+
+  public void testPaillerWithKeyGenerationGeneral() throws Exception
+  {
+    // Test without requiring highest bit to be set
+    logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = false");
+    testPaillierWithKeyGenerationBitSetOption(-1);
+
+    // Test requiring highest bit to be set
+    logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = true");
+    testPaillierWithKeyGenerationBitSetOption(5);
+  }
+
+  public void testPaillierWithKeyGenerationBitSetOption(int ensureBitSet) throws Exception
+  {
+    Random r = new Random();
+    int lowBitLength = 3073; // inclusive
+    int highBitLength = 7001; // exclusive
+
+    int loopVal = 1; // int loopVal = 1000; //change this and re-test for high loop testing
+    for (int i = 0; i < loopVal; ++i)
+    {
+      logger.info("i = " + i);
+
+      basicTestPaillierWithKeyGeneration(bitLength, certainty, ensureBitSet);
+      basicTestPaillierWithKeyGeneration(3072, certainty, ensureBitSet);
+
+      // Test with random bit length between 3073 and 7000
+      int randomLargeBitLength = r.nextInt(highBitLength - lowBitLength) + lowBitLength;
+      basicTestPaillierWithKeyGeneration(randomLargeBitLength, certainty, ensureBitSet);
+    }
+  }
+
+  private void basicTestPaillierWithKeyGeneration(int bitLengthInput, int certaintyInput, int ensureBitSet) throws Exception
+  {
+    Paillier pailler = new Paillier(bitLengthInput, certaintyInput, ensureBitSet);
+    BigInteger generatedN = pailler.getN();
+    BigInteger geneartedNsquared = generatedN.multiply(generatedN);
+
+    // Check the decrypting the encryption yields the message
+    BigInteger encM1 = pailler.encrypt(m1);
+    BigInteger encM2 = pailler.encrypt(m2);
+    logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+    BigInteger decM1 = pailler.decrypt(encM1);
+    BigInteger decM2 = pailler.decrypt(encM2);
+    logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+    assertEquals(decM1, m1);
+    assertEquals(decM2, m2);
+
+    // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+    BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(geneartedNsquared);
+    BigInteger multDecrypt = pailler.decrypt(encM1_times_encM2);
+    BigInteger m1_plus_m2 = (m1.add(m2)).mod(N);
+
+    logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " multDecrypt = " + multDecrypt.intValue() + " m1_plus_m2 = " + m1_plus_m2.intValue());
+
+    assertEquals(multDecrypt, m1_plus_m2);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
new file mode 100644
index 0000000..c1022ad
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Class to functionally test the bit conversion utils
+ */
+public class PartitionUtilsTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(PartitionUtilsTest.class);
+
+  @Test
+  public void testMask()
+  {
+    logger.info("Starting testMask: ");
+
+    assertEquals(0, PrimitiveTypePartitioner.formBitMask(0).intValue());
+
+    assertEquals(0b000000000000001, PrimitiveTypePartitioner.formBitMask(1).intValue());
+    assertEquals(0b000000000001111, PrimitiveTypePartitioner.formBitMask(4).intValue());
+    assertEquals(0b000000001111111, PrimitiveTypePartitioner.formBitMask(7).intValue());
+    assertEquals(0b111111111111111, PrimitiveTypePartitioner.formBitMask(15).intValue());
+    
+    assertEquals(new BigInteger("FFFFF", 16), PrimitiveTypePartitioner.formBitMask(20));
+    assertEquals(new BigInteger("FFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(32));
+    assertEquals(new BigInteger("3FFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(42));
+    assertEquals(new BigInteger("7FFFFFFFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(63));
+
+    logger.info("Successfully completed testMask");
+  }
+
+  @Test
+  public void testPartitionBits() throws PIRException
+  {
+    logger.info("Starting testPartitionBits: ");
+
+    BigInteger value = new BigInteger("245"); // 11110101
+    BigInteger value2 = new BigInteger("983"); // 1111010111
+
+    BigInteger mask4 = PrimitiveTypePartitioner.formBitMask(4); // 1111
+    BigInteger mask8 = PrimitiveTypePartitioner.formBitMask(8); // 11111111
+
+    List<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask4);
+    assertEquals(2, partitions.size());
+    assertEquals(0b1111, partitions.get(0).intValue());
+    assertEquals(0b0101, partitions.get(1).intValue());
+
+    partitions = PrimitiveTypePartitioner.partitionBits(value2, 4, mask4);
+    assertEquals(3, partitions.size());
+    assertEquals(0b1111, partitions.get(0).intValue()); 
+    assertEquals(0b0101, partitions.get(1).intValue());
+    assertEquals(0b0011, partitions.get(2).intValue());
+
+    partitions = PrimitiveTypePartitioner.partitionBits(value, 8, mask8);
+    assertEquals(1, partitions.size());
+    assertEquals(0b11110101, partitions.get(0).intValue());
+
+    try
+    {
+      partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask8);
+      fail("BitConversionUtils.partitionBits did not throw error for mismatched partitionSize and mask size");
+    } catch (Exception ignore)
+    {
+      // Expected.
+    }
+
+    logger.info("Successfully completed testPartitionBits");
+  }
+
+  @Test
+  public void testPartitions() throws Exception
+  {
+    logger.info("Starting testToPartitions:");
+
+    PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
+    IPDataPartitioner ipPartitioner = new IPDataPartitioner();
+    ISO8601DatePartitioner datePartitioner = new ISO8601DatePartitioner();
+
+    // Test IP
+    String ipTest = "127.0.0.1";
+    List<BigInteger> partsIP = ipPartitioner.toPartitions(ipTest, PrimitiveTypePartitioner.STRING);
+    assertEquals(4, partsIP.size());
+    assertEquals(ipTest, ipPartitioner.fromPartitions(partsIP, 0, PrimitiveTypePartitioner.STRING));
+
+    // Test Date
+    String dateTest = "2016-02-20T23:29:05.000Z";
+    List<BigInteger> partsDate = datePartitioner.toPartitions(dateTest, null);
+    assertEquals(8, partsDate.size());
+    assertEquals(dateTest, datePartitioner.fromPartitions(partsDate, 0, null));
+
+    // Test byte
+    byte bTest = Byte.parseByte("10");
+    List<BigInteger> partsByte = primitivePartitioner.toPartitions(bTest, PrimitiveTypePartitioner.BYTE);
+    assertEquals(1, partsByte.size());
+    assertEquals(bTest, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
+
+    partsByte = primitivePartitioner.toPartitions("12", PrimitiveTypePartitioner.BYTE);
+    assertEquals(1, partsByte.size());
+    assertEquals((byte) 12, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
+
+    List<BigInteger> partsByteMax = primitivePartitioner.toPartitions(Byte.MAX_VALUE, PrimitiveTypePartitioner.BYTE);
+    assertEquals(1, partsByteMax.size());
+    assertEquals(Byte.MAX_VALUE, primitivePartitioner.fromPartitions(partsByteMax, 0, PrimitiveTypePartitioner.BYTE));
+
+    // Test string
+    String stringBits = SystemConfiguration.getProperty("pir.stringBits");
+    SystemConfiguration.setProperty("pir.stringBits", "64");
+    testString("testString"); // over the allowed bit size
+    testString("t"); // under the allowed bit size
+    SystemConfiguration.setProperty("pir.stringBits", stringBits);
+
+    // Test short
+    short shortTest = Short.valueOf("2456");
+    List<BigInteger> partsShort = primitivePartitioner.toPartitions(shortTest, PrimitiveTypePartitioner.SHORT);
+    assertEquals(2, partsShort.size());
+    assertEquals(shortTest, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+    partsShort = primitivePartitioner.toPartitions("32767", PrimitiveTypePartitioner.SHORT);
+    assertEquals(2, partsShort.size());
+    assertEquals((short) 32767, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+    partsShort = primitivePartitioner.toPartitions((short) -42, PrimitiveTypePartitioner.SHORT);
+    assertEquals(2, partsShort.size());
+    assertEquals((short) -42, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+    List<BigInteger> partsShortMax = primitivePartitioner.toPartitions(Short.MAX_VALUE, PrimitiveTypePartitioner.SHORT);
+    assertEquals(2, partsShortMax.size());
+    assertEquals(Short.MAX_VALUE, primitivePartitioner.fromPartitions(partsShortMax, 0, PrimitiveTypePartitioner.SHORT));
+
+    // Test int
+    int intTest = Integer.parseInt("-5789");
+    List<BigInteger> partsInt = primitivePartitioner.toPartitions(intTest, PrimitiveTypePartitioner.INT);
+    assertEquals(4, partsInt.size());
+    assertEquals(intTest, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+    partsInt = primitivePartitioner.toPartitions("2016", PrimitiveTypePartitioner.INT);
+    assertEquals(4, partsInt.size());
+    assertEquals(2016, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+    partsInt = primitivePartitioner.toPartitions(1386681237, PrimitiveTypePartitioner.INT);
+    assertEquals(4, partsInt.size());
+    assertEquals(1386681237, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+    List<BigInteger> partsIntMax = primitivePartitioner.toPartitions(Integer.MAX_VALUE, PrimitiveTypePartitioner.INT);
+    assertEquals(4, partsIntMax.size());
+    assertEquals(Integer.MAX_VALUE, primitivePartitioner.fromPartitions(partsIntMax, 0, PrimitiveTypePartitioner.INT));
+
+    // Test long
+    long longTest = Long.parseLong("56789");
+    List<BigInteger> partsLong = primitivePartitioner.toPartitions(longTest, PrimitiveTypePartitioner.LONG);
+    assertEquals(8, partsLong.size());
+    assertEquals(longTest, primitivePartitioner.fromPartitions(partsLong, 0, PrimitiveTypePartitioner.LONG));
+
+    List<BigInteger> partsLongMax = primitivePartitioner.toPartitions(Long.MAX_VALUE, PrimitiveTypePartitioner.LONG);
+    assertEquals(8, partsLongMax.size());
+    assertEquals(Long.MAX_VALUE, primitivePartitioner.fromPartitions(partsLongMax, 0, PrimitiveTypePartitioner.LONG));
+
+    // Test float
+    float floatTest = Float.parseFloat("567.77");
+    List<BigInteger> partsFloat = primitivePartitioner.toPartitions(floatTest, PrimitiveTypePartitioner.FLOAT);
+    assertEquals(4, partsFloat.size());
+    assertEquals(floatTest, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
+
+    partsFloat = primitivePartitioner.toPartitions(-99.99f, PrimitiveTypePartitioner.FLOAT);
+    assertEquals(4, partsFloat.size());
+    assertEquals(-99.99f, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
+
+    List<BigInteger> partsFloatMax = primitivePartitioner.toPartitions(Float.MAX_VALUE, PrimitiveTypePartitioner.FLOAT);
+    assertEquals(4, partsFloatMax.size());
+    assertEquals(Float.MAX_VALUE, primitivePartitioner.fromPartitions(partsFloatMax, 0, PrimitiveTypePartitioner.FLOAT));
+
+    // Test double
+    double doubleTest = Double.parseDouble("567.77");
+    List<BigInteger> partsDouble = primitivePartitioner.toPartitions(doubleTest, PrimitiveTypePartitioner.DOUBLE);
+    assertEquals(8, partsDouble.size());
+    assertEquals(doubleTest, primitivePartitioner.fromPartitions(partsDouble, 0, PrimitiveTypePartitioner.DOUBLE));
+
+    List<BigInteger> partsDoubleMax = primitivePartitioner.toPartitions(Double.MAX_VALUE, PrimitiveTypePartitioner.DOUBLE);
+    assertEquals(8, partsDoubleMax.size());
+    assertEquals(Double.MAX_VALUE, primitivePartitioner.fromPartitions(partsDoubleMax, 0, PrimitiveTypePartitioner.DOUBLE));
+
+    // Test char
+    char charTest = 'b';
+    List<BigInteger> partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+    assertEquals(2, partsChar.size());
+    assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+    // Ensure Endianness preserved
+    charTest = '\uFFFE';
+    partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+    assertEquals(2, partsChar.size());
+    assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+    charTest = '\uFEFF';
+    partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+    assertEquals(2, partsChar.size());
+    assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+    List<BigInteger> partsCharMax = primitivePartitioner.toPartitions(Character.MAX_VALUE, PrimitiveTypePartitioner.CHAR);
+    assertEquals(2, partsCharMax.size());
+    assertEquals(Character.MAX_VALUE, primitivePartitioner.fromPartitions(partsCharMax, 0, PrimitiveTypePartitioner.CHAR));
+
+    logger.info("Sucessfully completed testToPartitions:");
+  }
+
+  @Test
+  public void testPaddedPartitions() throws PIRException
+  {
+    PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
+
+    List<String> primitiveTypes = Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.SHORT,
+        PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE,
+        PrimitiveTypePartitioner.STRING);
+    for (String type : primitiveTypes)
+    {
+      assertEquals(primitivePartitioner.getNumPartitions(type), primitivePartitioner.getPaddedPartitions(type).size());
+    }
+  }
+
+  private void testString(String testString) throws Exception
+  {
+    PrimitiveTypePartitioner ptp = new PrimitiveTypePartitioner();
+
+    List<BigInteger> partsString = ptp.toPartitions(testString, PrimitiveTypePartitioner.STRING);
+    int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / 8;
+    assertEquals(numParts, partsString.size());
+
+    logger.info("testString.getBytes().length = " + testString.getBytes().length);
+    int offset = numParts;
+    if (testString.getBytes().length < numParts)
+    {
+      offset = testString.getBytes().length;
+    }
+    String element = new String(testString.getBytes(), 0, offset);
+    assertEquals(element, ptp.fromPartitions(partsString, 0, PrimitiveTypePartitioner.STRING));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java b/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
new file mode 100644
index 0000000..b3db140
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.pirk.schema.data.DataSchema;
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.QueryParserUtils;
+import org.apache.pirk.utils.StringUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class for testing the QueryParser methods
+ */
+public class QueryParserUtilsTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(QueryParserUtilsTest.class);
+
+  private static MapWritable doc = null; // MapWritable with arrays in json string representation
+  private static MapWritable docWAW = null; // MapWritable with arrays as WritableArrayWritable objects
+  private static Map<String,Object> docMap = null; // arrays as ArrayList<String>
+
+  private static DataSchema dSchema = null;
+
+  @BeforeClass
+  public static void setup() throws Exception
+  {
+    ArrayList<JSONObject> dataElementsJSON = Inputs.createJSONDataElements();
+
+    // Reset the schema properties and registries
+    DataSchemaRegistry.clearRegistry();
+    QuerySchemaRegistry.clearRegistry();
+    SystemConfiguration.setProperty("data.schemas", "none");
+    SystemConfiguration.setProperty("query.schemas", "none");
+
+    Inputs.createSchemaFiles(null, false, null);
+
+    dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME);
+
+    // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
+    // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
+    // + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
+    //
+    doc = StringUtils.jsonStringToMapWritableWithArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+    docWAW = StringUtils.jsonStringToMapWritableWithWritableArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+    docMap = StringUtils.jsonStringToMap(dataElementsJSON.get(0).toJSONString(), dSchema);
+  }
+
+  @AfterClass
+  public static void teardown()
+  {
+    // Reset the schema properties and registries
+    DataSchemaRegistry.clearRegistry();
+    QuerySchemaRegistry.clearRegistry();
+    SystemConfiguration.setProperty("data.schemas", "none");
+    SystemConfiguration.setProperty("query.schemas", "none");
+  }
+
+  @Test
+  public void testSingleQuery()
+  {
+    String query1 = "?q=src_ip:55.55.55.55";
+    assertTrue(QueryParserUtils.checkRecord(query1, doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query1, docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query1, docMap, dSchema));
+
+    String query2 = "?q=qname:a.b.c.com";
+    assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+    String query3 = "?q=qname:d.b.c.com";
+    assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+  }
+
+  @Test
+  public void testQueryFieldDoesNotExist()
+  {
+    logger.info("running testQueryFieldDoesNotExist");
+
+    // Field does not exist, this should not be found
+    String query = "?q=nonexistent-field:*check*";
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+    // First field does not exist, but second should be found
+    String query2 = "?q=nonexistent-field:*check*+OR+qname:*a.b.c.com*";
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+    // First field does not exist, second field does, but AND operator makes query false
+    String query3 = "?q=nonexistent-field:*check*+AND+qname:*a.b.c.com*";
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+    logger.info("completed testQueryFieldDoesNotExist");
+  }
+
+  @Test
+  public void testIgnoreCase()
+  {
+    logger.info("running testIgnoreCase");
+
+    // with case sensitivity, should NOT be found
+    String query = "?q=qname:*A.b.c.com*";
+    assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+    // with case sensitivity, should be found
+    String query2 = "?q=qname:*a.b.c.com*";
+    assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+    // adds @ flag = case insensitivity, thus should be found
+    String query3 = "?q=qname@:*A.b.c.com*";
+    assertTrue(QueryParserUtils.checkRecord(query3, doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+    logger.info("completed testIgnoreCase");
+  }
+
+  @Test
+  public void testSingleValueRangeQuery()
+  {
+    testSingleValueRangeQueryMapWritable();
+    testSingleValueRangeQueryMap();
+    testSingleValueRangeQueryMapWritableWAW();
+  }
+
+  private void testSingleValueRangeQueryMapWritable()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", doc, dSchema));
+  }
+
+  private void testSingleValueRangeQueryMap()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", docMap, dSchema));
+  }
+
+  private void testSingleValueRangeQueryMapWritableWAW()
+  {
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[0+TO+2]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{-1+TO+2}", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[-1+TO+0]", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{0+TO+3}", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[3+TO+10]", docWAW, dSchema));
+  }
+
+  @Test
+  public void testIPRangeQuery()
+  {
+    testIPRangeQueryMapWritable();
+    testIPRangeQueryMap();
+    testIPRangeQueryMapWritableWAW();
+  }
+
+  public void testIPRangeQueryMapWritable()
+  {
+    // src_ip: 55.55.55.55
+    // ip: 10.20.30.40,10.20.30.60
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", doc, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", doc, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", doc, dSchema));
+  }
+
+  public void testIPRangeQueryMapWritableWAW()
+  {
+    // src_ip: 55.55.55.55
+    // ip: 10.20.30.40,10.20.30.60
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docWAW, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docWAW, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=ip:[10.20.30.50+TO+10.20.30.69]", docWAW, dSchema));
+  }
+
+  public void testIPRangeQueryMap()
+  {
+    // src_ip: 55.55.55.55
+    // ip: 10.20.30.40,10.20.30.60
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docMap, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docMap, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", docMap, dSchema));
+  }
+
+  @Test
+  public void testDateRangeQuery()
+  {
+    testDateRangeQueryMapWritable();
+    testDateRangeQueryMapWritableWAW();
+    testDateRangeQueryMap();
+  }
+
+  private void testDateRangeQueryMapWritable()
+  {
+    // date: 2016-02-20T23:29:05.000Z
+
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", doc, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", doc, dSchema));
+  }
+
+  private void testDateRangeQueryMap()
+  {
+    // date: 2016-02-20T23:29:05.000Z
+
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docMap, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docMap, dSchema));
+  }
+
+  private void testDateRangeQueryMapWritableWAW()
+  {
+    // date: 2016-02-20T23:29:05.000Z
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docWAW, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docWAW, dSchema));
+  }
+
+  @Test
+  public void testBooleanQuery()
+  {
+    testBooleanQueryMapWritable();
+    testBooleanQueryMapMapWritableWAW();
+    testBooleanQueryMap();
+  }
+
+  private void testBooleanQueryMapWritable()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", doc, dSchema));
+  }
+
+  private void testBooleanQueryMap()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", docMap, dSchema));
+  }
+
+  private void testBooleanQueryMapMapWritableWAW()
+  {
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+        docWAW, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]",
+        docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+        docWAW, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+qtype:2+OR+rcode:0", docWAW, dSchema));
+  }
+
+  @Test
+  public void testAllQuery()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=*", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=*", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=*", docWAW, dSchema));
+  }
+
+  @Test
+  public void testWildcardQuery()
+  {
+    testWildcardQueryMapWritable();
+    testWildcardQueryMap();
+    testWildcardQueryMapWritableWAW();
+  }
+
+  private void testWildcardQueryMapWritable()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", doc, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", doc, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", doc, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", doc, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", doc, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", doc, dSchema));
+  }
+
+  private void testWildcardQueryMap()
+  {
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", docMap, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", docMap, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", docMap, dSchema));
+    assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", docMap, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", docMap, dSchema));
+    assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", docMap, dSchema));
+  }
+
+  private void testWildcardQueryMapWritableWAW()
+  {
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.com", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c*m", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b*", docWAW, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.org", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:mrtf*", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljnik*.uk", docWAW, dSchema));
+
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c?m", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.?.com", docWAW, dSchema));
+    assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?.b.c.com", docWAW, dSchema));
+
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:medelj?ikafera.com", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljntkafer?.com", docWAW, dSchema));
+    assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?edeljnikrfera.com", docWAW, dSchema));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java b/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
new file mode 100644
index 0000000..9f52af3
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.schema.data;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.pirk.schema.data.DataSchema;
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Test suite for LoadDataSchema and DataSchema
+ */
+public class LoadDataSchemaTest
+{
+  private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemaTest.class);
+
+  private String dataSchemaName = "fakeDataSchema";
+
+  private String element1 = "elementName1";
+  private String element2 = "elementName2";
+  private String element3 = "elementName3";
+
+  @Test
+  public void testGeneralSchemaLoad() throws Exception
+  {
+    // Pull off the property and reset upon completion
+    String schemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+
+    // Write the schema file
+    try
+    {
+      createDataSchema("schemaFile");
+    } catch (IOException e)
+    {
+      e.printStackTrace();
+      fail(e.toString());
+    }
+
+    // Force the schema to load
+    DataSchemaLoader.initialize();
+
+    // Check the entries
+    DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName);
+
+    assertEquals(dataSchemaName, dSchema.getSchemaName());
+
+    assertEquals(3, dSchema.getElementNames().size());
+
+    // TODO: check Hadoop text names
+
+    assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1));
+    assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2));
+    assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3));
+
+    assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1));
+    if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner))
+    {
+      fail("Partitioner instance for element1 must be PrimitiveTypePartitioner");
+    }
+    assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3));
+    if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner))
+    {
+      fail("Partitioner instance for element3 must be IPDataPartitioner");
+    }
+
+    assertEquals(2, dSchema.getArrayElements().size());
+    assertTrue(dSchema.getArrayElements().contains(element2));
+    assertTrue(dSchema.getArrayElements().contains(element3));
+
+    assertEquals(1, dSchema.getNonArrayElements().size());
+    assertTrue(dSchema.getNonArrayElements().contains(element1));
+
+    // Reset original data.schemas property
+    SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+    // Force the schema to load
+    if (!schemasProp.equals("none"))
+    {
+      DataSchemaLoader.initialize();
+    }
+  }
+
+  @Test
+  public void testIncorrectJavaType() throws Exception
+  {
+    // Pull off the property and reset upon completion
+    String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+    // Write the schema file
+    try
+    {
+      createDataSchemaIncorrectJavaType("wrongJavaType");
+    } catch (IOException e)
+    {
+      e.printStackTrace();
+      fail(e.toString());
+    }
+
+    try
+    {
+      // Force the schema to load
+      DataSchemaLoader.initialize();
+      fail("DataSchemaLoader did not throw exception for incorrect javaType");
+    } catch (Exception ignore)
+    {}
+
+    // Reset original data.schemas property
+    SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+    // Force the schema to load
+    DataSchemaLoader.initialize();
+  }
+
+  @Test
+  public void testUnknownPartitioner() throws Exception
+  {
+    // Pull off the property and reset upon completion
+    String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+    // Write the schema file
+    try
+    {
+      createDataSchemaUnknownPartitioner("unknownPartitioner");
+    } catch (IOException e)
+    {
+      e.printStackTrace();
+      fail(e.toString());
+    }
+
+    try
+    {
+      // Force the schema to load
+      DataSchemaLoader.initialize();
+      fail("DataSchemaLoader did not throw exception for unknown partitioner");
+    } catch (Exception ignore)
+    {}
+
+    // Reset original data.schemas property
+    SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+    // Force the schema to load
+    DataSchemaLoader.initialize();
+  }
+
+  // Create the file that contains an unknown partitioner
+  private void createDataSchemaUnknownPartitioner(String schemaFile) throws IOException
+  {
+    // Create a temporary file for the test schema, set in the properties
+    File file = File.createTempFile(schemaFile, ".xml");
+    file.deleteOnExit();
+    logger.info("file = " + file.toString());
+    SystemConfiguration.setProperty("data.schemas", file.toString());
+
+    // Write to the file
+    try
+    {
+      DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+      Document doc = dBuilder.newDocument();
+
+      // root element
+      Element rootElement = doc.createElement("schema");
+      doc.appendChild(rootElement);
+
+      // Add the schemaName
+      Element schemaNameElement = doc.createElement("schemaName");
+      schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+      rootElement.appendChild(schemaNameElement);
+
+      // Add the element - unknown partitioner
+      TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.INT, "false", "fakePartitioner");
+
+      // Write to a xml file
+      TransformerFactory transformerFactory = TransformerFactory.newInstance();
+      Transformer transformer = transformerFactory.newTransformer();
+      DOMSource source = new DOMSource(doc);
+      StreamResult result = new StreamResult(file);
+      transformer.transform(source, result);
+
+      // Output for testing
+      StreamResult consoleResult = new StreamResult(System.out);
+      transformer.transform(source, consoleResult);
+      System.out.println();
+    } catch (Exception e)
+    {
+      e.printStackTrace();
+    }
+  }
+
+  // Create the test data schema file
+  private void createDataSchema(String schemaFile) throws IOException
+  {
+    // Create a temporary file for the test schema, set in the properties
+    File file = File.createTempFile(schemaFile, ".xml");
+    file.deleteOnExit();
+    logger.info("file = " + file.toString());
+    SystemConfiguration.setProperty("data.schemas", file.toString());
+
+    // Write to the file
+    try
+    {
+      DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+      Document doc = dBuilder.newDocument();
+
+      // root element
+      Element rootElement = doc.createElement("schema");
+      doc.appendChild(rootElement);
+
+      // Add the schemaName
+      Element schemaNameElement = doc.createElement("schemaName");
+      schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+      rootElement.appendChild(schemaNameElement);
+
+      // Add the elements
+      // element1 -- single String
+      // TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
+      TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", null);
+
+      // element2 - -- array of Integers
+      TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
+
+      // element3 -- array of IP addresses
+      TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
+
+      // Write to a xml file
+      TransformerFactory transformerFactory = TransformerFactory.newInstance();
+      Transformer transformer = transformerFactory.newTransformer();
+      DOMSource source = new DOMSource(doc);
+      StreamResult result = new StreamResult(file);
+      transformer.transform(source, result);
+
+      // Output for testing
+      StreamResult consoleResult = new StreamResult(System.out);
+      transformer.transform(source, consoleResult);
+      System.out.println();
+
+    } catch (Exception e)
+    {
+      e.printStackTrace();
+    }
+  }
+
+  // Create the test schema file
+  private void createDataSchemaIncorrectJavaType(String schemaFile) throws IOException
+  {
+    // Create a temporary file for the test schema, set in the properties
+    File file = File.createTempFile(schemaFile, ".xml");
+    file.deleteOnExit();
+    logger.info("file = " + file.toString());
+    SystemConfiguration.setProperty("data.schemas", file.toString());
+
+    // Write to the file
+    try
+    {
+      DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+      Document doc = dBuilder.newDocument();
+
+      // root element
+      Element rootElement = doc.createElement("schema");
+      doc.appendChild(rootElement);
+
+      // Add the schemaName
+      Element schemaNameElement = doc.createElement("schemaName");
+      schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+      rootElement.appendChild(schemaNameElement);
+
+      // Add the element - unknown Java type
+      TestUtils.addElement(doc, rootElement, element1, "bogus", "false", PrimitiveTypePartitioner.class.getName());
+
+      // Write to a xml file
+      TransformerFactory transformerFactory = TransformerFactory.newInstance();
+      Transformer transformer = transformerFactory.newTransformer();
+      DOMSource source = new DOMSource(doc);
+      StreamResult result = new StreamResult(file);
+      transformer.transform(source, result);
+
+      // Output for testing
+      StreamResult consoleResult = new StreamResult(System.out);
+      transformer.transform(source, consoleResult);
+      System.out.println();
+
+    } catch (Exception e)
+    {
+      e.printStackTrace();
+    }
+  }
+}



Mime
View raw message