Mailing-List: contact commits-help@pirk.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@pirk.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: charris@apache.org
To: commits@pirk.incubator.apache.org
Date: Fri, 29 Jul 2016 02:19:46 -0000
Message-Id: <78e7180e255f4a8f9ae0ca07cfac399e@git.apache.org>
Subject: [1/2] incubator-pirk git commit: PIRK-27 Improve Pirk's Use of
 Property Files - closes apache/incubator-pirk#27
archived-at: Fri, 29 Jul 2016 02:19:53 -0000

Repository: incubator-pirk
Updated Branches:
  refs/heads/master de7a9c89d -> 1b38ea673


http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/1b38ea67/src/main/java/org/apache/pirk/utils/SystemConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/utils/SystemConfiguration.java b/src/main/java/org/apache/pirk/utils/SystemConfiguration.java
index 7bea235..b5ac98f 100755
--- a/src/main/java/org/apache/pirk/utils/SystemConfiguration.java
+++ b/src/main/java/org/apache/pirk/utils/SystemConfiguration.java
@@ -20,12 +20,11 @@ package org.apache.pirk.utils;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
 
-import org.apache.pirk.schema.data.DataSchemaLoader;
-import org.apache.pirk.schema.query.QuerySchemaLoader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,7 +33,9 @@ import org.slf4j.LoggerFactory;
  * <p>
  * 1) Load in the DEFAULT_PROPERTY_FILE, if found on the classpath. (Currently 'pirk.properties')
  * <p>
- * 2) Load in any properties from LOCAL_PROPERTY_FILE
+ * 2) Load in any properties files in the LOCAL_PROPERTIES_DIR. The filenames must end with '.properties'
+ * <p>
+ * 3) Load in properties from the QUERIER_PROPERTIES_FILE and RESPONDER_PROPERTIES_FILE
  * 
  */
 public class SystemConfiguration
@@ -44,78 +45,31 @@ public class SystemConfiguration
   private static final Properties props;
 
   /**
-   * By default, if the pirk.properties file is found on the root of the classpath, it is loaded first.
+   * By default, these files should be found on the root of the classpath
    */
   private static final String DEFAULT_PROPERTY_FILE = "pirk.properties";
+  private static final String QUERIER_PROPERTIES_FILE = "querier.properties";
+  private static final String RESPONDER_PROPERTIES_FILE = "responder.properties";
 
-  /**
-   * By default, if the local.pirk.properties file is found on the root of the classpath, it is loaded after pirk.properites.
-   */
-  private static final String LOCAL_PROPERTY_FILE = "local.pirk.properties";
+  private static final String LOCAL_PROPERTIES_DIR = "local.pirk.properties.dir";
 
   static
   {
     props = new Properties();
     initialize();
-
-    // Load any data schema files indicated in the properties
-    try
-    {
-      DataSchemaLoader.class.newInstance();
-    } catch (Exception e)
-    {
-      logger.error("Issue when invoking DataSchemaLoader");
-      e.printStackTrace();
-    }
-
-    // Load any query schema files indicated in the properties
-    try
-    {
-      QuerySchemaLoader.class.newInstance();
-    } catch (Exception e)
-    {
-      logger.error("Issue when invoking DataSchemaLoader");
-      e.printStackTrace();
-    }
   }
 
   public static void initialize()
   {
     // First try to load the default properties file
-    try
-    {
-      InputStream stream = SystemConfiguration.class.getClassLoader().getResourceAsStream(DEFAULT_PROPERTY_FILE);
-      if (stream != null)
-      {
-        logger.info("Loading default properties file '" + DEFAULT_PROPERTY_FILE + "'");
-        props.load(stream);
-        stream.close();
-      }
-      else
-      {
-        logger.error("No default configuration file found '" + DEFAULT_PROPERTY_FILE + "'");
-      }
-    } catch (IOException e)
-    {
-      logger.error("Problem loading default properties file '" + DEFAULT_PROPERTY_FILE + "'");
-      e.printStackTrace();
-    }
+    loadPropsFromStream(DEFAULT_PROPERTY_FILE);
 
-    // Try to load the local properties file, if one exists
-    File localFile = new File(getProperty(LOCAL_PROPERTY_FILE));
-    if (localFile.exists())
-    {
-      try (InputStream stream = new FileInputStream(localFile);)
-      {
-        logger.info("Loading local properties file '" + localFile.getAbsolutePath() + "'");
-        props.load(stream);
-        stream.close();
-      } catch (IOException e)
-      {
-        logger.error("Problem loading local properties file '" + localFile.getAbsolutePath() + "'");
-        e.printStackTrace();
-      }
-    }
+    // Try to load props from the querier and responder property files, if they exist
+    loadPropsFromStream(QUERIER_PROPERTIES_FILE);
+    loadPropsFromStream(RESPONDER_PROPERTIES_FILE);
+
+    // Try to load the local properties files, if they exists
+    loadPropsFromDir(LOCAL_PROPERTIES_DIR);
   }
 
   /**
@@ -152,6 +106,31 @@ public class SystemConfiguration
     props.setProperty(propertyName, value);
   }
 
+  public static boolean hasProperty(String propertyName)
+  {
+    return props.containsKey(propertyName);
+  }
+
+  /**
+   * Append a property via a comma separated list
+   * <p>
+   * If the property does not exist, it adds it
+   */
+  public static void appendProperty(String property, String propToAdd)
+  {
+    String value = props.getProperty(property);
+
+    if (value != null && !value.equals("none"))
+    {
+      value += "," + propToAdd;
+    }
+    else
+    {
+      value = propToAdd;
+    }
+    props.setProperty(property, value);
+  }
+
   /**
    * Reset all properties to the default values
    */
@@ -160,4 +139,78 @@ public class SystemConfiguration
     clearProperties();
     initialize();
   }
+
+  /**
+   * Loads the properties from local properties file in the specified directory
+   * <p>
+   * Only files ending in '.properties' will be loaded
+   */
+  public static void loadPropsFromDir(String dirName)
+  {
+    File[] directoryListing = new File(dirName).listFiles(new FilenameFilter()
+    {
+      @Override
+      public boolean accept(File dir, String name)
+      {
+        return name.endsWith(".properties");
+      }
+    });
+
+    if (directoryListing != null)
+    {
+      for (File file : directoryListing)
+      {
+        loadPropsFromFile(file);
+      }
+    }
+  }
+
+  /**
+   * Loads the properties from the specified file
+   */
+  public static void loadPropsFromFile(File file)
+  {
+    if (file.exists())
+    {
+      try (InputStream stream = new FileInputStream(file);)
+      {
+        logger.info("Loading properties file '" + file.getAbsolutePath() + "'");
+        props.load(stream);
+        stream.close();
+      } catch (IOException e)
+      {
+        logger.error("Problem loading properties file '" + file.getAbsolutePath() + "'");
+        e.printStackTrace();
+      }
+    }
+    else
+    {
+      logger.warn("Properties file does not exist: '" + file.getAbsolutePath() + "'");
+    }
+  }
+
+  /**
+   * Loads the properties from the specified file on the classpath
+   */
+  public static void loadPropsFromStream(String name)
+  {
+    try
+    {
+      InputStream stream = SystemConfiguration.class.getClassLoader().getResourceAsStream(name);
+      if (stream != null)
+      {
+        logger.info("Loading file '" + name + "'");
+        props.load(stream);
+        stream.close();
+      }
+      else
+      {
+        logger.error("No file found '" + name + "'");
+      }
+    } catch (IOException e)
+    {
+      logger.error("Problem loading file '" + name + "'");
+      e.printStackTrace();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/1b38ea67/src/main/resources/pirk.properties
----------------------------------------------------------------------
diff --git a/src/main/resources/pirk.properties b/src/main/resources/pirk.properties
index d6ea68d..eb386da 100755
--- a/src/main/resources/pirk.properties
+++ b/src/main/resources/pirk.properties
@@ -26,7 +26,7 @@ log4jPropertiesFile=logging/log4j2.properties
 
 #Name of the local properties file - used when running with the 
 #hadoop jar command
-local.pirk.properties=/root/local.pirk.properties
+local.pirk.properties=/root/
 
 ##
 ##Spark path for SparkLauncher
@@ -36,8 +36,7 @@ spark.home = /usr
 ##
 ## Data schema properties
 ##
-## Each data schema should be specified in an xml file of the form; 
-## 		all items are treated in a case insensitive manner:
+## Each data schema should be specified in an xml file of the form:
 ##
 ##<schema>
 ##  <schemaName> name of the schema </schemaName>
@@ -201,7 +200,7 @@ pir.esQuery = none
 pir.outputFile = none
 
 #Fully qualified dir in hdfs of file containing stoplist terms
-pir.stopListFile = none
+pir.stopListFile = stopListFile
 
 #Number of reduce tasks
 pir.numReduceTasks = 100

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/1b38ea67/src/main/resources/querier.properties
----------------------------------------------------------------------
diff --git a/src/main/resources/querier.properties b/src/main/resources/querier.properties
new file mode 100644
index 0000000..2051d46
--- /dev/null
+++ b/src/main/resources/querier.properties
@@ -0,0 +1,123 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+###############################################################################
+
+##
+## Required Properties
+##
+
+#action -- required - 'encrypt' or 'decrypt' -- The action performed by the QuerierDriver
+querier.action=
+
+#inputFile - required - Fully qualified file containing input
+#The input is either:
+#(1) For Encryption: A query file - Contains the query selectors, one per line;
+#the first line must be the query number
+#OR
+#(2) For Decryption: A response file - Contains the serialized Response object
+querier.inputFile=
+
+#outputFile -- required - Fully qualified file for the result output.
+#The output file specifies either:
+#(1) For encryption:
+#(a) A file to contain the serialized Querier object named: <outputFile>-querier
+#AND
+#(b) A file to contain the serialized Query object named: <outputFile>-query
+#OR
+#(2) A file to contain the decryption results where each line is where each line
+#corresponds to one hit and is a JSON object with the schema QuerySchema
+querier.outputFile=
+
+#numThreads -- required -- Number of threads to use for encryption/decryption
+querier.numThreads=
+
+##
+## Optional Args
+##
+
+#dataSchemas -- optional -- Comma separated list of data schema file names to load
+#querier.dataSchemas=
+
+#querySchemas -- optional -- Comma separated list of query schema file names to load
+#querier.dataSchemas=
+
+
+## Optional, but required for Encryption (ignored if not encrypting)
+
+#bitset -- required for encryption -- Ensure that this bit position is set in the Paillier
+#modulus (will generate Paillier moduli until finding one in which this bit is set)
+#querier.bitSet=
+
+#certainty -- required for encryption -- Certainty of prime generation for Paillier
+#must be greater than or equal to 128
+#querier.certainty=
+
+#dataPartitionBitSize -- required for encryption -- Partition bit size in data partitioning
+#querier.dataPartitionBitSize=
+
+#embedSelector - required for encryption -- 'true' or 'false'
+#Whether or not to embed the selector in the results to reduce false positives
+#Defaults to 'true'
+#querier.embedSelector=
+
+#embedQuerySchema - true or false
+#Whether or not to embed the QuerySchema in the Query (via QueryInfo)
+#Defaults to 'false'
+#querier.embedQuerySchema=
+ 
+#hashBitSize - required for encryption-- Bit size of keyed hash
+#querier.hashBitSize=
+
+#hashKey -- required for encryption -- String key for the keyed hash functionality
+#querier.hashKey=
+
+#useHDFSLookupTable -- required for encryption -- 'true' or 'false'
+#Whether or not to generate and use a hdfs modular exponentation lookup table
+#Defaults to 'false'
+#querier.useHDFSLookupTable=
+
+#memLookupTable -- required for encryption -- 'true' or 'false'
+#Whether or not to generate and use an in memory modular exponentation lookup table - only for
+#standalone/testing right now...
+#Defaults to 'false'
+#querier.memLookupTable=
+
+#paillierBitSize -- required for encryption -- Paillier modulus size N
+#querier.paillierBitSize=
+ 
+#queryName -- required for encryption -- Name of the query
+#querier.queryName=
+
+#queryType -- required for encryption
+#Type of the query as defined in the 'schemaName' tag of the corresponding query schema file
+#querier.queryType=
+
+#secureRandomAlg -- specify the SecureRandom algorithm
+#Defaults to NativePRNG
+#querier.secureRandomAlg=
+
+#secureRandomProvider -- specify the SecureRandom provider
+#Defaults to SUN
+#querier.secureRandomProvider=
+  
+## Optional, but required for Decryption (ignored if not decrypting)
+
+#querierFile -- required for decryption
+#Fully qualified file containing the serialized Querier object
+#querier.querierFile=
+  
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/1b38ea67/src/main/resources/responder.properties
----------------------------------------------------------------------
diff --git a/src/main/resources/responder.properties b/src/main/resources/responder.properties
new file mode 100644
index 0000000..11ad7f6
--- /dev/null
+++ b/src/main/resources/responder.properties
@@ -0,0 +1,137 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+###############################################################################
+
+##
+## Required Properties
+##
+
+#dataInputFormat -- required -- 'base', 'elasticsearch', or 'standalone' -- Specify the input format
+pir.dataInputFormat=
+
+#outputFile -- required -- Fully qualified name of output file in hdfs
+pir.outputFile=
+
+#platform -- required -- 'mapreduce', 'spark', or 'standalone'
+#Processing platform technology for the responder                
+platform= 
+
+#queryInput -- required -- Fully qualified dir in hdfs of Query files
+pir.queryInput=
+
+
+##
+## Optional Args - Leave empty if not using/not changing default values
+##
+
+#inputData -- required if baseInputFormat = 'base'
+#Fully qualified name of input file/directory in hdfs; used if inputFormat = 'base'
+#pir.inputData=
+ 
+#dataSchemas -- optional -- Comma separated list of data schema file names to load
+#responder.dataSchemas=
+
+#querySchemas -- optional -- Comma separated list of query schema file names to load
+#responder.querySchemas=                                    
+
+#allowAdHocQuerySchemas -- 'true' or 'false'
+#If true, allows embedded QuerySchemas for a query.
+#Defaults to 'false'
+#pir.allowEmbeddedQuerySchemas=
+
+#colMultReduceByKey -- 'true' or 'false' -- Spark only
+#If true, uses reduceByKey in performing column multiplication; if false, uses groupByKey -> reduce
+#Defaults to 'false' 
+#pir.colMultReduceByKey=
+
+#baseInputFormat -- required if baseInputFormat = 'base'
+#Full class name of the InputFormat to use when reading in the data - must extend BaseInputFormat
+#pir.baseInputFormat=
+
+#esQuery -- required if baseInputFormat = 'elasticsearch' -- ElasticSearch query
+#if using 'elasticsearch' input format
+#pir.esQuery=
+
+#esResource --  required if baseInputFormat = 'elasticsearch'
+#Requires the format <index>/<type> : Elasticsearch resource where data is read and written to
+#pir.esResource=
+
+#useHDFSLookupTable -- 'true' or 'false' - Whether or not to generate and use the
+#hdfs lookup table for modular exponentiation
+#Defaults to 'false'
+#pir.useHDFSLookupTable=
+
+#baseQuery -- ElasticSearch-like query if using 'base' input format -
+#used to filter records in the RecordReader
+#Defaults to ?q=*
+#pir.baseQuery=
+
+#limitHitsPerSelector -- 'true' or 'false'
+#Whether or not to limit the number of hits per selector
+#Defaults to 'true'
+#pir.limitHitsPerSelector=
+
+#mapreduceMapJavaOpts -- Amount of heap (in MB) to allocate per map task
+#Defaults to -Xmx2800m
+#mapreduce.map.java.opts=
+
+#mapreduceMapMemoryMb -- Amount of memory (in MB) to allocate per map task
+#Defaults to 3000
+#mapreduce.map.memory.mb=
+     
+#mapreduceReduceJavaOpts
+#Amount of heap (in MB) to allocate per reduce task
+#Defaults to -Xmx2800m
+#mapreduce.reduce.java.opts=
+
+#mapreduceReduceMemoryMb
+#Amount of memory (in MB) to allocate per reduce task
+#Defaults to 3000
+#mapreduce.reduce.memory.mb=
+
+#stopListFile -- optional (unless using StopListFilter) -- Fully qualified file in hdfs
+#containing stoplist terms; used by the StopListFilter
+#pir.stopListFile=
+
+#useLocalCache -- 'true' or 'false'
+#Whether or not to use the local cache for modular exponentiation
+#Defaults to 'true'
+#pir.useLocalCache=
+
+#useModExpJoin -- 'true' or 'false' -- Spark only
+#Whether or not to pre-compute the modular exponentiation table and join it to the data
+#partitions when performing the encrypted row calculations
+#Defaults to 'false'
+#pir.useModExpJoin=
+
+#numReduceTasks -- optional -- Number of reduce tasks
+#pir.numReduceTasks=
+
+#numColMultPartitions -- optional, Spark only
+#Number of partitions to use when performing  column multiplication
+#pir.numColMultPartitions=        
+
+#maxHitsPerSelector --  optional -- Max number of hits encrypted per selector
+#pir.maxHitsPerSelector=
+
+#dataParts -- optional -- Number of partitions for the input data
+#pir.numDataPartitions=
+
+#numExpLookupPartitions -- optional -- Number of partitions for the exp lookup table
+#pir.numExpLookupPartitions=
+ 
\ No newline at end of file