metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmiklav...@apache.org
Subject [2/3] metron git commit: METRON-1188: Ambari global configuration management (mmiklavc) closes apache/metron#760
Date Thu, 21 Sep 2017 15:59:12 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index 36863e3..fcc8050 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -17,18 +17,13 @@
  */
 package org.apache.metron.common.configuration;
 
-import org.apache.commons.io.FilenameUtils;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
-import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.dsl.StellarFunctions;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.zookeeper.KeeperException;
+import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT;
+import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL;
+import static org.apache.metron.common.configuration.ConfigurationType.INDEXING;
+import static org.apache.metron.common.configuration.ConfigurationType.PARSER;
+import static org.apache.metron.common.configuration.ConfigurationType.PROFILER;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
@@ -38,8 +33,17 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-
-import static org.apache.metron.common.configuration.ConfigurationType.*;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.StellarFunctions;
+import org.apache.zookeeper.KeeperException;
 
 public class ConfigurationsUtils {
 
@@ -55,7 +59,7 @@ public class ConfigurationsUtils {
     }
   }
   public static void writeGlobalConfigToZookeeper(Map<String, Object> globalConfig, CuratorFramework client) throws Exception {
-    writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSON(globalConfig), client);
+    writeGlobalConfigToZookeeper(JSONUtils.INSTANCE.toJSONPretty(globalConfig), client);
   }
 
   public static void writeGlobalConfigToZookeeper(byte[] globalConfig, String zookeeperUrl) throws Exception {
@@ -76,7 +80,7 @@ public class ConfigurationsUtils {
   }
 
   public static void writeSensorParserConfigToZookeeper(String sensorType, SensorParserConfig sensorParserConfig, String zookeeperUrl) throws Exception {
-    writeSensorParserConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorParserConfig), zookeeperUrl);
+    writeSensorParserConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorParserConfig), zookeeperUrl);
   }
 
   public static void writeSensorParserConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception {
@@ -93,7 +97,7 @@ public class ConfigurationsUtils {
   }
 
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, Map<String, Object> sensorIndexingConfig, String zookeeperUrl) throws Exception {
-    writeSensorIndexingConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorIndexingConfig), zookeeperUrl);
+    writeSensorIndexingConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorIndexingConfig), zookeeperUrl);
   }
 
   public static void writeSensorIndexingConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception {
@@ -109,7 +113,7 @@ public class ConfigurationsUtils {
   }
 
   public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, SensorEnrichmentConfig sensorEnrichmentConfig, String zookeeperUrl) throws Exception {
-    writeSensorEnrichmentConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSON(sensorEnrichmentConfig), zookeeperUrl);
+    writeSensorEnrichmentConfigToZookeeper(sensorType, JSONUtils.INSTANCE.toJSONPretty(sensorEnrichmentConfig), zookeeperUrl);
   }
 
   public static void writeSensorEnrichmentConfigToZookeeper(String sensorType, byte[] configData, String zookeeperUrl) throws Exception {
@@ -125,13 +129,37 @@ public class ConfigurationsUtils {
   }
 
   public static void writeConfigToZookeeper(String name, Map<String, Object> config, String zookeeperUrl) throws Exception {
-    writeConfigToZookeeper(name, JSONUtils.INSTANCE.toJSON(config), zookeeperUrl);
+    writeConfigToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, JSONUtils.INSTANCE.toJSONPretty(config), zookeeperUrl);
   }
 
-  public static void writeConfigToZookeeper(String name, byte[] config, String zookeeperUrl) throws Exception {
-    try(CuratorFramework client = getClient(zookeeperUrl)) {
+  public static void writeConfigToZookeeper(ConfigurationType configType, byte[] configData,
+      String zookeeperUrl) throws Exception {
+    writeConfigToZookeeper(configType, Optional.empty(), configData, zookeeperUrl);
+  }
+
+  public static void writeConfigToZookeeper(ConfigurationType configType,
+      Optional<String> configName, byte[] configData, String zookeeperUrl) throws Exception {
+    writeConfigToZookeeper(getConfigZKPath(configType, configName), configData, zookeeperUrl);
+  }
+
+  public static void writeConfigToZookeeper(ConfigurationType configType,Optional<String> configName,
+      byte[] configData, CuratorFramework client) throws Exception {
+    writeToZookeeper(getConfigZKPath(configType, configName), configData, client);
+  }
+
+  private static String getConfigZKPath(ConfigurationType configType, Optional<String> configName) {
+    String pathSuffix = configName.isPresent() && configType != GLOBAL ? "/" + configName : "";
+    return configType.getZookeeperRoot() + pathSuffix;
+  }
+
+  /**
+   * Writes config to path in Zookeeper, /metron/topology/$CONFIG_TYPE/$CONFIG_NAME
+   */
+  public static void writeConfigToZookeeper(String configPath, byte[] config, String zookeeperUrl)
+      throws Exception {
+    try (CuratorFramework client = getClient(zookeeperUrl)) {
       client.start();
-      writeToZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, config, client);
+      writeToZookeeper(configPath, config, client);
     }
   }
 
@@ -203,8 +231,30 @@ public class ConfigurationsUtils {
     return readFromZookeeper(Constants.ZOOKEEPER_TOPOLOGY_ROOT + "/" + name, client);
   }
 
+  public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType,
+      String zookeeperUrl) throws Exception {
+    return readConfigBytesFromZookeeper(configType, Optional.empty(), zookeeperUrl);
+  }
+
+  public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType, Optional<String> configName,
+      CuratorFramework client) throws Exception {
+    return readFromZookeeper(getConfigZKPath(configType, configName), client);
+  }
+
+  public static byte[] readConfigBytesFromZookeeper(ConfigurationType configType, Optional<String> configName,
+      String zookeeperUrl) throws Exception {
+    return readFromZookeeper(getConfigZKPath(configType, configName), zookeeperUrl);
+  }
+
+  public static byte[] readFromZookeeper(String path, String zookeeperUrl) throws Exception {
+    try (CuratorFramework client = getClient(zookeeperUrl)) {
+      client.start();
+      return readFromZookeeper(path, client);
+    }
+  }
+
   public static byte[] readFromZookeeper(String path, CuratorFramework client) throws Exception {
-    if(client != null && client.getData() != null && path != null) {
+    if (client != null && client.getData() != null && path != null) {
       return client.getData().forPath(path);
     }
     return new byte[]{};
@@ -216,7 +266,7 @@ public class ConfigurationsUtils {
                                               String indexingConfigPath,
                                               String profilerConfigPath,
                                               String zookeeperUrl) throws Exception {
-    try(CuratorFramework client = getClient(zookeeperUrl)) {
+    try (CuratorFramework client = getClient(zookeeperUrl)) {
       client.start();
       uploadConfigsToZookeeper(globalConfigPath, parsersConfigPath, enrichmentsConfigPath, indexingConfigPath, profilerConfigPath, client);
     }
@@ -226,6 +276,61 @@ public class ConfigurationsUtils {
     uploadConfigsToZookeeper(rootFilePath, rootFilePath, rootFilePath, rootFilePath, rootFilePath, client);
   }
 
+  /**
+   * Uploads config to Zookeeper based on the specified rootPath and configuration type. The local
+   * file and zookeeper paths are dynamically calculated based on the rootPath and config type.
+   * When grabbing files from the local FS, the rootPath is used. When reading/writing to Zookeeper,
+   * the path returned by
+   * {@link org.apache.metron.common.configuration.ConfigurationType#getZookeeperRoot()} is used.
+   * For example, when grabbing GLOBAL config from the local FS, the path is based on 'rootPath/.'
+   * whereas PARSER would be based on 'rootPath/parsers'.
+   *
+   * @param rootFilePath base configuration path on the local FS
+   * @param client zk client
+   * @param type config type to upload configs for
+   */
+  public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client,
+      ConfigurationType type) throws Exception {
+    uploadConfigsToZookeeper(rootFilePath, client, type, Optional.empty());
+  }
+
+  /**
+   * Does the same as
+   * {@link org.apache.metron.common.configuration.ConfigurationsUtils#uploadConfigsToZookeeper(
+   * java.lang.String, org.apache.curator.framework.CuratorFramework,
+   * org.apache.metron.common.configuration.ConfigurationType)}
+   * with the addition of being able to specify a specific config name for the given configuration
+   * type. e.g. config type=PARSER, config name=bro
+   *
+   * @param rootFilePath base configuration path on the local FS
+   * @param client zk client
+   * @param type config type to upload configs for
+   * @param configName specific config under the specified config type
+   */
+  public static void uploadConfigsToZookeeper(String rootFilePath, CuratorFramework client,
+      ConfigurationType type, Optional<String> configName) throws Exception {
+    switch (type) {
+      case GLOBAL:
+        final byte[] globalConfig = readGlobalConfigFromFile(rootFilePath);
+        if (globalConfig.length > 0) {
+          setupStellarStatically(client, Optional.of(new String(globalConfig)));
+          writeGlobalConfigToZookeeper(globalConfig, client);
+        }
+        break;
+      case PARSER: // intentional pass-through
+      case ENRICHMENT: // intentional pass-through
+      case INDEXING:
+        Map<String, byte[]> sensorIndexingConfigs = readSensorConfigsFromFile(rootFilePath, type,
+            configName);
+        for (String sensorType : sensorIndexingConfigs.keySet()) {
+          writeConfigToZookeeper(type, configName, sensorIndexingConfigs.get(sensorType), client);
+        }
+        break;
+      default:
+        throw new IllegalArgumentException("Configuration type not found: " + type);
+    }
+  }
+
   public static void uploadConfigsToZookeeper(String globalConfigPath,
                                               String parsersConfigPath,
                                               String enrichmentsConfigPath,
@@ -311,7 +416,7 @@ public class ConfigurationsUtils {
 
   public static byte[] readGlobalConfigFromFile(String rootPath) throws IOException {
     byte[] globalConfig = new byte[0];
-    File configPath = new File(rootPath, GLOBAL.getName() + ".json");
+    File configPath = new File(rootPath, GLOBAL.getTypeName() + ".json");
     if (configPath.exists()) {
       globalConfig = Files.readAllBytes(configPath.toPath());
     }
@@ -319,15 +424,15 @@ public class ConfigurationsUtils {
   }
 
   public static Map<String, byte[]> readSensorParserConfigsFromFile(String rootPath) throws IOException {
-    return readSensorConfigsFromFile(rootPath, PARSER);
+    return readSensorConfigsFromFile(rootPath, PARSER, Optional.empty());
   }
 
   public static Map<String, byte[]> readSensorEnrichmentConfigsFromFile(String rootPath) throws IOException {
-    return readSensorConfigsFromFile(rootPath, ENRICHMENT);
+    return readSensorConfigsFromFile(rootPath, ENRICHMENT, Optional.empty());
   }
 
   public static Map<String, byte[]> readSensorIndexingConfigsFromFile(String rootPath) throws IOException {
-    return readSensorConfigsFromFile(rootPath, INDEXING);
+    return readSensorConfigsFromFile(rootPath, INDEXING, Optional.empty());
   }
 
   /**
@@ -337,7 +442,7 @@ public class ConfigurationsUtils {
   public static byte[] readProfilerConfigFromFile(String rootPath) throws IOException {
 
     byte[] config = new byte[0];
-    File configPath = new File(rootPath, PROFILER.getName() + ".json");
+    File configPath = new File(rootPath, PROFILER.getTypeName() + ".json");
     if (configPath.exists()) {
       config = Files.readAllBytes(configPath.toPath());
     }
@@ -346,19 +451,101 @@ public class ConfigurationsUtils {
   }
 
   public static Map<String, byte[]> readSensorConfigsFromFile(String rootPath, ConfigurationType configType) throws IOException {
+    return readSensorConfigsFromFile(rootPath, configType, Optional.empty());
+  }
+
+  /**
+   * Will read configs from local disk at the specified rootPath. Will read all configs for a given
+   * configuration type. If an optional specific config name is also provided, it will only read
+   * configs for that configuration type and name combo. e.g. PARSER, bro
+   * @param rootPath root FS location to read configs from
+   * @param configType e.g. GLOBAL, PARSER, ENRICHMENT, etc.
+   * @param configName a specific config, for instance a sensor name like bro, yaf, snort, etc.
+   * @return map of file names to the contents of that file as a byte array
+   * @throws IOException
+   */
+  public static Map<String, byte[]> readSensorConfigsFromFile(String rootPath,
+      ConfigurationType configType, Optional<String> configName) throws IOException {
     Map<String, byte[]> sensorConfigs = new HashMap<>();
     File configPath = new File(rootPath, configType.getDirectory());
-    if (configPath.exists()) {
+    if (configPath.exists() && configPath.isDirectory()) {
       File[] children = configPath.listFiles();
-      if (children != null) {
+      if (!configName.isPresent()) {
         for (File file : children) {
-          sensorConfigs.put(FilenameUtils.removeExtension(file.getName()), Files.readAllBytes(file.toPath()));
+          sensorConfigs.put(FilenameUtils.removeExtension(file.getName()),
+              Files.readAllBytes(file.toPath()));
+        }
+      } else {
+        for (File file : children) {
+          if (FilenameUtils.removeExtension(file.getName()).equals(configName.get())) {
+            sensorConfigs.put(FilenameUtils.removeExtension(file.getName()),
+                Files.readAllBytes(file.toPath()));
+          }
+        }
+        if (sensorConfigs.isEmpty()) {
+          throw new RuntimeException("Unable to find configuration for " + configName.get());
         }
       }
     }
     return sensorConfigs;
   }
 
+  /**
+   * Reads Json data for the specified config type from zookeeper,
+   * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format.
+   * Patching JSON flattens existing formatting, so this will keep configs readable.
+   * Starts up curatorclient based on zookeeperUrl.
+   *
+   * @param configurationType GLOBAL, PARSER, etc.
+   * @param configName e.g. bro, yaf, snort
+   * @param patchData a JSON patch in the format specified by RFC 6902
+   * @param zookeeperUrl configs are here
+   */
+  public static void applyConfigPatchToZookeeper(ConfigurationType configurationType,
+      byte[] patchData, String zookeeperUrl) throws Exception {
+    applyConfigPatchToZookeeper(configurationType, Optional.empty(), patchData, zookeeperUrl);
+  }
+
+  /**
+   * Reads Json data for the specified config type and config name (if applicable) from zookeeper,
+   * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format.
+   * Patching JSON flattens existing formatting, so this will keep configs readable.
+   * Starts up curatorclient based on zookeeperUrl.
+   *
+   * @param configurationType GLOBAL, PARSER, etc.
+   * @param configName e.g. bro, yaf, snort
+   * @param patchData a JSON patch in the format specified by RFC 6902
+   * @param zookeeperUrl configs are here
+   */
+  public static void applyConfigPatchToZookeeper(ConfigurationType configurationType,
+      Optional<String> configName, byte[] patchData, String zookeeperUrl) throws Exception {
+    try (CuratorFramework client = getClient(zookeeperUrl)) {
+      client.start();
+      applyConfigPatchToZookeeper(configurationType, configName, patchData, client);
+    }
+  }
+
+  /**
+   * Reads Json data for the specified config type and config name (if applicable) from zookeeper,
+   * applies the patch from patchData, and writes it back to Zookeeper in a pretty print format.
+   * Patching JSON flattens existing formatting, so this will keep configs readable. The
+   * curatorclient should be started already.
+   *
+   * @param configurationType GLOBAL, PARSER, etc.
+   * @param configName e.g. bro, yaf, snort
+   * @param patchData a JSON patch in the format specified by RFC 6902
+   * @param client access to zookeeeper
+   */
+  public static void applyConfigPatchToZookeeper(ConfigurationType configurationType,
+      Optional<String> configName,
+      byte[] patchData, CuratorFramework client) throws Exception {
+    byte[] configData = readConfigBytesFromZookeeper(configurationType, configName, client);
+    JsonNode source = JSONUtils.INSTANCE.readTree(configData);
+    JsonNode patch = JSONUtils.INSTANCE.readTree(patchData);
+    JsonNode patchedConfig = JSONUtils.INSTANCE.applyPatch(patch, source);
+    writeConfigToZookeeper(configurationType, configName,
+        JSONUtils.INSTANCE.toJSONPretty(patchedConfig), client);
+  }
 
   public interface ConfigurationVisitor{
     void visit(ConfigurationType configurationType, String name, String data);
@@ -368,14 +555,14 @@ public class ConfigurationsUtils {
     visitConfigs(client, (type, name, data) -> {
       setupStellarStatically(client, Optional.ofNullable(data));
       callback.visit(type, name, data);
-    }, GLOBAL);
-    visitConfigs(client, callback, PARSER);
-    visitConfigs(client, callback, INDEXING);
-    visitConfigs(client, callback, ENRICHMENT);
-    visitConfigs(client, callback, PROFILER);
+    }, GLOBAL, Optional.empty());
+    visitConfigs(client, callback, PARSER, Optional.empty());
+    visitConfigs(client, callback, INDEXING, Optional.empty());
+    visitConfigs(client, callback, ENRICHMENT, Optional.empty());
+    visitConfigs(client, callback, PROFILER, Optional.empty());
   }
 
-  public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, ConfigurationType configType) throws Exception {
+  public static void visitConfigs(CuratorFramework client, ConfigurationVisitor callback, ConfigurationType configType, Optional<String> configName) throws Exception {
 
     if (client.checkExists().forPath(configType.getZookeeperRoot()) != null) {
 
@@ -388,20 +575,52 @@ public class ConfigurationsUtils {
         callback.visit(configType, "profiler", new String(profilerConfigData));
       }
       else if (configType.equals(PARSER) || configType.equals(ENRICHMENT) || configType.equals(INDEXING)) {
-        List<String> children = client.getChildren().forPath(configType.getZookeeperRoot());
-        for (String child : children) {
-
-          byte[] data = client.getData().forPath(configType.getZookeeperRoot() + "/" + child);
-          callback.visit(configType, child, new String(data));
+        if (configName.isPresent()) {
+          byte[] data = readConfigBytesFromZookeeper(configType, configName,  client);
+          callback.visit(configType, configName.get(), new String(data));
+        } else {
+          List<String> children = client.getChildren().forPath(configType.getZookeeperRoot());
+          for (String child : children) {
+            byte[] data = client.getData().forPath(configType.getZookeeperRoot() + "/" + child);
+            callback.visit(configType, child, new String(data));
+          }
         }
       }
     }
   }
 
+  /**
+   * Writes all config content to the provided print stream.
+   *
+   * @param out stream to use as output
+   * @param client zk client
+   * @throws Exception
+   */
   public static void dumpConfigs(PrintStream out, CuratorFramework client) throws Exception {
     ConfigurationsUtils.visitConfigs(client, (type, name, data) -> {
       type.deserialize(data);
-      out.println(type + " Config: " + name + "\n" + data);
+      out.println(type + " Config: " + name + System.lineSeparator() + data);
     });
   }
+
+  /**
+   * Writes config content for a specific config type to the provided print stream. Optionally
+   * provide a config name in addition to the config type and it will only print the json for a
+   * specific config, e.g. bro, yaf, snort, etc.
+   *
+   * @param out stream to use as output
+   * @param client zk client
+   * @param configType GLOBAL, PARSER, ENRICHMENT, etc.
+   * @param configName Typically a sensor name like bro, snort, yaf, etc.
+   * @throws Exception
+   */
+  public static void dumpConfigs(PrintStream out, CuratorFramework client,
+      ConfigurationType configType, Optional<String> configName) throws Exception {
+    ConfigurationsUtils.visitConfigs(client, (type, name, data) -> {
+      setupStellarStatically(client, Optional.ofNullable(data));
+      type.deserialize(data);
+      out.println(type + " Config: " + name + System.lineSeparator() + data);
+    }, configType, configName);
+  }
 }
+

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
index bf5b856..a28b15c 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfigurations.java
@@ -44,6 +44,6 @@ public class EnrichmentConfigurations extends Configurations {
     }
 
     private String getKey(String sensorType) {
-        return ConfigurationType.ENRICHMENT.getName() + "." + sensorType;
+        return ConfigurationType.ENRICHMENT.getTypeName() + "." + sensorType;
     }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
index 1d60084..3bd7645 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
@@ -59,7 +59,7 @@ public class IndexingConfigurations extends Configurations {
   }
 
   private String getKey(String sensorType) {
-    return ConfigurationType.INDEXING.getName() + "." + sensorType;
+    return ConfigurationType.INDEXING.getTypeName() + "." + sensorType;
   }
 
   public boolean isDefault(String sensorName, String writerName) {

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java
index ab7463f..0ec0ed4 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ParserConfigurations.java
@@ -44,6 +44,6 @@ public class ParserConfigurations extends Configurations {
   }
 
   private String getKey(String sensorType) {
-    return ConfigurationType.PARSER.getName() + "." + sensorType;
+    return ConfigurationType.PARSER.getTypeName() + "." + sensorType;
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java
index c098787..e001d74 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfilerConfigurations.java
@@ -48,6 +48,6 @@ public class ProfilerConfigurations extends Configurations {
   }
 
   private String getKey() {
-    return ConfigurationType.PROFILER.getName();
+    return ConfigurationType.PROFILER.getTypeName();
   }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
index 60d009f..280b167 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,22 +21,26 @@ package org.apache.metron.common.utils;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.flipkart.zjsonpatch.JsonPatch;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
 
-import java.io.*;
-import java.util.Map;
-
 public enum JSONUtils {
   INSTANCE;
 
   private static ThreadLocal<JSONParser> _parser = ThreadLocal.withInitial(() ->
-          new JSONParser());
+      new JSONParser());
 
   private static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() ->
-          new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
+      new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
 
   public <T> T convert(Object original, Class<T> targetClass) {
     return _mapper.get().convertValue(original, targetClass);
@@ -83,8 +87,12 @@ public enum JSONUtils {
     }
   }
 
-  public byte[] toJSON(Object config) throws JsonProcessingException {
-    return _mapper.get().writeValueAsBytes(config);
+  public byte[] toJSONPretty(String config) throws IOException {
+    return toJSONPretty(readTree(config));
+  }
+
+  public byte[] toJSONPretty(Object config) throws JsonProcessingException {
+    return _mapper.get().writerWithDefaultPrettyPrinter().writeValueAsBytes(config);
   }
 
   /**
@@ -93,4 +101,51 @@ public enum JSONUtils {
   public JSONObject toJSONObject(Object o) throws JsonProcessingException, ParseException {
     return (JSONObject) _parser.get().parse(toJSON(o, false));
   }
+
+  /**
+   * Reads a JSON string into a JsonNode Object
+   *
+   * @param json JSON value to deserialize
+   * @return deserialized JsonNode Object
+   */
+  public JsonNode readTree(String json) throws IOException {
+    return _mapper.get().readTree(json);
+  }
+
+  /**
+   * Reads a JSON byte array into a JsonNode Object
+   *
+   * @param json JSON value to deserialize
+   * @return deserialized JsonNode Object
+   */
+  public JsonNode readTree(byte[] json) throws IOException {
+    return _mapper.get().readTree(json);
+  }
+
+  /**
+   * Update JSON given a JSON Patch (see RFC 6902 at https://tools.ietf.org/html/rfc6902)
+   * Operations:
+   * <ul>
+   *   <li>add</li>
+   *   <li>remove</li>
+   *   <li>replace</li>
+   *   <li>move</li>
+   *   <li>copy</li>
+   *   <li>test</li>
+   * </ul>
+   *
+   * @param patch Array of JSON patches, e.g. [{ "op": "move", "from": "/a", "path": "/c" }]
+   * @param source Source JSON to apply patch to
+   * @return new json after applying the patch
+   */
+  public JsonNode applyPatch(String patch, String source) throws IOException {
+    JsonNode patchNode = readTree(patch);
+    JsonNode sourceNode = readTree(source);
+    return applyPatch(patchNode, sourceNode);
+  }
+
+  public JsonNode applyPatch(JsonNode patch, JsonNode source) throws IOException {
+    return JsonPatch.apply(patch, source);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java
index 9b0d77a..0882788 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/StringUtils.java
@@ -19,8 +19,6 @@
 package org.apache.metron.common.utils;
 
 import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-
 import java.util.Arrays;
 import java.util.Optional;
 
@@ -33,4 +31,16 @@ public class StringUtils {
              .toArray()
                                 );
   }
+
+  /**
+   * Strips specified number of lines from beginning for String val
+   * @param val
+   * @param numLines
+   */
+  public static String stripLines(String val, int numLines) {
+    int start = org.apache.commons.lang3.StringUtils.ordinalIndexOf(val, System.lineSeparator(), numLines);
+    start = start >= 0 ? start : 0;
+    return val.substring(start);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java
index 68ef604..44612cd 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBoltTest.java
@@ -66,7 +66,7 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest {
     this.zookeeperUrl = testZkServer.getConnectString();
     byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
     ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl);
-    enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getName());
+    enrichmentConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName());
     Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
     for (String sensorType : sensorEnrichmentConfigs.keySet()) {
       ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl);
@@ -105,13 +105,13 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest {
     Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
     sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
     ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
-    waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
     Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig());
 
     configsUpdated = new HashSet<>();
     sampleGlobalConfig.remove("newGlobalField");
     ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
-    waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
     Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
 
     configsUpdated = new HashSet<>();
@@ -133,4 +133,4 @@ public class ConfiguredEnrichmentBoltTest extends BaseConfiguredBoltTest {
     Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations());
     configuredBolt.cleanup();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
index db56892..06603a3 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/bolt/ConfiguredParserBoltTest.java
@@ -67,7 +67,7 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest {
     this.zookeeperUrl = testZkServer.getConnectString();
     byte[] globalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
     ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, zookeeperUrl);
-    parserConfigurationTypes.add(ConfigurationType.GLOBAL.getName());
+    parserConfigurationTypes.add(ConfigurationType.GLOBAL.getTypeName());
     Map<String, byte[]> sensorEnrichmentConfigs = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
     for (String sensorType : sensorEnrichmentConfigs.keySet()) {
       ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, sensorEnrichmentConfigs.get(sensorType), zookeeperUrl);
@@ -106,13 +106,13 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest {
     Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
     sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
     ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
-    waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
     Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig());
 
     configsUpdated = new HashSet<>();
     sampleGlobalConfig.remove("newGlobalField");
     ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
-    waitForConfigUpdate(ConfigurationType.GLOBAL.getName());
+    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
     Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
 
     configsUpdated = new HashSet<>();
@@ -129,4 +129,4 @@ public class ConfiguredParserBoltTest extends BaseConfiguredBoltTest {
     Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations());
     configuredBolt.cleanup();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
index c9a7d22..ed7f4bc 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationManagerIntegrationTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,33 +18,57 @@
 
 package org.apache.metron.common.cli;
 
+import static org.apache.metron.common.cli.ConfigurationManager.PatchMode.ADD;
+import static org.apache.metron.common.configuration.ConfigurationType.ENRICHMENT;
+import static org.apache.metron.common.configuration.ConfigurationType.GLOBAL;
+import static org.apache.metron.common.configuration.ConfigurationType.INDEXING;
+import static org.apache.metron.common.configuration.ConfigurationType.PARSER;
+import static org.apache.metron.common.utils.StringUtils.stripLines;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.fail;
+
 import com.google.common.base.Splitter;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.Iterables;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.nio.file.DirectoryNotEmptyException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import org.adrianwalker.multilinestring.Multiline;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.metron.TestConstants;
+import org.apache.metron.common.cli.ConfigurationManager.PatchMode;
 import org.apache.metron.common.configuration.ConfigurationType;
 import org.apache.metron.common.configuration.ConfigurationsUtils;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.integration.utils.TestUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.*;
-
 public class ConfigurationManagerIntegrationTest {
   private TestingServer testZkServer;
   private CuratorFramework client;
   private String zookeeperUrl;
   private String outDir = "target/configs";
+  private File tmpDir;
+  private File configDir;
+  private File parsersDir;
+  private File enrichmentsDir;
+  private File indexingDir;
   private Set<String> sensors = new HashSet<>();
 
   private void cleanDir(File rootDir) throws IOException {
@@ -69,12 +93,17 @@ public class ConfigurationManagerIntegrationTest {
     zookeeperUrl = testZkServer.getConnectString();
     client = ConfigurationsUtils.getClient(zookeeperUrl);
     client.start();
-    File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ConfigurationType.ENRICHMENT.getDirectory());
+    File sensorDir = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), ENRICHMENT.getDirectory());
     sensors.addAll(Collections2.transform(
              Arrays.asList(sensorDir.list())
             ,s -> Iterables.getFirst(Splitter.on('.').split(s), "null")
                                          )
                   );
+    tmpDir = TestUtils.createTempDir(this.getClass().getName());
+    configDir = TestUtils.createDir(tmpDir, "config");
+    parsersDir = TestUtils.createDir(configDir, "parsers");
+    enrichmentsDir = TestUtils.createDir(configDir, "enrichments");
+    indexingDir = TestUtils.createDir(configDir, "indexing");
     pushConfigs();
   }
 
@@ -112,11 +141,11 @@ public class ConfigurationManagerIntegrationTest {
   public void validateConfigsOnDisk(File configDir) throws IOException {
     File globalConfigFile = new File(configDir, "global.json");
     Assert.assertTrue("Global config does not exist", globalConfigFile.exists());
-    validateConfig("global", ConfigurationType.GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI()))));
+    validateConfig("global", GLOBAL, new String(Files.readAllBytes(Paths.get(globalConfigFile.toURI()))));
     for(String sensor : sensors) {
-      File sensorFile = new File(configDir, ConfigurationType.ENRICHMENT.getDirectory() + "/" + sensor + ".json");
+      File sensorFile = new File(configDir, ENRICHMENT.getDirectory() + "/" + sensor + ".json");
       Assert.assertTrue(sensor + " config does not exist", sensorFile.exists());
-      validateConfig(sensor, ConfigurationType.ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI()))));
+      validateConfig(sensor, ENRICHMENT, new String(Files.readAllBytes(Paths.get(sensorFile.toURI()))));
     }
   }
 
@@ -128,7 +157,7 @@ public class ConfigurationManagerIntegrationTest {
     try {
       //second time without force should
       pullConfigs(false);
-      Assert.fail("Should have failed to pull configs in a directory structure that already exists.");
+      fail("Should have failed to pull configs in a directory structure that already exists.");
     }
     catch(IllegalStateException t) {
       //make sure we didn't bork anything
@@ -142,7 +171,7 @@ public class ConfigurationManagerIntegrationTest {
       try {
         type.deserialize(data);
       } catch (Exception e) {
-        Assert.fail("Unable to load config " + name + ": " + data);
+        fail("Unable to load config " + name + ": " + data);
       }
   }
   @Test
@@ -155,7 +184,7 @@ public class ConfigurationManagerIntegrationTest {
       public void visit(ConfigurationType configurationType, String name, String data) {
         Assert.assertTrue(data.length() > 0);
         validateConfig(name, configurationType, data);
-        if(configurationType == ConfigurationType.GLOBAL) {
+        if(configurationType == GLOBAL) {
           validateConfig(name, configurationType, data);
           foundGlobal.set(true);
         }
@@ -168,6 +197,303 @@ public class ConfigurationManagerIntegrationTest {
     Assert.assertEquals(sensorsInZookeeper, sensors);
   }
 
+  /**
+   * { "a": "b" }
+   */
+  @Multiline
+  private static String someConfig;
+
+  @Test
+  public void writes_global_config_to_zookeeper() throws Exception {
+    File configFile = new File(configDir, "global.json");
+    TestUtils.write(configFile, someConfig);
+    pushConfigs(GLOBAL, configDir);
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  private void pushConfigs(ConfigurationType type, File configPath) throws Exception {
+    pushConfigs(type, configPath, Optional.empty());
+  }
+
+  private void pushConfigs(ConfigurationType type, File configPath, Optional<String> configName) throws Exception {
+    String[] args = new String[]{
+        "-z", zookeeperUrl,
+        "--mode", "PUSH",
+        "--config_type", type.toString(),
+        "--input_dir", configPath.getAbsolutePath()
+    };
+    if (configName.isPresent()) {
+      args = ArrayUtils.addAll(args, "--config_name", configName.get());
+    }
+    ConfigurationManager manager = new ConfigurationManager();
+    manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args));
+  }
+
+  private String dumpConfigs(ConfigurationType type) throws Exception {
+    return dumpConfigs(type, Optional.empty());
+  }
+
+  private String dumpConfigs(ConfigurationType type, Optional<String> configName) throws Exception {
+    String[] args = new String[]{
+        "-z", zookeeperUrl,
+        "--mode", "DUMP",
+        "--config_type", type.toString()
+    };
+    if (configName.isPresent()) {
+      args = ArrayUtils.addAll(args, "--config_name", configName.get());
+    }
+    ConfigurationManager manager = new ConfigurationManager();
+    return redirectSystemOut(args, (a) -> {
+        manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), a));
+    });
+  }
+
+  public interface RedirectCallback {
+
+    void call(String[] args) throws Exception;
+  }
+
+  private String redirectSystemOut(final String[] args, RedirectCallback callback)
+      throws Exception {
+    PrintStream os = System.out;
+    try (OutputStream baos = new ByteArrayOutputStream();
+        PrintStream ps = new PrintStream(baos)) {
+      System.setOut(ps);
+      callback.call(args);
+      System.out.flush();
+      System.setOut(os);
+      return baos.toString();
+    } finally {
+      System.setOut(os);
+    }
+  }
+
+  /**
+   *{
+   "parserClassName": "org.apache.metron.parsers.GrokParser",
+   "sensorTopic": "squid",
+   "parserConfig": {
+   "grokPath": "/patterns/squid",
+   "patternLabel": "SQUID_DELIMITED",
+   "timestampField": "timestamp"
+   },
+   "fieldTransformations" : [
+   {
+   "transformation" : "STELLAR"
+   ,"output" : [ "full_hostname", "domain_without_subdomains" ]
+   ,"config" : {
+   "full_hostname" : "URL_TO_HOST(url)"
+   ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
+   }
+   }
+   ]
+   }
+   */
+  @Multiline
+  private static String squidParserConfig;
+
+  @Test
+  public void writes_single_parser_config_to_zookeeper() throws Exception {
+    File configFile = new File(parsersDir, "myparser.json");
+    TestUtils.write(configFile, squidParserConfig);
+    pushConfigs(PARSER, configDir, Optional.of("myparser"));
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(squidParserConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  /**
+   * {
+   "enrichment" : {
+   "fieldMap": {
+   "geo": ["ip_dst_addr", "ip_src_addr"],
+   "host": ["host"]
+   }
+   },
+   "threatIntel": {
+   "fieldMap": {
+   "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+   },
+   "fieldToTypeMap": {
+   "ip_src_addr" : ["malicious_ip"],
+   "ip_dst_addr" : ["malicious_ip"]
+   }
+   }
+   }
+   */
+  @Multiline
+  private static String someEnrichmentConfig;
+
+  @Test
+  public void writes_single_enrichment_config_to_zookeeper() throws Exception {
+    File configFile = new File(enrichmentsDir, "myenrichment.json");
+    TestUtils.write(configFile, someEnrichmentConfig);
+    pushConfigs(ENRICHMENT, configDir, Optional.of("myenrichment"));
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someEnrichmentConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(ENRICHMENT, Optional.of("myenrichment")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  /**
+   * {
+   "hdfs" : {
+   "index": "myindex",
+   "batchSize": 5,
+   "enabled" : true
+   },
+   "elasticsearch" : {
+   "index": "myindex",
+   "batchSize": 5,
+   "enabled" : true
+   },
+   "solr" : {
+   "index": "myindex",
+   "batchSize": 5,
+   "enabled" : true
+   }
+   }
+   */
+  @Multiline
+  private static String someIndexingConfig;
+
+  @Test
+  public void writes_single_indexing_config_to_zookeeper() throws Exception {
+    File configFile = new File(indexingDir, "myindex.json");
+    TestUtils.write(configFile, someIndexingConfig);
+    pushConfigs(INDEXING, configDir, Optional.of("myindex"));
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(someIndexingConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(INDEXING, Optional.of("myindex")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  /**
+   * [ { "op": "replace", "path": "/a", "value": [ "new1", "new2" ] } ]
+   */
+  @Multiline
+  private static String somePatchConfig;
+
+  /**
+   * { "a": [ "new1", "new2" ] }
+   */
+  @Multiline
+  private static String expectedSomeConfig;
+
+  @Test
+  public void patches_global_config_from_file() throws Exception {
+    File patchFile = new File(tmpDir, "global-config-patch.json");
+    TestUtils.write(patchFile, somePatchConfig);
+    File configFile = new File(configDir, "global.json");
+    TestUtils.write(configFile, someConfig);
+    pushConfigs(GLOBAL, configDir, Optional.of("global"));
+    patchConfigs(GLOBAL, Optional.of(patchFile), Optional.of("global"), Optional.empty(), Optional.empty(), Optional.empty());
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedSomeConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  private void patchConfigs(ConfigurationType type, Optional<File> patchPath,
+      Optional<String> configName, Optional<PatchMode> patchMode, Optional<String> key,
+      Optional<String> value) throws Exception {
+    String[] args = new String[]{
+        "-z", zookeeperUrl,
+        "--mode", "PATCH",
+        "--config_type", type.toString()
+    };
+    if (configName.isPresent()) {
+      args = ArrayUtils.addAll(args, "--config_name", configName.get());
+    }
+    if (patchPath.isPresent()) {
+      args = ArrayUtils.addAll(args, "--patch_file", patchPath.get().getAbsolutePath());
+    } else if (patchMode.isPresent()) {
+      args = ArrayUtils.addAll(args,
+          "--patch_mode", patchMode.get().toString(),
+          "--patch_key", key.get(),
+          "--patch_value", value.get());
+    }
+    ConfigurationManager manager = new ConfigurationManager();
+    manager.run(ConfigurationManager.ConfigurationOptions.parse(new PosixParser(), args));
+  }
+
+  /**
+   * [ { "op": "replace", "path": "/parserConfig/timestampField", "value": "heyjoe" } ]
+   */
+  @Multiline
+  public static String someParserPatch;
+
+  /**
+   *{
+   "parserClassName": "org.apache.metron.parsers.GrokParser",
+   "sensorTopic": "squid",
+   "parserConfig": {
+   "grokPath": "/patterns/squid",
+   "patternLabel": "SQUID_DELIMITED",
+   "timestampField": "heyjoe"
+   },
+   "fieldTransformations" : [
+   {
+   "transformation" : "STELLAR"
+   ,"output" : [ "full_hostname", "domain_without_subdomains" ]
+   ,"config" : {
+   "full_hostname" : "URL_TO_HOST(url)"
+   ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)"
+   }
+   }
+   ]
+   }
+   */
+  @Multiline
+  public static String expectedPatchedParser;
+
+  @Test
+  public void patches_parser_config_from_file() throws Exception {
+    File patchFile = new File(tmpDir, "parser-patch.json");
+    TestUtils.write(patchFile, someParserPatch);
+    File configFile = new File(parsersDir, "myparser.json");
+    TestUtils.write(configFile, squidParserConfig);
+    pushConfigs(PARSER, configDir, Optional.of("myparser"));
+    patchConfigs(PARSER, Optional.of(patchFile), Optional.of("myparser"), Optional.empty(), Optional.empty(), Optional.empty());
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  @Test
+  public void patches_parser_config_from_key_value() throws Exception {
+    File configFile = new File(parsersDir, "myparser.json");
+    TestUtils.write(configFile, squidParserConfig);
+    pushConfigs(PARSER, configDir, Optional.of("myparser"));
+    patchConfigs(PARSER, Optional.empty(), Optional.of("myparser"), Optional.of(ADD), Optional.of("/parserConfig/timestampField"), Optional.of("\"\"heyjoe\"\""));
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedPatchedParser);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(PARSER, Optional.of("myparser")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
+  /**
+   * {
+   *   "a": "b",
+   *   "foo": {
+   *     "bar": {
+   *       "baz": [ "bazval1", "bazval2" ]
+   *     }
+   *   }
+   * }
+   */
+  @Multiline
+  private static String expectedComplexConfig;
+
+  @Test
+  public void patches_global_config_from_complex_key_value() throws Exception {
+    File configFile = new File(configDir, "global.json");
+    TestUtils.write(configFile, someConfig);
+    pushConfigs(GLOBAL, configDir, Optional.of("global"));
+    patchConfigs(GLOBAL, Optional.empty(), Optional.of("global"), Optional.of(ADD), Optional.of("/foo"), Optional.of("{ \"bar\" : { \"baz\" : [ \"bazval1\", \"bazval2\" ] } }"));
+    byte[] expected = JSONUtils.INSTANCE.toJSONPretty(expectedComplexConfig);
+    byte[] actual = JSONUtils.INSTANCE.toJSONPretty(stripLines(dumpConfigs(GLOBAL, Optional.of("global")), 1));
+    Assert.assertThat(actual, equalTo(expected));
+  }
+
   @After
   public void tearDown() throws IOException {
     client.close();

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
deleted file mode 100644
index 0c72183..0000000
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/cli/ConfigurationsUtilsTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.common.cli;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
-import org.apache.metron.common.utils.JSONUtils;
-import org.junit.Assert;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-public class ConfigurationsUtilsTest {
-
-  private TestingServer testZkServer;
-  private String zookeeperUrl;
-  private CuratorFramework client;
-  private byte[] expectedGlobalConfig;
-  private Map<String, byte[]> expectedSensorParserConfigMap;
-  private Map<String, byte[]> expectedSensorEnrichmentConfigMap;
-
-  @Before
-  public void setup() throws Exception {
-    testZkServer = new TestingServer(true);
-    zookeeperUrl = testZkServer.getConnectString();
-    client = ConfigurationsUtils.getClient(zookeeperUrl);
-    client.start();
-    expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
-    expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
-    expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
-  }
-
-  @Test
-  public void test() throws Exception {
-    Assert.assertTrue(expectedGlobalConfig.length > 0);
-    ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl);
-    byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client);
-    Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes));
-
-    Assert.assertTrue(expectedSensorParserConfigMap.size() > 0);
-    String testSensorType = "yaf";
-    byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType);
-    ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl);
-    byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client);
-    Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes));
-
-    Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0);
-    byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType);
-    ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl);
-    byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client);
-    Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes));
-
-    String name = "testConfig";
-    Map<String, Object> testConfig = new HashMap<>();
-    testConfig.put("stringField", "value");
-    testConfig.put("intField", 1);
-    testConfig.put("doubleField", 1.1);
-    ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl);
-    byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client);
-    Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSON(testConfig), readConfigBytes));
-
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    client.close();
-    testZkServer.close();
-    testZkServer.stop();
-  }
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
new file mode 100644
index 0000000..145d7ce
--- /dev/null
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigurationsUtilsTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.common.configuration;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConfigurationsUtilsTest {
+
+  private TestingServer testZkServer;
+  private String zookeeperUrl;
+  private CuratorFramework client;
+  private byte[] expectedGlobalConfig;
+  private Map<String, byte[]> expectedSensorParserConfigMap;
+  private Map<String, byte[]> expectedSensorEnrichmentConfigMap;
+
+  @Before
+  public void setup() throws Exception {
+    testZkServer = new TestingServer(true);
+    zookeeperUrl = testZkServer.getConnectString();
+    client = ConfigurationsUtils.getClient(zookeeperUrl);
+    client.start();
+    expectedGlobalConfig = ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH);
+    expectedSensorParserConfigMap = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
+    expectedSensorEnrichmentConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH);
+  }
+
+  @Test
+  public void test() throws Exception {
+    Assert.assertTrue(expectedGlobalConfig.length > 0);
+    ConfigurationsUtils.writeGlobalConfigToZookeeper(expectedGlobalConfig, zookeeperUrl);
+    byte[] actualGlobalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client);
+    Assert.assertTrue(Arrays.equals(expectedGlobalConfig, actualGlobalConfigBytes));
+
+    Assert.assertTrue(expectedSensorParserConfigMap.size() > 0);
+    String testSensorType = "yaf";
+    byte[] expectedSensorParserConfigBytes = expectedSensorParserConfigMap.get(testSensorType);
+    ConfigurationsUtils.writeSensorParserConfigToZookeeper(testSensorType, expectedSensorParserConfigBytes, zookeeperUrl);
+    byte[] actualSensorParserConfigBytes = ConfigurationsUtils.readSensorParserConfigBytesFromZookeeper(testSensorType, client);
+    Assert.assertTrue(Arrays.equals(expectedSensorParserConfigBytes, actualSensorParserConfigBytes));
+
+    Assert.assertTrue(expectedSensorEnrichmentConfigMap.size() > 0);
+    byte[] expectedSensorEnrichmentConfigBytes = expectedSensorEnrichmentConfigMap.get(testSensorType);
+    ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(testSensorType, expectedSensorEnrichmentConfigBytes, zookeeperUrl);
+    byte[] actualSensorEnrichmentConfigBytes = ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(testSensorType, client);
+    Assert.assertTrue(Arrays.equals(expectedSensorEnrichmentConfigBytes, actualSensorEnrichmentConfigBytes));
+
+    String name = "testConfig";
+    Map<String, Object> testConfig = new HashMap<>();
+    testConfig.put("stringField", "value");
+    testConfig.put("intField", 1);
+    testConfig.put("doubleField", 1.1);
+    ConfigurationsUtils.writeConfigToZookeeper(name, testConfig, zookeeperUrl);
+    byte[] readConfigBytes = ConfigurationsUtils.readConfigBytesFromZookeeper(name, client);
+    Assert.assertTrue(Arrays.equals(JSONUtils.INSTANCE.toJSONPretty(testConfig), readConfigBytes));
+
+  }
+
+  /**
+   * {
+   *   "foo" : "bar"
+   * }
+   */
+  @Multiline
+  private static String someConfig;
+
+  @Test
+  public void modifies_global_configuration() throws Exception {
+    ConfigurationType type = ConfigurationType.GLOBAL;
+    ConfigurationsUtils
+        .writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl);
+    byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl);
+    assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig)));
+  }
+
+  @Test
+  public void modifies_single_parser_configuration() throws Exception {
+    ConfigurationType type = ConfigurationType.PARSER;
+    String parserName = "a-happy-metron-parser";
+    ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName),
+        JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl);
+    byte[] actual = ConfigurationsUtils
+        .readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl);
+    assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(someConfig)));
+  }
+
+  /**
+   * [
+   *  {
+   *     "op": "replace",
+   *     "path": "/foo",
+   *     "value": "baz"
+   *  }
+   * ]
+   */
+  @Multiline
+  private static String patchSomeConfig;
+
+  /**
+   * {
+   *   "foo" : "baz"
+   * }
+   */
+  @Multiline
+  private static String modifiedSomeConfig;
+
+  /**
+   * Note: the current configuration structure mixes abstractions based on the configuration type
+   * and requires testing each type. GLOBAL is a actually representative of the final node name,
+   * whereas the other types, e.g. PARSER, represent a directory/path and not a ZK node where values
+   * are stored. The semantics are similar but slightly different.
+   */
+  @Test
+  public void patches_global_configuration_via_patch_json() throws Exception {
+    ConfigurationType type = ConfigurationType.GLOBAL;
+    String parserName = "patched-metron-global-config";
+    ConfigurationsUtils.writeConfigToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl);
+    ConfigurationsUtils.applyConfigPatchToZookeeper(type, JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl);
+    byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, zookeeperUrl);
+    assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig)));
+  }
+
+  @Test
+  public void patches_parser_configuration_via_patch_json() throws Exception {
+    ConfigurationType type = ConfigurationType.PARSER;
+    String parserName = "patched-metron-parser";
+    ConfigurationsUtils.writeConfigToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(someConfig), zookeeperUrl);
+    ConfigurationsUtils.applyConfigPatchToZookeeper(type, Optional.of(parserName), JSONUtils.INSTANCE.toJSONPretty(patchSomeConfig), zookeeperUrl);
+    byte[] actual = ConfigurationsUtils.readConfigBytesFromZookeeper(type, Optional.of(parserName), zookeeperUrl);
+    assertThat(actual, equalTo(JSONUtils.INSTANCE.toJSONPretty(modifiedSomeConfig)));
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    client.close();
+    testZkServer.close();
+    testZkServer.stop();
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
index c478de5..7f4846e 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,29 +15,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.metron.common.utils;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.test.utils.UnitTestHelper;
-import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
 public class JSONUtilsTest {
+
   private static File tmpDir;
 
   /**
-   {
-   "a" : "hello",
-   "b" : "world"
-   }
+   * { "a" : "hello", "b" : "world" }
    */
   @Multiline
   private static String config;
@@ -55,9 +55,10 @@ public class JSONUtilsTest {
       put("a", "hello");
       put("b", "world");
     }};
-    Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, new TypeReference<Map<String, Object>>() {
-    });
-    Assert.assertThat("config not equal", actual, equalTo(expected));
+    Map<String, Object> actual = JSONUtils.INSTANCE
+        .load(configFile, new TypeReference<Map<String, Object>>() {
+        });
+    assertThat("config not equal", actual, equalTo(expected));
   }
 
   @Test
@@ -67,18 +68,19 @@ public class JSONUtilsTest {
       put("b", "world");
     }};
     Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, Map.class);
-    Assert.assertThat("config not equal", actual, equalTo(expected));
+    assertThat("config not equal", actual, equalTo(expected));
   }
 
   @Test
   public void loads_file_with_custom_class() throws Exception {
     TestConfig expected = new TestConfig().setA("hello").setB("world");
     TestConfig actual = JSONUtils.INSTANCE.load(configFile, TestConfig.class);
-    Assert.assertThat("a not equal", actual.getA(), equalTo(expected.getA()));
-    Assert.assertThat("b not equal", actual.getB(), equalTo(expected.getB()));
+    assertThat("a not equal", actual.getA(), equalTo(expected.getA()));
+    assertThat("b not equal", actual.getB(), equalTo(expected.getB()));
   }
 
   public static class TestConfig {
+
     private String a;
     private String b;
 
@@ -100,4 +102,67 @@ public class JSONUtilsTest {
       return this;
     }
   }
+
+  /**
+   * { "a": "b" }
+   */
+  @Multiline
+  public static String sourceJson;
+
+  /**
+   * [{ "op": "move", "from": "/a", "path": "/c" }]
+   */
+  @Multiline
+  public static String patchJson;
+
+  /**
+   * { "c": "b" }
+   */
+  @Multiline
+  public static String expectedJson;
+
+  @Test
+  public void applyPatch_modifies_source_json_doc() throws IOException {
+    JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson);
+    JsonNode expected = JSONUtils.INSTANCE.readTree(expectedJson);
+    assertThat(actual, equalTo(expected));
+  }
+
+  /**
+   * {
+   *    "foo" : {
+   *      "bar" : {
+   *        "baz" : [ "val1", "val2" ]
+   *      }
+   *    }
+   * }
+   */
+  @Multiline
+  public static String complexJson;
+
+  /**
+   * [{ "op": "add", "path": "/foo/bar/baz", "value": [ "new1", "new2" ] }]
+   */
+  @Multiline
+  public static String patchComplexJson;
+
+  /**
+   * {
+   *    "foo" : {
+   *      "bar" : {
+   *        "baz" : [ "new1", "new2" ]
+   *      }
+   *    }
+   * }
+   */
+  @Multiline
+  public static String expectedComplexJson;
+
+  @Test
+  public void applyPatch_modifies_complex_source_json_doc() throws IOException {
+    JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson);
+    JsonNode expected = JSONUtils.INSTANCE.readTree(expectedComplexJson);
+    assertThat(actual, equalTo(expected));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
index f8b9b66..133f9c5 100755
--- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
@@ -42,15 +42,15 @@ threat.intel.join.cache.size={{threatintel_join_cache_size}}
 
 ##### Enrichment #####
 hbase.provider.impl={{enrichment_hbase_provider_impl}}
-enrichment.simple.hbase.table={{enrichment_table}}
-enrichment.simple.hbase.cf={{enrichment_cf}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
 enrichment.host.known_hosts={{enrichment_host_known_hosts}}
 
 ##### Threat Intel #####
-threat.intel.tracker.table={{threatintel_table}}
-threat.intel.tracker.cf={{threatintel_cf}}
-threat.intel.simple.hbase.table={{threatintel_table}}
-threat.intel.simple.hbase.cf={{threatintel_cf}}
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
 
 ##### Parallelism #####
 kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index f77f16e..c457e86 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -17,6 +17,13 @@
  */
 package org.apache.metron.enrichment.integration;
 
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULES_KEY;
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_COMMENT;
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_NAME;
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_REASON;
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_SCORE;
+import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_SCORE_KEY;
+
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
@@ -24,6 +31,18 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.TestConstants;
 import org.apache.metron.common.Constants;
@@ -36,8 +55,8 @@ import org.apache.metron.enrichment.integration.components.ConfigUploadComponent
 import org.apache.metron.enrichment.lookup.LookupKV;
 import org.apache.metron.enrichment.lookup.accesstracker.PersistentBloomTrackerCreator;
 import org.apache.metron.enrichment.stellar.SimpleHBaseEnrichmentFunctions;
-import org.apache.metron.hbase.mock.MockHTable;
 import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
 import org.apache.metron.integration.BaseIntegrationTest;
 import org.apache.metron.integration.ComponentRunner;
 import org.apache.metron.integration.ProcessorResult;
@@ -53,21 +72,6 @@ import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.*;
-
 public class EnrichmentIntegrationTest extends BaseIntegrationTest {
   private static final String ERROR_TOPIC = "enrichment_error";
   private static final String SRC_IP = "ip_src_addr";
@@ -129,15 +133,15 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
       setProperty("enrichment_join_cache_size", "1000");
       setProperty("threatintel_join_cache_size", "1000");
       setProperty("enrichment_hbase_provider_impl", "" + MockHBaseTableProvider.class.getName());
-      setProperty("enrichment_table", enrichmentsTableName);
-      setProperty("enrichment_cf", cf);
+      setProperty("enrichment_hbase_table", enrichmentsTableName);
+      setProperty("enrichment_hbase_cf", cf);
       setProperty("enrichment_host_known_hosts", "[{\"ip\":\"10.1.128.236\", \"local\":\"YES\", \"type\":\"webserver\", \"asset_value\" : \"important\"}," +
               "{\"ip\":\"10.1.128.237\", \"local\":\"UNKNOWN\", \"type\":\"unknown\", \"asset_value\" : \"important\"}," +
               "{\"ip\":\"10.60.10.254\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}," +
               "{\"ip\":\"10.0.2.15\", \"local\":\"YES\", \"type\":\"printer\", \"asset_value\" : \"important\"}]");
 
-      setProperty("threatintel_table", threatIntelTableName);
-      setProperty("threatintel_cf", cf);
+      setProperty("threatintel_hbase_table", threatIntelTableName);
+      setProperty("threatintel_hbase_cf", cf);
 
 
       setProperty("enrichment_kafka_spout_parallelism", "1");

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/pom.xml b/metron-platform/metron-indexing/pom.xml
index 7d07665..55fe455 100644
--- a/metron-platform/metron-indexing/pom.xml
+++ b/metron-platform/metron-indexing/pom.xml
@@ -74,25 +74,6 @@
             </exclusions>
             <scope>provided</scope>
         </dependency>        
-        <dependency>
-            <groupId>com.flipkart.zjsonpatch</groupId>
-            <artifactId>zjsonpatch</artifactId>
-            <version>0.3.1</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-annotations</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
 
         <dependency>
             <groupId>org.apache.storm</groupId>

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index c890544..32bfab9 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metron.common.configuration.writer.WriterConfiguration;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -125,7 +124,7 @@ public class HBaseDao implements IndexDao {
     Put put = new Put(update.getGuid().getBytes());
     long ts = update.getTimestamp() == null?System.currentTimeMillis():update.getTimestamp();
     byte[] columnQualifier = Bytes.toBytes(ts);
-    byte[] doc = JSONUtils.INSTANCE.toJSON(update.getDocument());
+    byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument());
     put.addColumn(cf, columnQualifier, doc);
     getTableInterface().put(put);
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 745dccd..be5d4fe 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -19,8 +19,12 @@ package org.apache.metron.indexing.dao;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.flipkart.zjsonpatch.JsonPatch;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.indexing.dao.search.GetRequest;
 import org.apache.metron.indexing.dao.search.GroupRequest;
 import org.apache.metron.indexing.dao.search.GroupResponse;
@@ -28,16 +32,9 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
 import org.apache.metron.indexing.dao.search.SearchResponse;
 import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
 import org.apache.metron.indexing.dao.update.PatchRequest;
 import org.apache.metron.indexing.dao.update.ReplaceRequest;
-import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
-
-import java.io.IOException;
-import org.apache.metron.indexing.dao.search.FieldType;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
 
 public interface IndexDao {
 
@@ -115,7 +112,7 @@ public interface IndexDao {
       }
     }
     JsonNode originalNode = JSONUtils.INSTANCE.convert(latest, JsonNode.class);
-    JsonNode patched = JsonPatch.apply(request.getPatch(), originalNode);
+    JsonNode patched = JSONUtils.INSTANCE.applyPatch(request.getPatch(), originalNode);
     Map<String, Object> updated = JSONUtils.INSTANCE.getMapper()
                                            .convertValue(patched, new TypeReference<Map<String, Object>>() {});
     Document d = new Document( updated

http://git-wip-us.apache.org/repos/asf/metron/blob/c18faaa9/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
index 67acb33..e67d3c9 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/utils/TestUtils.java
@@ -18,8 +18,15 @@
 package org.apache.metron.integration.utils;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -34,4 +41,79 @@ public class TestUtils {
     br.close();
     return ret;
   }
+
+  public static void write(File file, String[] contents) throws IOException {
+    StringBuilder b = new StringBuilder();
+    for (String line : contents) {
+      b.append(line);
+      b.append(System.lineSeparator());
+    }
+    write(file, b.toString());
+  }
+
+  /**
+   * Returns file passed in after writing
+   *
+   * @param file
+   * @param contents
+   * @return
+   * @throws IOException
+   */
+  public static File write(File file, String contents) throws IOException {
+    com.google.common.io.Files.createParentDirs(file);
+    com.google.common.io.Files.write(contents, file, StandardCharsets.UTF_8);
+    return file;
+  }
+
+  /**
+   * Cleans up after test run via runtime shutdown hooks
+   */
+  public static File createTempDir(String prefix) throws IOException {
+    final Path tmpDir = Files.createTempDirectory(prefix);
+    Runtime.getRuntime().addShutdownHook(new Thread() {
+      @Override
+      public void run() {
+        try {
+          cleanDir(tmpDir);
+        } catch (IOException e) {
+          System.out.println("Warning: Unable to clean tmp folder.");
+        }
+      }
+
+    });
+    return tmpDir.toFile();
+  }
+
+  public static void cleanDir(Path dir) throws IOException {
+    Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
+
+      @Override
+      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+        if (exc == null) {
+          return FileVisitResult.CONTINUE;
+        } else {
+          throw exc;
+        }
+      }
+    });
+  }
+
+  public static File createDir(File parent, String child) {
+    File newDir = new File(parent, child);
+    newDir.mkdirs();
+    return newDir;
+  }
+
 }


Mime
View raw message