metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sirs...@apache.org
Subject [46/85] [partial] incubator-metron git commit: Rename all OpenSOC files to Metron
Date Thu, 14 Jan 2016 17:03:41 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/pcap/PcapUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/pcap/PcapUtils.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/pcap/PcapUtils.java
new file mode 100644
index 0000000..8f9520f
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/pcap/PcapUtils.java
@@ -0,0 +1,421 @@
+package com.opensoc.pcap;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+
+/**
+ * The Class PcapUtils.
+ */
+public class PcapUtils {
+
+  /** The Constant SESSION_KEY_SEPERATOR. */
+  private static final char SESSION_KEY_SEPERATOR = '-';
+
+  /** The Constant protocolIdToNameMap. */
+  private static final BiMap<Integer, String> protocolIdToNameMap = HashBiMap
+      .create();
+
+  // private static final Map<Integer, String> protocolIdToNameMap = new
+  // HashMap();
+
+  static {
+
+    protocolIdToNameMap.put(0, "HOPOPT");
+    protocolIdToNameMap.put(1, "ICMP");
+    protocolIdToNameMap.put(2, "IGMP");
+    protocolIdToNameMap.put(3, "GGP");
+    protocolIdToNameMap.put(4, "IPV4");
+    protocolIdToNameMap.put(5, "ST");
+    protocolIdToNameMap.put(6, "TCP");
+    protocolIdToNameMap.put(7, "CBT");
+    protocolIdToNameMap.put(8, "EGP");
+    protocolIdToNameMap.put(9, "IGP");
+    protocolIdToNameMap.put(10, "BBN-RCC-MON");
+    protocolIdToNameMap.put(11, "NVP-II");
+    protocolIdToNameMap.put(12, "PUP");
+    protocolIdToNameMap.put(13, "ARGUS");
+    protocolIdToNameMap.put(14, "EMCON");
+    protocolIdToNameMap.put(15, "XNET");
+    protocolIdToNameMap.put(16, "CHAOS");
+    protocolIdToNameMap.put(17, "UDP");
+    protocolIdToNameMap.put(18, "MUX");
+    protocolIdToNameMap.put(19, "DCN-MEAS");
+    protocolIdToNameMap.put(20, "HMP");
+    protocolIdToNameMap.put(21, "PRM");
+    protocolIdToNameMap.put(22, "XNS-IDP");
+    protocolIdToNameMap.put(23, "TRUNK-1");
+    protocolIdToNameMap.put(24, "TRUNK-2");
+    protocolIdToNameMap.put(25, "LEAF-1");
+    protocolIdToNameMap.put(26, "LEAF-2");
+    protocolIdToNameMap.put(27, "RDP");
+    protocolIdToNameMap.put(28, "IRTP");
+    protocolIdToNameMap.put(29, "ISO-TP4");
+    protocolIdToNameMap.put(30, "NETBLT");
+    protocolIdToNameMap.put(31, "MFE-NSP");
+    protocolIdToNameMap.put(32, "MERIT-INP");
+    protocolIdToNameMap.put(33, "DCCP");
+    protocolIdToNameMap.put(34, "3PC");
+    protocolIdToNameMap.put(35, "IDPR");
+    protocolIdToNameMap.put(36, "XTP");
+    protocolIdToNameMap.put(37, "DDP");
+    protocolIdToNameMap.put(38, "IDPR-CMTP");
+    protocolIdToNameMap.put(39, "TP++");
+    protocolIdToNameMap.put(40, "IL");
+    protocolIdToNameMap.put(41, "IPV6");
+    protocolIdToNameMap.put(42, "SDRP");
+    protocolIdToNameMap.put(43, "IPV6-ROUTE");
+    protocolIdToNameMap.put(44, "IPV6-FRAG");
+    protocolIdToNameMap.put(45, "IDRP");
+    protocolIdToNameMap.put(46, "RSVP");
+    protocolIdToNameMap.put(47, "GRE");
+    protocolIdToNameMap.put(48, "DSR");
+    protocolIdToNameMap.put(49, "BNA");
+    protocolIdToNameMap.put(50, "ESP");
+    protocolIdToNameMap.put(51, "AH");
+    protocolIdToNameMap.put(52, "I-NLSP");
+    protocolIdToNameMap.put(53, "SWIPE");
+    protocolIdToNameMap.put(54, "NARP");
+    protocolIdToNameMap.put(55, "MOBILE");
+    protocolIdToNameMap.put(56, "TLSP");
+    protocolIdToNameMap.put(57, "SKIP");
+    protocolIdToNameMap.put(58, "IPV6-ICMP");
+    protocolIdToNameMap.put(59, "IPV6-NONXT");
+    protocolIdToNameMap.put(60, "IPV6-OPTS");
+    protocolIdToNameMap.put(62, "CFTP");
+    protocolIdToNameMap.put(64, "SAT-EXPAK");
+    protocolIdToNameMap.put(65, "KRYPTOLAN");
+    protocolIdToNameMap.put(66, "RVD");
+    protocolIdToNameMap.put(67, "IPPC");
+    protocolIdToNameMap.put(69, "SAT-MON");
+    protocolIdToNameMap.put(70, "VISA");
+    protocolIdToNameMap.put(71, "IPCV");
+    protocolIdToNameMap.put(72, "CPNX");
+    protocolIdToNameMap.put(73, "CPHB");
+    protocolIdToNameMap.put(74, "WSN");
+    protocolIdToNameMap.put(75, "PVP");
+    protocolIdToNameMap.put(76, "BR-SAT-MON");
+    protocolIdToNameMap.put(77, "SUN-ND");
+    protocolIdToNameMap.put(78, "WB-MON");
+    protocolIdToNameMap.put(79, "WB-EXPAK");
+    protocolIdToNameMap.put(80, "ISO-IP");
+    protocolIdToNameMap.put(81, "VMTP");
+    protocolIdToNameMap.put(82, "SECURE-VMTP");
+    protocolIdToNameMap.put(83, "VINES");
+    protocolIdToNameMap.put(84, "TTP");
+    protocolIdToNameMap.put(85, "NSFNET-IGP");
+    protocolIdToNameMap.put(86, "DGP");
+    protocolIdToNameMap.put(87, "TCF");
+    protocolIdToNameMap.put(88, "EIGRP");
+    protocolIdToNameMap.put(89, "OSPFIGP");
+    protocolIdToNameMap.put(90, "SPRITE-RPC");
+    protocolIdToNameMap.put(91, "LARP");
+    protocolIdToNameMap.put(92, "MTP");
+    protocolIdToNameMap.put(93, "AX.25");
+    protocolIdToNameMap.put(94, "IPIP");
+    protocolIdToNameMap.put(95, "MICP");
+    protocolIdToNameMap.put(96, "SCC-SP");
+    protocolIdToNameMap.put(97, "ETHERIP");
+    protocolIdToNameMap.put(98, "ENCAP");
+    protocolIdToNameMap.put(100, "GMTP");
+    protocolIdToNameMap.put(101, "IFMP");
+    protocolIdToNameMap.put(102, "PNNI");
+    protocolIdToNameMap.put(103, "PIM");
+    protocolIdToNameMap.put(104, "ARIS");
+    protocolIdToNameMap.put(105, "SCPS");
+    protocolIdToNameMap.put(106, "QNX");
+    protocolIdToNameMap.put(107, "A/N");
+    protocolIdToNameMap.put(108, "IPCOMP");
+    protocolIdToNameMap.put(109, "SNP");
+    protocolIdToNameMap.put(110, "COMPAQ-PEER");
+    protocolIdToNameMap.put(111, "IPX-IN-IP");
+    protocolIdToNameMap.put(112, "VRRP");
+    protocolIdToNameMap.put(113, "PGM");
+    protocolIdToNameMap.put(115, "L2TP");
+    protocolIdToNameMap.put(116, "DDX");
+    protocolIdToNameMap.put(117, "IATP");
+    protocolIdToNameMap.put(118, "STP");
+    protocolIdToNameMap.put(119, "SRP");
+    protocolIdToNameMap.put(120, "UTI");
+    protocolIdToNameMap.put(121, "SMP");
+    protocolIdToNameMap.put(122, "SM");
+    protocolIdToNameMap.put(123, "PTP");
+    protocolIdToNameMap.put(124, "ISIS OVER IPV4");
+    protocolIdToNameMap.put(125, "FIRE");
+    protocolIdToNameMap.put(126, "CRTP");
+    protocolIdToNameMap.put(127, "CRUDP");
+    protocolIdToNameMap.put(128, "SSCOPMCE");
+    protocolIdToNameMap.put(129, "IPLT");
+    protocolIdToNameMap.put(130, "SPS");
+    protocolIdToNameMap.put(131, "PIPE");
+    protocolIdToNameMap.put(132, "SCTP");
+    protocolIdToNameMap.put(133, "FC");
+    protocolIdToNameMap.put(134, "RSVP-E2E-IGNORE");
+    protocolIdToNameMap.put(135, "MOBILITY HEADER");
+    protocolIdToNameMap.put(136, "UDPLITE");
+    protocolIdToNameMap.put(137, "MPLS-IN-IP");
+    protocolIdToNameMap.put(138, "MANET");
+    protocolIdToNameMap.put(139, "HIP");
+    protocolIdToNameMap.put(140, "SHIM6");
+    protocolIdToNameMap.put(141, "WESP");
+    protocolIdToNameMap.put(142, "ROHC");
+  }
+
+  /** The Constant protocolNameToIdMap. */
+  private static final BiMap<String, Integer> protocolNameToIdMap = protocolIdToNameMap
+      .inverse();
+
+  // private static final Map<String, Integer> protocolNameToIdMap =
+  // invertMap(protocolIdToNameMap);
+
+  /**
+   * Convert ipv4 ip to hex.
+   * 
+   * @param ipAddress
+   *          the ip address
+   * @return the string
+   */
+  public static String convertIpv4IpToHex(String ipAddress) {
+    StringBuffer hexIp = new StringBuffer(64);
+    String[] ipSegments = ipAddress.split("\\.");
+
+    for (String ipSegment : ipSegments) {
+      hexIp.append(convertIpSegmentToHex(ipSegment));
+    }
+
+    return hexIp.toString();
+
+  }
+
+  /**
+   * Gets the session key.
+   * 
+   * @param srcIp
+   *          the src ip
+   * @param dstIp
+   *          the dst ip
+   * @param protocol
+   *          the protocol
+   * @param srcPort
+   *          the src port
+   * @param dstPort
+   *          the dst port
+   * @return the session key
+   */
+  public static String getSessionKey(String srcIp, String dstIp,
+      String protocol, String srcPort, String dstPort) {
+    return getSessionKey(srcIp, dstIp, protocol, srcPort, dstPort, null, null);
+  }
+
+  /**
+   * Gets the session key.
+   * 
+   * @param srcIp
+   *          the src ip
+   * @param dstIp
+   *          the dst ip
+   * @param protocol
+   *          the protocol
+   * @param srcPort
+   *          the src port
+   * @param dstPort
+   *          the dst port
+   * @param ipId
+   *          the ip id
+   * @param fragmentOffset
+   *          the fragment offset
+   * @return the session key
+   */
+  public static String getSessionKey(String srcIp, String dstIp,
+      String protocol, String srcPort, String dstPort, String ipId,
+      String fragmentOffset) {
+
+    StringBuffer sb = new StringBuffer(40);
+    sb.append(convertIpv4IpToHex(srcIp)).append(SESSION_KEY_SEPERATOR)
+        .append(convertIpv4IpToHex(dstIp)).append(SESSION_KEY_SEPERATOR)
+        .append(protocol == null ? "0" : protocol)
+        .append(SESSION_KEY_SEPERATOR).append(srcPort == null ? "0" : srcPort)
+        .append(SESSION_KEY_SEPERATOR).append(dstPort == null ? "0" : dstPort)
+        .append(SESSION_KEY_SEPERATOR).append(ipId == null ? "0" : ipId)
+        .append(SESSION_KEY_SEPERATOR)
+        .append(fragmentOffset == null ? "0" : fragmentOffset);
+
+    return sb.toString();
+  }
+
+  /**
+   * Gets the session key.
+   * 
+   * @param srcIp
+   *          the src ip
+   * @param dstIp
+   *          the dst ip
+   * @param protocol
+   *          the protocol
+   * @param srcPort
+   *          the src port
+   * @param dstPort
+   *          the dst port
+   * @param ipId
+   *          the ip id
+   * @param fragmentOffset
+   *          the fragment offset
+   * @return the session key
+   */
+  public static String getSessionKey(String srcIp, String dstIp, int protocol,
+      int srcPort, int dstPort, int ipId, int fragmentOffset) {
+    String keySeperator = "-";
+    StringBuffer sb = new StringBuffer(40);
+    sb.append(convertIpv4IpToHex(srcIp)).append(keySeperator)
+        .append(convertIpv4IpToHex(dstIp)).append(keySeperator)
+        .append(protocol).append(keySeperator).append(srcPort)
+        .append(keySeperator).append(dstPort).append(keySeperator).append(ipId)
+        .append(keySeperator).append(fragmentOffset);
+
+    return sb.toString();
+  }
+
+  /**
+   * Gets the short session key. (5-tuple only)
+   * 
+   * @param srcIp
+   *          the src ip
+   * @param dstIp
+   *          the dst ip
+   * @param protocol
+   *          the protocol
+   * @param srcPort
+   *          the src port
+   * @param dstPort
+   *          the dst port
+   * @return the session key
+   */
+  public static String getShortSessionKey(String srcIp, String dstIp, int protocol,
+      int srcPort, int dstPort) {
+    String keySeperator = "-";
+    StringBuffer sb = new StringBuffer(40);
+    sb.append(convertIpv4IpToHex(srcIp)).append(keySeperator)
+        .append(convertIpv4IpToHex(dstIp)).append(keySeperator)
+        .append(protocol).append(keySeperator).append(srcPort)
+        .append(keySeperator).append(dstPort);
+
+    return sb.toString();
+  }
+  
+  // public static String convertPortToHex(String portNumber) {
+  // return convertPortToHex(Integer.valueOf(portNumber));
+  //
+  // }
+  //
+  // public static String convertPortToHex(int portNumber) {
+  // return convertToHex(portNumber, 4);
+  //
+  // }
+  //
+  // public static String convertProtocolToHex(String protocol) {
+  // return convertProtocolToHex(Integer.valueOf(protocol));
+  //
+  // }
+  //
+  // public static String convertProtocolToHex(int protocol) {
+  // return convertToHex(protocol, 2);
+  // }
+
+  /**
+   * Convert ip segment to hex.
+   * 
+   * @param ipSegment
+   *          the ip segment
+   * @return the string
+   */
+  public static String convertIpSegmentToHex(String ipSegment) {
+    return convertIpSegmentToHex(Integer.valueOf(ipSegment));
+
+  }
+
+  /**
+   * Convert ip segment to hex.
+   * 
+   * @param ipSegment
+   *          the ip segment
+   * @return the string
+   */
+  public static String convertIpSegmentToHex(int ipSegment) {
+    return convertToHex(ipSegment, 2);
+
+  }
+
+  /**
+   * Convert to hex.
+   * 
+   * @param number
+   *          the number
+   * @param length
+   *          the length
+   * @return the string
+   */
+  public static String convertToHex(int number, int length) {
+    return StringUtils.leftPad(Integer.toHexString(number), length, '0');
+
+  }
+
+  /**
+   * Gets the protocol name.
+   * 
+   * @param protocolNumber
+   *          the protocol number
+   * 
+   * @return the protocol name
+   */
+  public static String getProtocolNameFromId(int protocolNumber) {
+    String protocolName = protocolIdToNameMap.get(protocolNumber);
+
+    if (protocolName == null) {
+      protocolName = String.valueOf(protocolNumber);
+    }
+    return protocolName;
+  }
+
+  /**
+   * Gets the protocol id from name.
+   * 
+   * @param protocolName
+   *          the protocol name
+   * @return the protocol id from name
+   */
+  public static int getProtocolIdFromName(String protocolName) {
+    Integer protocolNumber = protocolNameToIdMap
+        .get(protocolName.toUpperCase());
+
+    if (protocolNumber == null) {
+      protocolNumber = -1;
+    }
+    return protocolNumber;
+  }
+
+  /**
+   * Invert map.
+   * 
+   * @param <V>
+   *          the value type
+   * @param <K>
+   *          the key type
+   * @param map
+   *          the map
+   * @return the map
+   */
+  private static <V, K> Map<V, K> invertMap(Map<K, V> map) {
+
+    Map<V, K> inv = new HashMap<V, K>();
+
+    for (Entry<K, V> entry : map.entrySet())
+      inv.put(entry.getValue(), entry.getKey());
+
+    return inv;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractConfigTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractConfigTest.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractConfigTest.java
new file mode 100644
index 0000000..7484e16
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractConfigTest.java
@@ -0,0 +1,299 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.opensoc.helpers.topology.SettingsLoader;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractConfigTest</code> is
+ * an abstract base class for implementing JUnit tests that need to use
+ * config to connect to ZooKeeper and HBase. The <code>setup</code> method will
attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Oct 10, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractConfigTest  extends AbstractTestContext{
+         /**
+         * The configPath.
+         */
+        protected String configPath=null;   
+        
+        /**
+        * The configName.
+        */
+       protected String configName=null;           
+
+        /**
+         * The config.
+         */
+        private Configuration config=null;
+        
+         /**
+         * The settings.
+         */
+        Map<String, String> settings=null;       
+
+        /**
+         * The schemaJsonString.
+         */
+        private String schemaJsonString = null;
+        /**
+         * Any Object for mavenMode
+         * @parameter
+         *   expression="${mode}"
+         *   default-value="local"
+         */
+         private Object mode="local";        
+
+        /**
+         * Constructs a new <code>AbstractConfigTest</code> instance.
+         * @throws Exception 
+         */
+        public AbstractConfigTest() throws Exception {
+            super.setUp();
+        }
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @param name the name of the test case.
+         */
+        public AbstractConfigTest(String name) {
+            super(name);
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#setUp()
+         */
+        protected void setUp(String configName) throws Exception {
+            super.setUp();
+            this.setConfigPath("src/test/resources/config/"+getClass().getSimpleName()+".config");
+            try {
+                this.setConfig(new PropertiesConfiguration(this.getConfigPath()));
+               
+                Map configOptions= SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config,
configName+"=");
+                this.setSettings(SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config,
configName + "."));
+                this.getSettings().put(configName, (String) configOptions.get(configName));
+            } catch (ConfigurationException e) {
+                fail("Config not found !!"+e);
+                e.printStackTrace();
+            }               
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#tearDown()
+         */
+        @Override
+        protected void tearDown() throws Exception {
+
+        }
+
+        
+         /**
+         * validateJsonData
+         * @param jsonSchema
+         * @param jsonData
+         * @return
+         * @throws Exception
+         */
+         
+        protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+            throws Exception {
+    
+            final JsonNode d = JsonLoader.fromString(jsonData);
+            final JsonNode s = JsonLoader.fromString(jsonSchema);
+    
+            final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+            JsonValidator v = factory.getValidator();
+    
+            ProcessingReport report = v.validate(s, d);
+            System.out.println(report);
+            
+            return report.toString().contains("success");
+        }
+        
+        protected String readSchemaFromFile(URL schema_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    schema_url.getFile()));
+            String line;
+            StringBuilder sb = new StringBuilder();
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                sb.append(line);
+            }
+            br.close();
+
+            String schema_string = sb.toString().replaceAll("\n", "");
+            schema_string = schema_string.replaceAll(" ", "");
+
+            System.out.println("Read in schema: " + schema_string);
+
+            return schema_string;
+        }        
+  
+        protected String[] readTestDataFromFile(String test_data_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    new File(test_data_url)));
+            ArrayList<String> inputDataLines = new ArrayList<String>();
+           
+            String line;
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                inputDataLines.add(line.toString().replaceAll("\n", ""));
+            }
+            br.close();
+            String[] inputData = new String[inputDataLines.size()];
+            inputData = inputDataLines.toArray(inputData);
+
+            return inputData;
+        }          
+       /**
+        * Skip Tests
+        */
+       public boolean skipTests(Object mode){
+           if(mode.toString().equals("local")){
+               return true;
+           }else {
+               return false;
+           }
+       }
+       
+       /**
+        * Returns the mode.
+        * @return the mode.
+        */
+       
+       public Object getMode() {
+           return mode;
+       }
+
+       /**
+        * Sets the mode.
+        * @param mode the mode.
+        */
+       
+       public void setMode(Object mode) {
+       
+           this.mode = mode;
+       }
+
+    
+         /**
+         * @param readSchemaFromFile
+         */
+        public void setSchemaJsonString(String schemaJsonString) {
+            this.schemaJsonString=schemaJsonString;
+        }
+
+    
+         /**
+         * @return
+         */
+        public String getSchemaJsonString() {
+           return this.schemaJsonString;
+        }
+        
+        /**
+        * Returns the configPath.
+        * @return the configPath.
+        */
+       public String getConfigPath() {
+           return configPath;
+       }
+    
+       /**
+        * Sets the configPath.
+        * @param configPath the configPath.
+        */
+       public void setConfigPath(String configPath) {
+           this.configPath = configPath;
+       }    
+       /**
+        * Returns the config.
+        * @return the config.
+        */
+       
+       public Configuration getConfig() {
+           return config;
+       }
+    
+       /**
+        * Sets the config.
+        * @param config the config.
+        */
+       
+       public void setConfig(Configuration config) {
+       
+           this.config = config;
+       }  
+       /**
+        * Returns the settings.
+        * @return the settings.
+        */
+       
+       public Map<String, String> getSettings() {
+           return settings;
+       }
+
+       /**
+        * Sets the settings.
+        * @param settings the settings.
+        */
+       
+       public void setSettings(Map<String, String> settings) {
+           this.settings = settings;
+       }   
+       /**
+       * Returns the configName.
+       * @return the configName.
+       */
+      public String getConfigName() {
+          return configName;
+      }
+
+      /**
+       * Sets the configName.
+       * @param configName the configName.
+       */
+      public void setConfigName(String configName) {  
+          this.configName = configName;
+      }       
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractSchemaTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractSchemaTest.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractSchemaTest.java
new file mode 100644
index 0000000..670d7f9
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractSchemaTest.java
@@ -0,0 +1,198 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.test;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.net.URL;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractSchemaTest</code> is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * Json Schema. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Aug 7, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractSchemaTest  extends AbstractConfigTest{
+        
+        
+         /**
+         * The schemaJsonString.
+         */
+        private String schemaJsonString = null;
+        /**
+         * Any Object for mavenMode
+         * @parameter
+         *   expression="${mode}"
+         *   default-value="local"
+         */
+         private Object mode="local";        
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @throws Exception 
+         */
+        public AbstractSchemaTest() throws Exception {
+            super.setUp();
+        }
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @param name the name of the test case.
+         */
+        public AbstractSchemaTest(String name) {
+            super(name);
+            try{
+                if(System.getProperty("mode")!=null){
+                    setMode(System.getProperty("mode") );                
+                }else
+                {
+                    setMode("local");
+                }
+            }catch(Exception ex){
+                setMode("local");
+            }            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#setUp()
+         */
+        @Override
+        protected void setUp() throws Exception {
+            super.setUp();
+            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#tearDown()
+         */
+        @Override
+        protected void tearDown() throws Exception {
+
+        }
+
+        
+         /**
+         * validateJsonData
+         * @param jsonSchema
+         * @param jsonData
+         * @return
+         * @throws Exception
+         */
+         
+        protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+            throws Exception {
+    
+            final JsonNode d = JsonLoader.fromString(jsonData);
+            final JsonNode s = JsonLoader.fromString(jsonSchema);
+    
+            final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+            JsonValidator v = factory.getValidator();
+    
+            ProcessingReport report = v.validate(s, d);
+            System.out.println(report);
+            
+            return report.toString().contains("success");
+        }
+        
+        protected String readSchemaFromFile(URL schema_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    schema_url.getFile()));
+            String line;
+            StringBuilder sb = new StringBuilder();
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                sb.append(line);
+            }
+            br.close();
+
+            String schema_string = sb.toString().replaceAll("\n", "");
+            schema_string = schema_string.replaceAll(" ", "");
+
+            System.out.println("Read in schema: " + schema_string);
+
+            return schema_string;
+
+        }        
+        
+       /**
+        * Skip Tests
+        */
+       public boolean skipTests(Object mode){
+           if(mode.toString().equals("local")){
+               return true;
+           }else {
+               return false;
+           }
+       }
+       
+       /**
+        * Returns the mode.
+        * @return the mode.
+        */
+       
+       public Object getMode() {
+           return mode;
+       }
+
+       /**
+        * Sets the mode.
+        * @param mode the mode.
+        */
+       
+       public void setMode(Object mode) {
+       
+           this.mode = mode;
+       }
+
+    
+     /**
+     
+     * @param readSchemaFromFile
+     */
+     
+    public void setSchemaJsonString(String schemaJsonString) {
+        this.schemaJsonString=schemaJsonString;
+    }
+
+    
+     /**
+     
+     * @return
+     */
+     
+    public String getSchemaJsonString() {
+       return this.schemaJsonString;
+    }
+     
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractTestContext.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractTestContext.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractTestContext.java
new file mode 100644
index 0000000..ea5b04f
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/test/AbstractTestContext.java
@@ -0,0 +1,189 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.test;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractTestContext</code> is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * test properties. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Aug 7, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractTestContext  extends TestCase{
+         /**
+         * The testProps.
+         */
+        protected File testPropFile=null;
+
+        /**
+         * The properties loaded for test.
+         */
+        protected Properties testProperties=new Properties();
+        
+        /**
+         * Any Object for mavenMode
+         * @parameter
+         *   expression="${mode}"
+         *   default-value="global"
+         */
+         private Object mode="local";        
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         */
+        public AbstractTestContext() {
+            super();
+        }
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @param name the name of the test case.
+         */
+        public AbstractTestContext(String name) {
+            super(name);
+            try{
+                if(System.getProperty("mode")!=null){
+                    setMode(System.getProperty("mode") );                
+                }else
+                {
+                    setMode("local");
+                }
+            }catch(Exception ex){
+                setMode("local");
+            }            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#setUp()
+         */
+        @Override
+        protected void setUp() throws Exception {
+            InputStream input=null;
+            File directory = new File("src/test/resources");
+            if (!directory.isDirectory()) {
+                return;
+            }
+            File file = new File(directory, getClass().getSimpleName() + ".properties");
+            if (!file.canRead()) {
+                return;
+            }
+            setTestPropFile(file);
+            try{
+                input=new FileInputStream(file);
+                testProperties.load(input);
+            }catch(IOException ex){
+                ex.printStackTrace();
+                fail("failed to load properties");
+            }
+            
+            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#tearDown()
+         */
+        @Override
+        protected void tearDown() throws Exception {
+
+        }
+
+        /**
+         * Returns the testProperties.
+         * @return the testProperties.
+         */
+        
+        public Properties getTestProperties() {
+            return testProperties;
+        }
+
+        /**
+         * Sets the testProperties.
+         * @param testProperties the testProperties.
+         */
+        
+        public void setTestProperties(Properties testProperties) {
+        
+            this.testProperties = testProperties;
+        }    
+        /**
+        * Returns the testPropFile.
+        * @return the testPropFile.
+        */
+       
+       public File getTestPropFile() {
+           return testPropFile;
+       }
+
+       /**
+        * Sets the testPropFile.
+        * @param testPropFile the testPropFile.
+        */
+       
+       public void setTestPropFile(File testPropFile) {
+       
+           this.testPropFile = testPropFile;
+       }     
+       
+       /**
+        * Skip Tests
+        */
+       public boolean skipTests(Object mode){
+           if(mode.toString().equals("local")){
+               return true;
+           }else {
+               return false;
+           }
+       }
+       
+       /**
+        * Returns the mode.
+        * @return the mode.
+        */
+       
+       public Object getMode() {
+           return mode;
+       }
+
+       /**
+        * Sets the mode.
+        * @param mode the mode.
+        */
+       
+       public void setMode(Object mode) {
+       
+           this.mode = mode;
+       }
+     
+    }
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/BasicTldExtractor.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/BasicTldExtractor.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/BasicTldExtractor.java
new file mode 100644
index 0000000..2dbcd95
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/BasicTldExtractor.java
@@ -0,0 +1,137 @@
+package com.opensoc.tldextractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class BasicTldExtractor implements Serializable {
+	private static final long serialVersionUID = -7440226111118873815L;
+	private StringBuilder sb = new StringBuilder();
+
+    private Pattern pattern;
+    
+    /**
+    * The inputFile.
+    */
+   private String inputFile ="effective_tld_names.dat";
+   
+   public BasicTldExtractor(String filePath) {
+       this.inputFile=filePath;
+       this.init();
+   }
+   
+	public BasicTldExtractor() {
+      this.init();
+	}
+
+	private void init(){
+	       try {
+	            ArrayList<String> terms = new ArrayList<String>();
+
+	            
+	            BufferedReader br = new BufferedReader(new InputStreamReader(
+	                    getClass().getClassLoader().getResourceAsStream(inputFile)));
+	            String s = null;
+	            while ((s = br.readLine()) != null) {
+	                s = s.trim();
+	                if (s.length() == 0 || s.startsWith("//") || s.startsWith("!"))
+	                    continue;
+	                terms.add(s);
+	            }
+	            Collections.sort(terms, new StringLengthComparator());
+	            for (String t : terms)
+	                add(t);
+	            compile();
+	            br.close();
+	        } catch (IOException e) {
+	            throw new IllegalStateException(e);
+	        }
+	}
+	protected void add(String s) {
+		s = s.replace(".", "\\.");
+		s = "\\." + s;
+		if (s.startsWith("*")) {
+			s = s.replace("*", ".+");
+			sb.append(s).append("|");
+		} else {
+			sb.append(s).append("|");
+		}
+	}
+
+	public void compile() {
+		if (sb.length() > 0)
+			sb.deleteCharAt(sb.length() - 1);
+		sb.insert(0, "[^.]+?(");
+		sb.append(")$");
+		pattern = Pattern.compile(sb.toString());
+		sb = null;
+	}
+
+	public String extract2LD(String host) {
+		Matcher m = pattern.matcher(host);
+		if (m.find()) {
+			return m.group(0);
+		}
+		return null;
+	}
+
+	public String extractTLD(String host) {
+		Matcher m = pattern.matcher(host);
+		if (m.find()) {
+			return m.group(1);
+		}
+		return null;
+	}
+
+	public static class StringLengthComparator implements Comparator<String> {
+		public int compare(String s1, String s2) {
+			if (s1.length() > s2.length())
+				return -1;
+			if (s1.length() < s2.length())
+				return 1;
+			return 0;
+		}
+	}
+    /**
+     * Returns the sb.
+     * @return the sb.
+     */
+    
+    public StringBuilder getSb() {
+        return sb;
+    }
+
+    /**
+     * Sets the sb.
+     * @param sb the sb.
+     */
+    
+    public void setSb(StringBuilder sb) {
+    
+        this.sb = sb;
+    }
+    /**
+     * Returns the inputFile.
+     * @return the inputFile.
+     */
+    
+    public String getInputFile() {
+        return inputFile;
+    }
+
+    /**
+     * Sets the inputFile.
+     * @param inputFile the inputFile.
+     */
+    
+    public void setInputFile(String inputFile) {
+    
+        this.inputFile = inputFile;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/test/BasicTldExtractorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/test/BasicTldExtractorTest.java
b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/test/BasicTldExtractorTest.java
new file mode 100644
index 0000000..03cc065
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/main/java/com/apache/metron/tldextractor/test/BasicTldExtractorTest.java
@@ -0,0 +1,125 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.tldextractor.test;
+
+import com.opensoc.test.AbstractConfigTest;
+import com.opensoc.tldextractor.BasicTldExtractor;
+
+
+ /**
+ * <ul>
+ * <li>Title: Basic TLD Extractor Test</li>
+ * <li>Description: Basic TLD Extractor class test</li>
+ * <li>Created: Feb 26, 2015</li>
+ * </ul>
+ * @author $Author:  $
+ * @version $Revision: 1.1 $
+ */
+public class BasicTldExtractorTest extends AbstractConfigTest {
+     /**
+     * The tldExtractor.
+     */
+     
+    private BasicTldExtractor tldExtractor=null;
+
+    /**
+     * Constructs a new <code>BasicTldExtractorTest</code> instance.
+     * @param name
+     */
+
+    public BasicTldExtractorTest(String name) {
+        super(name);
+    }
+
+    /**
+     
+     * @throws java.lang.Exception
+     */
+    protected static void setUpBeforeClass() throws Exception {
+    }
+
+    /**
+     
+     * @throws java.lang.Exception
+     */
+    protected static void tearDownAfterClass() throws Exception {
+    }
+
+    /* 
+     * (non-Javadoc)
+     * @see junit.framework.TestCase#setUp()
+     */
+
+    protected void setUp() throws Exception {
+        super.setUp("com.opensoc.tldextractor.test.BasicTldExtractorTest");
+        this.tldExtractor=new BasicTldExtractor(this.getConfig().getString("logFile"));
+    } 
+
+    /* 
+     * (non-Javadoc)
+     * @see junit.framework.TestCase#tearDown()
+     */
+
+    protected void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#BasicTldExtractor()}.
+     */
+    public void testBasicTldExtractor() {
+        assertNotNull(this.tldExtractor);
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extract2LD(java.lang.String)}.
+     */
+    public void testExtract2LD() {
+        //fail("Not yet implemented");
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extractTLD(java.lang.String)}.
+     */
+    public void testExtractTLD() 
+    {
+        String result = this.tldExtractor.extractTLD("cisco.com");
+        System.out.println("result ="+result);
+    }
+    /**
+     * Returns the tldExtractor.
+     * @return the tldExtractor.
+     */
+    
+    public BasicTldExtractor getTldExtractor() {
+        return tldExtractor;
+    }
+
+    /**
+     * Sets the tldExtractor.
+     * @param tldExtractor the tldExtractor.
+     */
+    
+    public void setTldExtractor(BasicTldExtractor tldExtractor) {
+    
+        this.tldExtractor = tldExtractor;
+    }
+    
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Common/src/test/resources/config/BasicTldExtractorTest.config
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/test/resources/config/BasicTldExtractorTest.config
b/metron-streaming/Metron-Common/src/test/resources/config/BasicTldExtractorTest.config
new file mode 100644
index 0000000..6b3dc05
--- /dev/null
+++ b/metron-streaming/Metron-Common/src/test/resources/config/BasicTldExtractorTest.config
@@ -0,0 +1,2 @@
+#BasicTldExtractorConfig
+logFile=effective_tld_names.dat



Mime
View raw message