metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [42/89] [abbrv] [partial] incubator-metron git commit: Rename all OpenSOC files to Metron
Date Tue, 26 Jan 2016 14:18:20 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapHelper.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapHelper.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapHelper.java
new file mode 100644
index 0000000..5224945
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapHelper.java
@@ -0,0 +1,205 @@
+package com.opensoc.pcapservice;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.mortbay.log.Log;
+import org.springframework.util.Assert;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * utility class which holds methods related to time conversions, building
+ * reverse keys.
+ */
+public class PcapHelper {
+
+  /** The Constant LOGGER. */
+  private static final Logger LOGGER = Logger.getLogger(PcapHelper.class);
+
+  /** The cell timestamp comparator. */
+  private static CellTimestampComparator CELL_TIMESTAMP_COMPARATOR = new CellTimestampComparator();
+
+  /**
+   * The Enum TimeUnit.
+   */
+  public enum TimeUnit {
+
+    /** The seconds. */
+    SECONDS,
+    /** The millis. */
+    MILLIS,
+    /** The micros. */
+    MICROS,
+    /** The unknown. */
+    UNKNOWN
+  };
+
+  /**
+   * Converts the given time to the 'hbase' data creation time unit.
+   * 
+   * @param inputTime
+   *          the input time
+   * @return the long
+   */
+  public static long convertToDataCreationTimeUnit(long inputTime) {
+    if (inputTime <= 9999999999L) {
+      return convertSecondsToDataCreationTimeUnit(inputTime); // input time unit
+                                                              // is in seconds
+    } else if (inputTime <= 9999999999999L) {
+      return convertMillisToDataCreationTimeUnit(inputTime); // input time unit
+                                                             // is in millis
+    } else if (inputTime <= 9999999999999999L) {
+      return convertMicrosToDataCreationTimeUnit(inputTime); // input time unit
+                                                             // it in micros
+    }
+    return inputTime; // input time unit is unknown
+  }
+
+  /**
+   * Returns the 'hbase' data creation time unit by reading
+   * 'hbase.table.data.time.unit' property in 'hbase-config' properties file; If
+   * none is mentioned in properties file, returns <code>TimeUnit.UNKNOWN</code>
+   * 
+   * @return TimeUnit
+   */
+  @VisibleForTesting
+  public static TimeUnit getDataCreationTimeUnit() {
+    String timeUnit = ConfigurationUtil.getConfiguration().getString(
+        "hbase.table.data.time.unit");
+    LOGGER.debug("hbase.table.data.time.unit=" + timeUnit.toString());
+    if (StringUtils.isNotEmpty(timeUnit)) {
+      return TimeUnit.valueOf(timeUnit);
+    }
+    return TimeUnit.UNKNOWN;
+  }
+
+  /**
+   * Convert seconds to data creation time unit.
+   * 
+   * @param inputTime
+   *          the input time
+   * @return the long
+   */
+  @VisibleForTesting
+  public static long convertSecondsToDataCreationTimeUnit(long inputTime) {
+    System.out.println("convert Seconds To DataCreation TimeUnit");
+    TimeUnit dataCreationTimeUnit = getDataCreationTimeUnit();
+    if (TimeUnit.SECONDS == dataCreationTimeUnit) {
+      return inputTime;
+    } else if (TimeUnit.MILLIS == dataCreationTimeUnit) {
+      return inputTime * 1000;
+    } else if (TimeUnit.MICROS == dataCreationTimeUnit) {
+      return inputTime * 1000 * 1000;
+    }
+    return inputTime;
+  }
+
+  /**
+   * Builds the reverseKey to fetch the pcaps in the reverse traffic
+   * (destination to source).
+   * 
+   * @param key
+   *          indicates hbase rowKey (partial or full) in the format
+   *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
+   * @return String indicates the key in the format
+   *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
+   */
+  public static String reverseKey(String key) {
+    Assert.hasText(key, "key must not be null or empty");
+    String delimeter = HBaseConfigConstants.PCAP_KEY_DELIMETER;
+    String regex = "\\" + delimeter;
+    StringBuffer sb = new StringBuffer();
+    try {
+      String[] tokens = key.split(regex);
+      Assert
+          .isTrue(
+              (tokens.length == 5 || tokens.length == 6 || tokens.length == 7),
+              "key is not in the format : 'srcAddr-dstAddr-protocol-srcPort-dstPort-{ipId-fragment identifier}'");
+      sb.append(tokens[1]).append(delimeter).append(tokens[0])
+          .append(delimeter).append(tokens[2]).append(delimeter)
+          .append(tokens[4]).append(delimeter).append(tokens[3]);
+    } catch (Exception e) {
+      Log.warn("Failed to reverse the key. Reverse scan won't be performed.", e);
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Builds the reverseKeys to fetch the pcaps in the reverse traffic
+   * (destination to source). If all keys in the input are not in the expected
+   * format, it returns an empty list;
+   * 
+   * @param keys
+   *          indicates list of hbase rowKeys (partial or full) in the format
+   *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
+   * @return List<String> indicates the list of keys in the format
+   *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
+   */
+  public static List<String> reverseKey(List<String> keys) {
+    Assert.notEmpty(keys, "'keys' must not be null or empty");
+    List<String> reverseKeys = new ArrayList<String>();
+    for (String key : keys) {
+      if (key != null) {
+        String reverseKey = reverseKey(key);
+        if (StringUtils.isNotEmpty(reverseKey)) {
+          reverseKeys.add(reverseKey);
+        }
+      }
+    }
+    return reverseKeys;
+  }
+
+  /**
+   * Returns Comparator for sorting pcaps cells based on the timestamp (dsc).
+   * 
+   * @return CellTimestampComparator
+   */
+  public static CellTimestampComparator getCellTimestampComparator() {
+    return CELL_TIMESTAMP_COMPARATOR;
+  }
+
+  /**
+   * Convert millis to data creation time unit.
+   * 
+   * @param inputTime
+   *          the input time
+   * @return the long
+   */
+  @VisibleForTesting
+  private static long convertMillisToDataCreationTimeUnit(long inputTime) {
+    System.out.println("convert Millis To DataCreation TimeUnit");
+    TimeUnit dataCreationTimeUnit = getDataCreationTimeUnit();
+    if (TimeUnit.SECONDS == dataCreationTimeUnit) {
+      return (inputTime / 1000);
+    } else if (TimeUnit.MILLIS == dataCreationTimeUnit) {
+      return inputTime;
+    } else if (TimeUnit.MICROS == dataCreationTimeUnit) {
+      return inputTime * 1000;
+    }
+    return inputTime;
+  }
+
+  /**
+   * Convert micros to data creation time unit.
+   * 
+   * @param inputTime
+   *          the input time
+   * @return the long
+   */
+  @VisibleForTesting
+  private static long convertMicrosToDataCreationTimeUnit(long inputTime) {
+    System.out.println("convert Micros To DataCreation TimeUnit");
+    TimeUnit dataCreationTimeUnit = getDataCreationTimeUnit();
+    if (TimeUnit.SECONDS == dataCreationTimeUnit) {
+      return inputTime / (1000 * 1000);
+    } else if (TimeUnit.MILLIS == dataCreationTimeUnit) {
+      return inputTime / 1000;
+    } else if (TimeUnit.MICROS == dataCreationTimeUnit) {
+      return inputTime;
+    }
+    return inputTime;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapReceiverImplRestEasy.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapReceiverImplRestEasy.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapReceiverImplRestEasy.java
new file mode 100644
index 0000000..55c6b78
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapReceiverImplRestEasy.java
@@ -0,0 +1,256 @@
+package com.opensoc.pcapservice;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.opensoc.dataservices.auth.AuthTokenFilter;
+import com.opensoc.pcap.PcapUtils;
+
+@Path("/pcap/")
+public class PcapReceiverImplRestEasy {
+
+	/** The Constant LOGGER. */
+	private static final Logger LOGGER = Logger
+			.getLogger(PcapReceiverImplRestEasy.class);
+
+	/** The Constant HEADER_CONTENT_DISPOSITION_NAME. */
+	private static final String HEADER_CONTENT_DISPOSITION_NAME = "Content-Disposition";
+
+	/** The Constant HEADER_CONTENT_DISPOSITION_VALUE. */
+	private static final String HEADER_CONTENT_DISPOSITION_VALUE = "attachment; filename=\"managed-threat.pcap\"";
+
+	/** partial response key header name. */
+	private static final String HEADER_PARTIAL_RESPONE_KEY = "lastRowKey";
+
+	@AuthTokenFilter
+	@GET
+	@Path("/pcapGetter/getPcapsByKeys")
+	public Response getPcapsByKeys(
+			@QueryParam("keys") List<String> keys,
+			@QueryParam("lastRowKey") String lastRowKey,
+			@DefaultValue("-1") @QueryParam("startTime") long startTime,
+			@DefaultValue("-1") @QueryParam("endTime") long endTime,
+			@QueryParam("includeDuplicateLastRow") boolean includeDuplicateLastRow,
+			@QueryParam("includeReverseTraffic") boolean includeReverseTraffic,
+			@QueryParam("maxResponseSize") String maxResponseSize,
+			@Context HttpServletResponse response) throws IOException {
+		PcapsResponse pcapResponse = null;
+
+		LOGGER.debug( "/pcapGetter/getPcapsByKeys");
+		
+		if (keys == null || keys.size() == 0) {
+			LOGGER.debug( "no keys provided" );
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'keys' must not be null or empty").build();
+		}
+
+		try {
+			IPcapGetter pcapGetter = PcapGetterHBaseImpl.getInstance();
+			pcapResponse = pcapGetter.getPcaps(parseKeys(keys), lastRowKey,
+					startTime, endTime, includeReverseTraffic,
+					includeDuplicateLastRow,
+					ConfigurationUtil.validateMaxResultSize(maxResponseSize));
+			LOGGER.info("pcaps response in REST layer ="
+					+ pcapResponse.toString());
+
+			// return http status '204 No Content' if the pcaps response size is
+			// 0
+			if (pcapResponse == null || pcapResponse.getResponseSize() == 0) {
+
+				return Response.status(Response.Status.NO_CONTENT).build();
+			}
+
+			// return http status '206 Partial Content', the partial response
+			// file and
+			// 'lastRowKey' header , if the pcaps response status is 'PARTIAL'
+
+			response.setHeader(HEADER_CONTENT_DISPOSITION_NAME,
+					HEADER_CONTENT_DISPOSITION_VALUE);
+
+			if (pcapResponse.getStatus() == PcapsResponse.Status.PARTIAL) {
+
+				response.setHeader(HEADER_PARTIAL_RESPONE_KEY,
+						pcapResponse.getLastRowKey());
+
+				return Response
+						.ok(pcapResponse.getPcaps(),
+								MediaType.APPLICATION_OCTET_STREAM).status(206)
+						.build();
+
+			}
+
+		} catch (IOException e) {
+			LOGGER.error(
+					"Exception occurred while fetching Pcaps for the keys :"
+							+ keys.toString(), e);
+			throw e;
+		}
+
+		// return http status '200 OK' along with the complete pcaps response
+		// file,
+		// and headers
+		// return new ResponseEntity<byte[]>(pcapResponse.getPcaps(), headers,
+		// HttpStatus.OK);
+
+		return Response
+				.ok(pcapResponse.getPcaps(), MediaType.APPLICATION_OCTET_STREAM)
+				.status(200).build();
+
+	}
+	
+	@AuthTokenFilter
+	@GET
+	@Path("/pcapGetter/getPcapsByKeyRange")
+
+	  public Response getPcapsByKeyRange(
+	      @QueryParam("startKey") String startKey,
+	      @QueryParam("endKey")String endKey,
+	      @QueryParam("maxResponseSize") String maxResponseSize,
+	      @DefaultValue("-1") @QueryParam("startTime")long startTime,
+	      @DefaultValue("-1") @QueryParam("endTime") long endTime, 
+	      @Context HttpServletResponse servlet_response) throws IOException {
+
+		if (startKey == null || startKey.isEmpty())
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'start key' must not be null or empty").build();
+		
+		if (endKey == null || endKey.isEmpty())
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'end key' must not be null or empty").build();
+		
+		
+	    byte[] response = null;
+	    try {
+	      IPcapScanner pcapScanner = PcapScannerHBaseImpl.getInstance();
+	      response = pcapScanner.getPcaps(startKey, endKey,
+	          ConfigurationUtil.validateMaxResultSize(maxResponseSize), startTime,
+	          endTime);
+	      if (response == null || response.length == 0) {
+	    	  
+	    	  return Response.status(Response.Status.NO_CONTENT).entity("No Data").build();
+	        
+	      }
+	      servlet_response.setHeader(HEADER_CONTENT_DISPOSITION_NAME,
+					HEADER_CONTENT_DISPOSITION_VALUE);
+
+	    } catch (IOException e) {
+	      LOGGER.error(
+	          "Exception occurred while fetching Pcaps for the key range : startKey="
+	              + startKey + ", endKey=" + endKey, e);
+	      throw e;
+	    }
+	    // return http status '200 OK' along with the complete pcaps response file,
+	    // and headers
+	    
+		return Response
+				.ok(response, MediaType.APPLICATION_OCTET_STREAM)
+				.status(200).build();
+	  }
+
+	  /*
+	   * (non-Javadoc)
+	   * 
+	   * @see
+	   * com.cisco.opensoc.hbase.client.IPcapReceiver#getPcapsByIdentifiers(java.lang
+	   * .String, java.lang.String, java.lang.String, java.lang.String,
+	   * java.lang.String, long, long, boolean,
+	   * javax.servlet.http.HttpServletResponse)
+	   */
+	@AuthTokenFilter  
+	@GET
+	@Path("/pcapGetter/getPcapsByIdentifiers")
+
+	  public Response getPcapsByIdentifiers(
+	      @QueryParam ("srcIp") String srcIp, 
+	      @QueryParam ("dstIp") String dstIp,
+	      @QueryParam ("protocol") String protocol, 
+	      @QueryParam ("srcPort") String srcPort,
+	      @QueryParam ("dstPort") String dstPort,
+	      @DefaultValue("-1") @QueryParam ("startTime")long startTime,
+	      @DefaultValue("-1") @QueryParam ("endTime")long endTime,
+	      @DefaultValue("false") @QueryParam ("includeReverseTraffic") boolean includeReverseTraffic,
+	      @Context HttpServletResponse servlet_response)
+	      
+	      throws IOException {
+		
+		if (srcIp == null || srcIp.equals(""))
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'srcIp' must not be null or empty").build();
+		
+		if (dstIp == null || dstIp.equals(""))
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'dstIp' must not be null or empty").build();
+		
+		if (protocol == null || protocol.equals(""))
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'protocol' must not be null or empty").build();
+		
+		if (srcPort == null || srcPort.equals(""))
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'srcPort' must not be null or empty").build();
+		
+		if (dstPort == null || dstPort.equals(""))
+			return Response.serverError().status(Response.Status.NO_CONTENT)
+					.entity("'dstPort' must not be null or empty").build();
+		
+	
+	    PcapsResponse response = null;
+	    try {
+	      String sessionKey = PcapUtils.getSessionKey(srcIp, dstIp, protocol,
+	          srcPort, dstPort);
+	      LOGGER.info("sessionKey =" + sessionKey);
+	      IPcapGetter pcapGetter = PcapGetterHBaseImpl.getInstance();
+	      response = pcapGetter.getPcaps(Arrays.asList(sessionKey), null,
+	          startTime, endTime, includeReverseTraffic, false,
+	          ConfigurationUtil.getDefaultResultSize());
+	      if (response == null || response.getResponseSize() == 0) {
+	         return Response.status(Response.Status.NO_CONTENT).build();
+	      }
+	      servlet_response.setHeader(HEADER_CONTENT_DISPOSITION_NAME,
+					HEADER_CONTENT_DISPOSITION_VALUE);
+
+	    } catch (IOException e) {
+	      LOGGER.error("Exception occurred while fetching Pcaps by identifiers :",
+	          e);
+	      throw e;
+	    }
+	    // return http status '200 OK' along with the complete pcaps response file,
+	    // and headers
+	    return Response
+				.ok(response.getPcaps(), MediaType.APPLICATION_OCTET_STREAM)
+				.status(200).build();
+	  }
+	/**
+	 * This method parses the each value in the List using delimiter ',' and
+	 * builds a new List;.
+	 * 
+	 * @param keys
+	 *            list of keys to be parsed
+	 * @return list of keys
+	 */
+	@VisibleForTesting
+	List<String> parseKeys(List<String> keys) {
+		// Assert.notEmpty(keys);
+		List<String> parsedKeys = new ArrayList<String>();
+		for (String key : keys) {
+			parsedKeys.addAll(Arrays.asList(StringUtils.split(
+					StringUtils.trim(key), ",")));
+		}
+		return parsedKeys;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapScannerHBaseImpl.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapScannerHBaseImpl.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapScannerHBaseImpl.java
new file mode 100644
index 0000000..b1f0179
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapScannerHBaseImpl.java
@@ -0,0 +1,302 @@
+package com.opensoc.pcapservice;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.NoServerForRegionException;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.log4j.Logger;
+import org.springframework.util.Assert;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.opensoc.pcap.PcapMerger;
+
+/**
+ * Singleton class which integrates with HBase table and returns sorted pcaps
+ * based on the timestamp for the given range of keys. Creates HConnection if it
+ * is not already created and the same connection instance is being used for all
+ * requests
+ * 
+ * @author sheetal
+ * @version $Revision: 1.0 $
+ */
+public class PcapScannerHBaseImpl implements IPcapScanner {
+
+  /** The Constant LOGGER. */
+  private static final Logger LOGGER = Logger
+      .getLogger(PcapScannerHBaseImpl.class);
+
+  /** The Constant DEFAULT_HCONNECTION_RETRY_LIMIT. */
+  private static final int DEFAULT_HCONNECTION_RETRY_LIMIT = 0;
+
+  /** The pcap scanner h base. */
+  private static IPcapScanner pcapScannerHBase = null;
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see com.cisco.opensoc.hbase.client.IPcapScanner#getPcaps(java.lang.String,
+   * java.lang.String, long, long, long)
+   */
+  
+  public byte[] getPcaps(String startKey, String endKey, long maxResultSize,
+      long startTime, long endTime) throws IOException {
+    Assert.hasText(startKey, "startKey must no be null or empty");
+    byte[] cf = Bytes.toBytes(ConfigurationUtil.getConfiguration()
+        .getString("hbase.table.column.family"));
+    byte[] cq = Bytes.toBytes(ConfigurationUtil.getConfiguration()
+        .getString("hbase.table.column.qualifier"));
+    // create scan request
+    Scan scan = createScanRequest(cf, cq, startKey, endKey, maxResultSize,
+        startTime, endTime);
+    List<byte[]> pcaps = new ArrayList<byte[]>();
+    HTable table = null;
+    try {
+      pcaps = scanPcaps(pcaps, table, scan, cf, cq);
+    } catch (IOException e) {
+      LOGGER.error(
+          "Exception occurred while fetching Pcaps for the key range : startKey="
+              + startKey + ", endKey=" + endKey, e);
+      if (e instanceof ZooKeeperConnectionException
+          || e instanceof MasterNotRunningException
+          || e instanceof NoServerForRegionException) {
+        int maxRetryLimit = getConnectionRetryLimit();
+        for (int attempt = 1; attempt <= maxRetryLimit; attempt++) {
+          try {
+            HBaseConfigurationUtil.closeConnection(); // closing the existing
+                                                      // connection and retry,
+                                                      // it will create a new
+                                                      // HConnection
+            pcaps = scanPcaps(pcaps, table, scan, cf, cq);
+            break;
+          } catch (IOException ie) {
+            if (attempt == maxRetryLimit) {
+              System.out.println("Throwing the exception after retrying "
+                  + maxRetryLimit + " times.");
+              throw e;
+            }
+          }
+        }
+      } else {
+        throw e;
+      }
+    } finally {
+      if (table != null) {
+        table.close();
+      }
+    }
+    if (pcaps.size() == 1) {
+      return pcaps.get(0);
+    }
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PcapMerger.merge(baos, pcaps);
+    byte[] response = baos.toByteArray();
+    return response;
+  }
+
+  /**
+   * Creates the scan request.
+   * 
+   * @param cf
+   *          the cf
+   * @param cq
+   *          the cq
+   * @param startKey
+   *          the start key
+   * @param endKey
+   *          the end key
+   * @param maxResultSize
+   *          the max result size
+   * @param startTime
+   *          the start time
+   * @param endTime
+   *          the end time
+   * @return the scan
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  @VisibleForTesting
+  Scan createScanRequest(byte[] cf, byte[] cq, String startKey, String endKey,
+      long maxResultSize, long startTime, long endTime) throws IOException {
+    Scan scan = new Scan();
+    scan.addColumn(cf, cq);
+    scan.setMaxVersions(ConfigurationUtil.getConfiguration().getInt(
+        "hbase.table.column.maxVersions"));
+    scan.setStartRow(startKey.getBytes());
+    if (endKey != null) {
+      scan.setStopRow(endKey.getBytes());
+    }
+    scan.setMaxResultSize(maxResultSize);
+    boolean setTimeRange = true;
+    if (startTime < 0 && endTime < 0) {
+      setTimeRange = false;
+    }
+    if (setTimeRange) {
+      if (startTime < 0) {
+        startTime = 0;
+      } else {
+        startTime = PcapHelper.convertToDataCreationTimeUnit(startTime);
+      }
+      if (endTime < 0) {
+        endTime = Long.MAX_VALUE;
+      } else {
+        endTime = PcapHelper.convertToDataCreationTimeUnit(endTime);
+      }
+      Assert.isTrue(startTime < endTime,
+          "startTime value must be less than endTime value");
+    }
+    // create Scan request;
+    if (setTimeRange) {
+      scan.setTimeRange(startTime, endTime);
+    }
+    return scan;
+  }
+
+  /**
+   * Scan pcaps.
+   * 
+   * @param pcaps
+   *          the pcaps
+   * @param table
+   *          the table
+   * @param scan
+   *          the scan
+   * @param cf
+   *          the cf
+   * @param cq
+   *          the cq
+   * @return the list
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  @VisibleForTesting
+  List<byte[]> scanPcaps(List<byte[]> pcaps, HTable table, Scan scan,
+      byte[] cf, byte[] cq) throws IOException {
+    LOGGER.info("Scan =" + scan.toString());
+    table = (HTable) HBaseConfigurationUtil.getConnection().getTable(
+    		ConfigurationUtil.getConfiguration().getString("hbase.table.name"));
+    ResultScanner resultScanner = table.getScanner(scan);
+    List<Cell> scannedCells = new ArrayList<Cell>();
+    for (Result result = resultScanner.next(); result != null; result = resultScanner
+        .next()) {
+      List<Cell> cells = result.getColumnCells(cf, cq);
+      if (cells != null) {
+        for (Cell cell : cells) {
+          scannedCells.add(cell);
+        }
+      }
+    }
+    Collections.sort(scannedCells, PcapHelper.getCellTimestampComparator());
+    LOGGER.info("sorted cells :" + scannedCells.toString());
+    for (Cell sortedCell : scannedCells) {
+      pcaps.add(CellUtil.cloneValue(sortedCell));
+    }
+    return pcaps;
+  }
+
+  /**
+   * Gets the connection retry limit.
+   * 
+   * @return the connection retry limit
+   */
+  private int getConnectionRetryLimit() {
+    return ConfigurationUtil.getConfiguration().getInt(
+        "hbase.hconnection.retries.number", DEFAULT_HCONNECTION_RETRY_LIMIT);
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see com.cisco.opensoc.hbase.client.IPcapScanner#getPcaps(java.lang.String,
+   * java.lang.String)
+   */
+  
+  public byte[] getPcaps(String startKey, String endKey) throws IOException {
+    Assert.hasText(startKey, "startKey must no be null or empty");
+    Assert.hasText(endKey, "endKey must no be null or empty");
+    return getPcaps(startKey, endKey, ConfigurationUtil.getDefaultResultSize(),
+        -1, -1);
+  }
+
+  /**
+   * Always returns the singleton instance.
+   * 
+   * @return IPcapScanner singleton instance
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public static IPcapScanner getInstance() throws IOException {
+    if (pcapScannerHBase == null) {
+      synchronized (PcapScannerHBaseImpl.class) {
+        if (pcapScannerHBase == null) {
+          pcapScannerHBase = new PcapScannerHBaseImpl();
+        }
+      }
+    }
+    return pcapScannerHBase;
+  }
+
+  /**
+   * Instantiates a new pcap scanner h base impl.
+   */
+  private PcapScannerHBaseImpl() {
+  }
+
+  /**
+   * The main method.
+   */
+  // public static void main(String[] args) throws IOException {
+  // if (args == null || args.length < 3) {
+  // usage();
+  // return;
+  // }
+  // String outputFileName = null;
+  // String startKey = null;
+  // String stopKey = null;
+  // outputFileName = args[0];
+  // startKey = args[1];
+  // if (args.length > 2) { // NOPMD by sheetal on 1/29/14 3:55 PM
+  // stopKey = args[2];
+  // }
+  // PcapScannerHBaseImpl downloader = new PcapScannerHBaseImpl();
+  // byte[] pcaps = downloader.getPcaps(startKey, stopKey, defaultResultSize, 0,
+  // Long.MAX_VALUE);
+  // File file = new File(outputFileName);
+  // FileUtils.write(file, "", false);
+  // ByteArrayOutputStream baos = new ByteArrayOutputStream(); //
+  // $codepro.audit.disable
+  // // closeWhereCreated
+  // PcapMerger.merge(baos, pcaps);
+  // FileUtils.writeByteArrayToFile(file, baos.toByteArray(), true);
+  // }
+
+  /**
+   * Usage.
+   */
+  @SuppressWarnings("unused")
+  private static void usage() {
+    System.out.println("java " + PcapScannerHBaseImpl.class.getName() // NOPMD
+                                                                      // by
+        // sheetal
+        // <!-- //
+        // $codepro.audit.disable
+        // debuggingCode
+        // -->
+        // on
+        // 1/29/14
+        // 3:55
+        // PM
+        + " <zk quorum> <output file> <start key> [stop key]");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapsResponse.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapsResponse.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapsResponse.java
new file mode 100644
index 0000000..10af9e0
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/PcapsResponse.java
@@ -0,0 +1,153 @@
+/**
+ * 
+ */
+package com.opensoc.pcapservice;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.opensoc.pcap.PcapMerger;
+
+
+
+/**
+ * Holds pcaps data, status and the partial response key.
+ * 
+ * @author Sayi
+ */
+public class PcapsResponse {
+
+  /**
+   * The Enum Status.
+   */
+  public enum Status {
+    
+    /** The partial. */
+    PARTIAL, 
+ /** The complete. */
+ COMPLETE
+  };
+
+  /** response of the processed keys. */
+  private List<byte[]> pcaps = new ArrayList<byte[]>();;
+
+  /** partial response key. */
+  private String lastRowKey;
+
+  /** The status. */
+  private Status status = Status.COMPLETE;
+
+  /**
+   * Sets the pcaps.
+   * 
+   * @param pcaps
+   *          the new pcaps
+   */
+  public void setPcaps(List<byte[]> pcaps) {
+    this.pcaps = pcaps;
+  }
+
+  /**
+   * Adds the pcaps.
+   * 
+   * @param pcaps
+   *          the pcaps
+   */
+  public void addPcaps(byte[] pcaps) {
+    this.pcaps.add(pcaps);
+  }
+
+  /**
+   * Gets the partial response key.
+   * 
+   * @return the partial response key
+   */
+  public String getLastRowKey() {
+    return lastRowKey;
+  }
+
+  /**
+   * Sets the partial response key.
+   * 
+   * @param lastRowKey
+   *          the last row key
+   */
+  public void setLastRowKey(String lastRowKey) {
+    this.lastRowKey = lastRowKey;
+  }
+
+  /**
+   * Gets the status.
+   * 
+   * @return the status
+   */
+  public Status getStatus() {
+    return status;
+  }
+
+  /**
+   * Sets the status.
+   * 
+   * @param status
+   *          the new status
+   */
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  /**
+   * Checks if is resonse size within limit.
+   * 
+   * @param maxResultSize
+   *          the max result size
+   * @return true, if is resonse size within limit
+   */
+  public boolean isResonseSizeWithinLimit(long maxResultSize) {
+    // System.out.println("isResonseSizeWithinLimit() : getResponseSize() < (input|default result size - maximum packet size ) ="+
+    // getResponseSize()+ " < " + ( maxResultSize
+    // -ConfigurationUtil.getMaxRowSize()));
+    return getResponseSize() < (maxResultSize - ConfigurationUtil
+        .getMaxRowSize());
+  }
+
+  /**
+   * Gets the response size.
+   * 
+   * @return the response size
+   */
+  public long getResponseSize() {
+    long responseSize = 0;
+    for (byte[] pcap : this.pcaps) {
+      responseSize = responseSize + pcap.length;
+    }
+    return responseSize;
+  }
+
+  /**
+   * Gets the pcaps.
+   * 
+   * @return the pcaps
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public byte[] getPcaps() throws IOException {
+    if (pcaps.size() == 1) {
+      return pcaps.get(0);
+    }
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PcapMerger.merge(baos, pcaps);
+    return baos.toByteArray();
+  }
+
+  /* (non-Javadoc)
+   * @see java.lang.Object#toString()
+   */
+  @Override
+  public String toString() {
+    return "PcapsResponse [lastRowKey=" + lastRowKey
+        + ", status=" + status + ", pcapsSize="
+        + String.valueOf(getResponseSize()) + "]";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/RestTestingUtil.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/RestTestingUtil.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/RestTestingUtil.java
new file mode 100644
index 0000000..651affe
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/RestTestingUtil.java
@@ -0,0 +1,238 @@
+package com.opensoc.pcapservice;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
+
+/**
+ * The Class RestTestingUtil.
+ */
+public class RestTestingUtil {
+  
+  /** The host name. */
+  public static String hostName = null;
+
+  /**
+   * Gets the pcaps by keys.
+   * 
+   * @param keys
+   *          the keys
+   * @return the pcaps by keys
+   */
+  @SuppressWarnings("unchecked")
+  private static void getPcapsByKeys(String keys) {
+    System.out
+        .println("**********************getPcapsByKeys ******************************************************************************************");
+    // 1.
+    String url = "http://" + hostName
+        + "/cisco-rest/pcapGetter/getPcapsByKeys?keys={keys}"
+        + "&includeReverseTraffic={includeReverseTraffic}"
+        + "&startTime={startTime}" + "&endTime={endTime}"
+        + "&maxResponseSize={maxResponseSize}";
+    // default values
+    String startTime = "-1";
+    String endTime = "-1";
+    String maxResponseSize = "6";
+    String includeReverseTraffic = "false";
+
+    @SuppressWarnings("rawtypes")
+    Map map = new HashMap();
+    map.put("keys", keys);
+    map.put("includeReverseTraffic", includeReverseTraffic);
+    map.put("startTime", startTime);
+    map.put("endTime", endTime);
+    map.put("maxResponseSize", maxResponseSize);
+
+    RestTemplate template = new RestTemplate();
+
+    // set headers and entity to send
+    HttpHeaders headers = new HttpHeaders();
+    headers.set("Accept", MediaType.APPLICATION_OCTET_STREAM_VALUE);
+    HttpEntity<Object> requestEntity = new HttpEntity<Object>(headers);
+
+    // 1.
+    ResponseEntity<byte[]> response1 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeys : request= <keys=%s; includeReverseTraffic=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
+            response1);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+    // 2. with reverse traffic
+    includeReverseTraffic = "true";
+    map.put("includeReverseTraffic", includeReverseTraffic);
+    ResponseEntity<byte[]> response2 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeys : request= <keys=%s; includeReverseTraffic=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
+            response2);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+    // 3.with time range
+    startTime = System.getProperty("startTime", "-1");
+    endTime = System.getProperty("endTime", "-1");
+    map.put("startTime", startTime);
+    map.put("endTime", endTime);
+    ResponseEntity<byte[]> response3 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeys : request= <keys=%s; includeReverseTraffic=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
+            response3);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+    // 4.with maxResponseSize
+    maxResponseSize = System.getProperty("maxResponseSize", "6");
+    map.put("maxResponseSize", maxResponseSize);
+    ResponseEntity<byte[]> response4 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeys : request= <keys=%s; includeReverseTraffic=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
+            response4);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+  }
+
+  /**
+   * Gets the pcaps by keys range.
+   * 
+   * @param startKey
+   *          the start key
+   * @param endKey
+   *          the end key
+   * @return the pcaps by keys range
+   */
+  @SuppressWarnings("unchecked")
+  private static void getPcapsByKeysRange(String startKey, String endKey) {
+    System.out
+        .println("**********************getPcapsByKeysRange ******************************************************************************************");
+    // 1.
+    String url = "http://" + hostName
+        + "/cisco-rest/pcapGetter/getPcapsByKeyRange?startKey={startKey}"
+        + "&endKey={endKey}" + "&startTime={startTime}" + "&endTime={endTime}"
+        + "&maxResponseSize={maxResponseSize}";
+    // default values
+    String startTime = "-1";
+    String endTime = "-1";
+    String maxResponseSize = "6";
+    @SuppressWarnings("rawtypes")
+    Map map = new HashMap();
+    map.put("startKey", startKey);
+    map.put("endKey", "endKey");
+    map.put("startTime", startTime);
+    map.put("endTime", endTime);
+    map.put("maxResponseSize", maxResponseSize);
+
+    RestTemplate template = new RestTemplate();
+
+    // set headers and entity to send
+    HttpHeaders headers = new HttpHeaders();
+    headers.set("Accept", MediaType.APPLICATION_OCTET_STREAM_VALUE);
+    HttpEntity<Object> requestEntity = new HttpEntity<Object>(headers);
+
+    // 1.
+    ResponseEntity<byte[]> response1 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeysRange : request= <startKey=%s; endKey=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            startKey, endKey, startTime, endTime, maxResponseSize, response1);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+    // 2. with time range
+    startTime = System.getProperty("startTime", "-1");
+    endTime = System.getProperty("endTime", "-1");
+    map.put("startTime", startTime);
+    map.put("endTime", endTime);
+    ResponseEntity<byte[]> response2 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeysRange : request= <startKey=%s; endKey=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            startKey, endKey, startTime, endTime, maxResponseSize, response2);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+    // 3. with maxResponseSize
+    maxResponseSize = System.getProperty("maxResponseSize", "6");
+    map.put("maxResponseSize", maxResponseSize);
+    ResponseEntity<byte[]> response3 = template.exchange(url, HttpMethod.GET,
+        requestEntity, byte[].class, map);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out
+        .format(
+            "getPcapsByKeysRange : request= <startKey=%s; endKey=%s; startTime=%s; endTime=%s; maxResponseSize=%s> \n response= %s \n",
+            startKey, endKey, startTime, endTime, maxResponseSize, response3);
+    System.out
+        .println("----------------------------------------------------------------------------------------------------");
+    System.out.println();
+
+  }
+
+  /**
+   * The main method.
+   * 
+   * @param args
+   *          the arguments
+   */
+  public static void main(String[] args) {
+
+    /*
+     * Run this program with system properties
+     * 
+     * -DhostName=mon.hw.com:8090
+     * -Dkeys=18800006-1800000b-06-0019-b39d,18800006-
+     * 1800000b-06-0050-5af6-64840-40785
+     * -DstartKey=18000002-18800002-06-0436-0019-2440-34545
+     * -DendKey=18000002-18800002-06-b773-0019-2840-34585
+     */
+
+    hostName = System.getProperty("hostName");
+
+    String keys = System.getProperty("keys");
+
+    String statyKey = System.getProperty("startKey");
+    String endKey = System.getProperty("endKey");
+
+    getPcapsByKeys(keys);
+    getPcapsByKeysRange(statyKey, endKey);
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/JettyServiceRunner.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/JettyServiceRunner.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/JettyServiceRunner.java
new file mode 100644
index 0000000..1fdb025
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/JettyServiceRunner.java
@@ -0,0 +1,26 @@
+package com.opensoc.pcapservice.rest;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.core.Application;
+
+import com.opensoc.pcapservice.PcapReceiverImplRestEasy;
+
+public class JettyServiceRunner extends Application  {
+	
+
+	private static Set services = new HashSet(); 
+		
+	public  JettyServiceRunner() {     
+		// initialize restful services   
+		services.add(new PcapReceiverImplRestEasy());  
+	}
+	@Override
+	public  Set getSingletons() {
+		return services;
+	}  
+	public  static Set getServices() {  
+		return services;
+	} 
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/PcapService.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/PcapService.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/PcapService.java
new file mode 100644
index 0000000..5f47ead
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/pcapservice/rest/PcapService.java
@@ -0,0 +1,34 @@
+package com.opensoc.pcapservice.rest;
+
+import java.io.IOException;
+
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher;
+
+import com.opensoc.helpers.services.PcapServiceCli;
+
+
+public class PcapService {
+
+	public static void main(String[] args) throws IOException {
+
+		PcapServiceCli cli = new PcapServiceCli(args);
+		cli.parse();
+		
+		Server server = new Server(cli.getPort());
+		ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+		context.setContextPath("/");
+		ServletHolder h = new ServletHolder(new HttpServletDispatcher());
+		h.setInitParameter("javax.ws.rs.Application", "com.opensoc.pcapservice.rest.JettyServiceRunner");
+		context.addServlet(h, "/*");
+		server.setHandler(context);
+		try {
+			server.start();
+			server.join();
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/ElasticSearch_KafkaAlertsService.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/ElasticSearch_KafkaAlertsService.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/ElasticSearch_KafkaAlertsService.java
new file mode 100644
index 0000000..69ec5bc
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/ElasticSearch_KafkaAlertsService.java
@@ -0,0 +1,88 @@
+package com.opensoc.services.alerts;
+
+import javax.inject.Singleton;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.opensoc.dataservices.common.OpenSOCService;
+
+@Singleton
+public class ElasticSearch_KafkaAlertsService implements OpenSOCService {
+
+	private static final Logger logger = LoggerFactory.getLogger( ElasticSearch_KafkaAlertsService.class );
+	
+	@Override
+	public String identify() {
+		// TODO Auto-generated method stub
+		return "Elastic Search to Kafka Alerts Service";
+	}
+
+	@Override
+	public boolean init(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean login() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerRulesFromFile() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String viewRules() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean editRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean deleteRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerForAlertsTopic(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String receiveAlertAll() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean disconnectFromAlertsTopic(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String receiveAlertLast() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/Solr_KafkaAlertsService.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/Solr_KafkaAlertsService.java b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/Solr_KafkaAlertsService.java
new file mode 100644
index 0000000..36aaee9
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/java/com/apache/metron/services/alerts/Solr_KafkaAlertsService.java
@@ -0,0 +1,89 @@
+package com.opensoc.services.alerts;
+
+import javax.inject.Singleton;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.opensoc.dataservices.common.OpenSOCService;
+
+@Singleton
+public class Solr_KafkaAlertsService implements OpenSOCService {
+
+	private static final Logger logger = LoggerFactory.getLogger( Solr_KafkaAlertsService.class );	
+	
+	@Override
+	public String identify() {
+		// TODO Auto-generated method stub
+		return "Elastic Search to Solr Alerts Service";
+	}
+
+	@Override
+	public boolean init(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean login() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerRulesFromFile() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String viewRules() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean editRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean deleteRules() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public boolean registerForAlertsTopic(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String receiveAlertAll() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+
+	@Override
+	public boolean disconnectFromAlertsTopic(String topicname) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public String receiveAlertLast() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/config-definition-hbase.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/config-definition-hbase.xml b/metron-streaming/Metron-DataServices/src/main/resources/config-definition-hbase.xml
new file mode 100644
index 0000000..efe05e8
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/config-definition-hbase.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+
+<configuration>
+	<header>
+		<result delimiterParsingDisabled="true" forceReloadCheck="true"></result>
+		<lookups>
+      		<lookup config-prefix="expr"
+              	config-class="org.apache.commons.configuration.interpol.ExprLookup">
+        		<variables>
+          			<variable name="System" value="Class:java.lang.System"/>
+          			<variable name="net" value="Class:java.net.InetAddress"/>
+          			<variable name="String" value="Class:org.apache.commons.lang.StringUtils"/>
+        		</variables>
+      		</lookup>
+    	</lookups>
+	</header>
+	<override>
+		<!-- 1. properties from 'hbae-config.properties' are loaded first; 
+				if a property is not present in this file, then it will search in the files in the order they are defined here.
+		     2. 'refreshDelay' indicates the minimum delay in milliseconds between checks to see if the underlying file is changed.
+		     3. 'config-optional' indicates this file is not required --> 
+		
+		<properties fileName="${expr:System.getProperty('configPath')+'/hbase-config.properties'}"  config-optional="true">
+			<reloadingStrategy refreshDelay="${expr:System.getProperty('configRefreshDelay')}"
+	      config-class="org.apache.commons.configuration.reloading.FileChangedReloadingStrategy"/>
+	     </properties>
+		
+		<properties fileName="hbase-config-default.properties" config-optional="true">
+<!-- 					<reloadingStrategy refreshDelay="${expr:System.getProperty('defaultConfigRefreshDelay')}"
+	      config-class="org.apache.commons.configuration.reloading.FileChangedReloadingStrategy"/>
+ -->	     </properties>
+		
+	</override>
+</configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/hbase-config-default.properties
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/hbase-config-default.properties b/metron-streaming/Metron-DataServices/src/main/resources/hbase-config-default.properties
new file mode 100644
index 0000000..4ee56b6
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/hbase-config-default.properties
@@ -0,0 +1,40 @@
+#hbase zoo keeper configuration
+hbase.zookeeper.quorum=zkpr1,zkpr2,zkpr3
+hbase.zookeeper.clientPort=2181
+hbase.client.retries.number=1
+zookeeper.session.timeout=60000
+zookeeper.recovery.retry=0
+
+#hbase table configuration
+hbase.table.name=pcap
+hbase.table.column.family=t
+hbase.table.column.qualifier=pcap
+hbase.table.column.maxVersions=5
+
+# scan size limit configuration in MB or KB; if the input is negative or greater than max value throw an error.
+hbase.scan.result.size.unit=MB
+hbase.scan.default.result.size=6
+hbase.scan.max.result.size=60
+
+# time stamp conversion configuration; possible values 'SECONDS'(seconds), 'MILLIS'(milli seconds), 'MICROS' (micro seconds)
+hbase.table.data.time.unit=MILLIS
+
+#number of retries in case of ZooKeeper or HBase server down
+hbase.hconnection.retries.number=3
+
+#configuration for including pcaps in the reverse traffic
+pcaps.include.reverse.traffic = false
+
+#maximum table row size in KB or MB 
+hbase.table.row.size.unit = KB
+hbase.table.max.row.size = 70
+
+# tokens of row key configuration
+hbase.table.row.key.tokens=7
+rest.api.input.key.min.tokens=5
+
+# whether or not to include the last row from the previous request, applicable for only partial response scenario
+hbase.table.scan.include.duplicate.lastrow= true;
+
+#number of digits for appending tokens of the row key
+hbase.table.row.key.token.appending.digits=5

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/hbase-site.xml b/metron-streaming/Metron-DataServices/src/main/resources/hbase-site.xml
new file mode 100644
index 0000000..5c3c819
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/hbase-site.xml
@@ -0,0 +1,127 @@
+<!--Tue Apr  1 18:16:39 2014-->
+  <configuration>
+    <property>
+    <name>hbase.tmp.dir</name>
+    <value>/disk/h/hbase</value>
+  </property>
+    <property>
+    <name>hbase.hregion.memstore.chunkpool.maxsize</name>
+    <value>0.5</value>
+  </property>
+    <property>
+    <name>hbase.regionserver.codecs</name>
+    <value>lzo,gz,snappy</value>
+  </property>
+    <property>
+    <name>hbase.hstore.flush.retries.number</name>
+    <value>120</value>
+  </property>
+    <property>
+    <name>hbase.client.keyvalue.maxsize</name>
+    <value>10485760</value>
+  </property>
+    <property>
+    <name>hbase.rootdir</name>
+    <value>hdfs://nn1:8020/apps/hbase/data</value>
+  </property>
+    <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+  </property>
+    <property>
+    <name>hbase.client.scanner.caching</name>
+    <value>100</value>
+  </property>
+    <property>
+    <name>hbase.superuser</name>
+    <value>hbase</value>
+  </property>
+    <property>
+    <name>hfile.block.cache.size</name>
+    <value>0.40</value>
+  </property>
+    <property>
+    <name>hbase.regionserver.checksum.verify</name>
+    <value>true</value>
+  </property>
+    <property>
+    <name>hbase.hregion.memstore.mslab.enabled</name>
+    <value>true</value>
+  </property>
+    <property>
+    <name>hbase.hregion.max.filesize</name>
+    <value>107374182400</value>
+  </property>
+    <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+  </property>
+    <property>
+    <name>zookeeper.session.timeout</name>
+    <value>30000</value>
+  </property>
+    <property>
+    <name>zookeeper.znode.parent</name>
+    <value>/hbase-unsecure</value>
+  </property>
+    <property>
+    <name>hbase.regionserver.global.memstore.lowerLimit</name>
+    <value>0.38</value>
+  </property>
+    <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>240</value>
+  </property>
+    <property>
+    <name>hbase.hregion.memstore.mslab.chunksize</name>
+    <value>8388608</value>
+  </property>
+    <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>zkpr1,zkpr2,zkpr3</value>
+  </property>
+    <property>
+    <name>hbase.zookeeper.useMulti</name>
+    <value>true</value>
+  </property>
+    <property>
+    <name>hbase.hregion.majorcompaction</name>
+    <value>86400000</value>
+  </property>
+    <property>
+    <name>hbase.hstore.blockingStoreFiles</name>
+    <value>200</value>
+  </property>
+    <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>2181</value>
+  </property>
+    <property>
+    <name>hbase.hregion.memstore.flush.size</name>
+    <value>134217728</value>
+  </property>
+    <property>
+    <name>hbase.security.authorization</name>
+    <value>false</value>
+  </property>
+    <property>
+    <name>hbase.regionserver.global.memstore.upperLimit</name>
+    <value>0.4</value>
+  </property>
+    <property>
+    <name>hbase.hstore.compactionThreshold</name>
+    <value>4</value>
+  </property>
+    <property>
+    <name>hbase.hregion.memstore.block.multiplier</name>
+    <value>8</value>
+  </property>
+    <property>
+    <name>hbase.security.authentication</name>
+    <value>simple</value>
+  </property>
+    <property>
+    <name>dfs.client.read.shortcircuit</name>
+    <value>true</value>
+  </property>
+  </configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/log4j.xml b/metron-streaming/Metron-DataServices/src/main/resources/log4j.xml
new file mode 100644
index 0000000..834b3b3
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/log4j.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+  <appender name="console" class="org.apache.log4j.ConsoleAppender"> 
+    <param name="Target" value="System.out"/> 
+    <layout class="org.apache.log4j.PatternLayout"> 
+      <param name="ConversionPattern" value="%-5p %c{1} - %m%n"/> 
+    </layout> 
+  </appender> 
+
+  <root> 
+    <priority value ="info" /> 
+    <appender-ref ref="console" /> 
+  </root>
+  
+</log4j:configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/WEB-INF/shiro.ini
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/WEB-INF/shiro.ini b/metron-streaming/Metron-DataServices/src/main/resources/webroot/WEB-INF/shiro.ini
new file mode 100644
index 0000000..de22ff4
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/WEB-INF/shiro.ini
@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2013 Les Hazlewood and contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# INI configuration is very powerful and flexible, while still remaining succinct.
+# Please http://shiro.apache.org/configuration.html and
+# http://shiro.apache.org/web.html for more.
+
+[main]
+
+shiro.loginUrl = /login.jsp
+
+# Let's use some in-memory caching to reduce the number of runtime lookups against Stormpath.  A real
+# application might want to use a more robust caching solution (e.g. ehcache or a distributed cache).  When using such
+# caches, be aware of your cache TTL settings: too high a TTL and the cache won't reflect any potential
+# changes in Stormpath fast enough.  Too low and the cache could evict too often, reducing performance.
+cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
+securityManager.cacheManager = $cacheManager
+
+ldapRealm = org.apache.shiro.realm.ldap.JndiLdapRealm
+ldapRealm.contextFactory.url = ldap://54.210.98.199:389
+ldapRealm.contextFactory.authenticationMechanism = simple
+securityManager.realm = $ldapRealm
+
+
+[urls]
+/login = anon
+/logout = logout
+/withsocket.jsp = authc
+/withsocket2.jsp = anon
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/include.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/include.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/include.jsp
new file mode 100644
index 0000000..feb68d3
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/include.jsp
@@ -0,0 +1,20 @@
+<%--
+  ~ Copyright (c) 2013 Les Hazlewood and contributors
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  --%>
+<%-- General purpose JSP includes, useful on other pages.  This allows us to
+     consolidate these entries in one place instead of having to manually
+     repeat them on every page. --%>
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %>
+<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/logged_in.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/logged_in.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/logged_in.jsp
new file mode 100644
index 0000000..98b1159
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/logged_in.jsp
@@ -0,0 +1,22 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+
+<html>
+	
+	<head>
+		<title>Login</title>
+	</head>
+	
+	<body>
+		<h3>Logged In!</h3>
+		
+		 <ul>
+    		<c:forEach var="currentCookie" items="${cookie}">
+        		<li>
+            		<c:out value="${currentCookie.key}"/>:Object=<c:out value="${currentCookie.value.name}"/>, value=<c:out value="${currentCookie.value.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+		
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/login.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/login.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login.jsp
new file mode 100644
index 0000000..3b49f99
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login.jsp
@@ -0,0 +1,70 @@
+<%--
+  ~ Copyright (c) 2013 Les Hazlewood and contributors
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  --%>
+<jsp:include page="include.jsp"/>
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Apache Shiro Tutorial Webapp : Login</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <!-- Add some nice styling and functionality.  We'll just use Twitter Bootstrap -->
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.2/css/bootstrap.min.css">
+    <link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.0.2/css/bootstrap-theme.min.css">
+    <style>
+        body{padding-top:20px;}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="row">
+            <div class="col-md-4 col-md-offset-4">
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <h3 class="panel-title">Please sign in</h3>
+                    </div>
+                    <div class="panel-body">
+                        <form name="loginform" action="/login" method="POST" accept-charset="UTF-8" role="form">
+                            <fieldset>
+                                <div class="form-group">
+                                    <input class="form-control" placeholder="Username or Email" name="username" type="text">
+                                </div>
+                                <div class="form-group">
+                                    <input class="form-control" placeholder="Password" name="password" type="password" value="">
+                                </div>
+                                <div class="checkbox">
+                                    <label>
+                                        <input name="rememberMe" type="checkbox" value="true"> Remember Me
+                                    </label>
+                                </div>
+                                <input class="btn btn-lg btn-success btn-block" type="submit" value="Login">
+                            </fieldset>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="https://code.jquery.com/jquery.js"></script>
+    <script src="//netdna.bootstrapcdn.com/bootstrap/3.0.2/js/bootstrap.min.js"></script>
+    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+    <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+    <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
+    <![endif]-->
+</body>
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_failed.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_failed.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_failed.jsp
new file mode 100644
index 0000000..457726f
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_failed.jsp
@@ -0,0 +1,14 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+
+<html>
+	
+	<head>
+		<title>Login</title>
+	</head>
+	
+	<body>
+		<h3>Fail!!!!</h3>
+		
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_old.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_old.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_old.jsp
new file mode 100644
index 0000000..dbe6e48
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/login_old.jsp
@@ -0,0 +1,21 @@
+<html>
+	
+	<head>
+		<title>Login</title>
+	</head>
+	
+	<body>
+		<h3>Login:</h3>
+		
+		<form action="/login" method="POST">
+			
+			<label for="userName">Username:</label><input type="text" name="userName" id="userName"></input>
+			<br />
+			<label for="password">Password:</label><input type="password" name="password" id="password" ></input>
+			<br />
+			<button name="login">Login</button>
+		</form>
+		
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket.jsp
new file mode 100644
index 0000000..0530eaa
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket.jsp
@@ -0,0 +1,100 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+
+<html>
+	
+	<head>
+		<title>Do something with WebSockets</title>
+		
+		<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>	
+		<script type="text/javascript">
+		$(document).ready(function() {
+			
+				var exampleSocket = null;
+				var groupId = null;
+			
+			   	$("#startBtn").click(
+					function() {
+						exampleSocket = new WebSocket("wss://localhost:8443/ws/messages");
+						if( exampleSocket != null )
+						{
+							
+							exampleSocket.onmessage = function (event)
+							{
+								
+								var msg = event.data;
+								var index = msg.search( "groupId:" ); 
+								if(  index >= 0 )
+								{
+									console.log( "groupId message" );
+									// this is our groupId from the server, save it for future use
+									
+									groupId = msg.substring( index + 8 ).trim();
+									console.log( "groupId: " + groupId );
+								}
+								else
+								{
+									// console.log( "alert message" );
+									// this is an alert message, just render it...
+									
+									// console.log( event.data );
+									$('#msgList').append( '<li>' + msg + '</li>' );
+								}
+							}
+							
+							exampleSocket.onopen = function() {
+									console.log( "exampleSocket is open.")
+									console.log( "sending startMessages signal" );
+									exampleSocket.send( "startMessages" );		
+							}
+						}
+						else
+						{
+							alert( "exampleSocket is null!" );
+						}
+						
+					}	   
+			   )
+			   
+			   $("#stopBtn").click(
+					function() {
+						exampleSocket.send( "stopMessages");
+					}
+				)
+			   
+			})
+		
+		</script>
+	</head>
+	
+	<body>
+		<h3>Get some data with a WebSocket</h3>
+		
+		<button id="startBtn">Start</button> <span style="min-width:140px;"></span> <button id="stopBtn">Stop</button>
+
+		<div id="msgArea" style="background-color:#F8F8F8;color:red;min-height:100px; min-width:400px;">
+			<ul id="msgList" >
+			
+			</ul>
+		
+		</div>
+		
+		<div id="debug">
+		<ul>
+    		<c:forEach var="req" items="${header}">
+        		<li>
+            		<c:out value="${req.key}"/> &nbsp; = &nbsp; <c:out value="${req.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+    	<hr />
+    	<ul>
+    		<c:forEach var="currentCookie" items="${cookie}">
+        		<li>
+            		<c:out value="${currentCookie.key}"/> &nbsp; = &nbsp; <c:out value="${currentCookie.value.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+		</div>
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket2.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket2.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket2.jsp
new file mode 100644
index 0000000..7ea83ba
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket2.jsp
@@ -0,0 +1,73 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+
+<html>
+	
+	<head>
+		<title>Do something with WebSockets</title>
+		
+		<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>	
+		<script type="text/javascript">
+		$(document).ready(function(){
+			   $("#startBtn").click(
+					function() {
+						var exampleSocket = new WebSocket("wss://localhost:8443/ws/messages");
+						if( exampleSocket != null )
+						{
+							
+							exampleSocket.onmessage = function (event)
+							{
+								
+								// console.log( event.data );
+								$('#msgList').append( '<li>' + event.data + '</li>' );
+							}
+							
+							exampleSocket.onopen = function() {
+									console.log( "exampleSocket is open.")
+									console.log( "sending startMessages signal" );
+									exampleSocket.send( "startMessages" );		
+							}
+						}
+						else
+						{
+							alert( "exampleSocket is null!" );
+						}
+						
+					}	   
+			   )
+			})
+		
+		</script>
+	</head>
+	
+	<body>
+		<h3>Get some data with a WebSocket</h3>
+		
+		<button id="startBtn">Start</button>
+
+		<div id="msgArea" style="background-color:#F8F8F8;color:red;min-height:100px; min-width:400px;">
+			<ul id="msgList" >
+			
+			</ul>
+		
+		</div>
+		
+		<div id="debug">
+		<ul>
+    		<c:forEach var="req" items="${header}">
+        		<li>
+            		<c:out value="${req.key}"/> &nbsp; = &nbsp; <c:out value="${req.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+    	<hr />
+    	<ul>
+    		<c:forEach var="currentCookie" items="${cookie}">
+        		<li>
+            		<c:out value="${currentCookie.key}"/> &nbsp; = &nbsp; <c:out value="${currentCookie.value.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+		</div>
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket3.jsp
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket3.jsp b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket3.jsp
new file mode 100644
index 0000000..05c929c
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/main/resources/webroot/withsocket3.jsp
@@ -0,0 +1,100 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+
+<html>
+	
+	<head>
+		<title>Do something with WebSockets</title>
+		
+		<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>	
+		<script type="text/javascript">
+		$(document).ready(function() {
+			
+				var exampleSocket = null;
+				var groupId = null;
+			
+			   	$("#startBtn").click(
+					function() {
+						exampleSocket = new WebSocket("wss://localhost:8443/ws/messages");
+						if( exampleSocket != null )
+						{
+							
+							exampleSocket.onmessage = function (event)
+							{
+								
+								var msg = event.data;
+								var index = msg.search( "groupId:" ); 
+								if(  index >= 0 )
+								{
+									console.log( "groupId message" );
+									// this is our groupId from the server, save it for future use
+									
+									groupId = msg.substring( index + 8 ).trim();
+									console.log( "groupId: " + groupId );
+								}
+								else
+								{
+									// console.log( "alert message" );
+									// this is an alert message, just render it...
+									
+									// console.log( event.data );
+									$('#msgList').append( '<li>' + msg + '</li>' );
+								}
+							}
+							
+							exampleSocket.onopen = function() {
+									console.log( "exampleSocket is open.")
+									console.log( "sending startMessages signal" );
+									exampleSocket.send( "startMessages:abc123" );		
+							}
+						}
+						else
+						{
+							alert( "exampleSocket is null!" );
+						}
+						
+					}	   
+			   )
+			   
+			   $("#stopBtn").click(
+					function() {
+						exampleSocket.send( "stopMessages");
+					}
+				)
+			   
+			})
+		
+		</script>
+	</head>
+	
+	<body>
+		<h3>Get some data with a WebSocket</h3>
+		
+		<button id="startBtn">Start</button> <span style="min-width:140px;"></span> <button id="stopBtn">Stop</button>
+
+		<div id="msgArea" style="background-color:#F8F8F8;color:red;min-height:100px; min-width:400px;">
+			<ul id="msgList" >
+			
+			</ul>
+		
+		</div>
+		
+		<div id="debug">
+		<ul>
+    		<c:forEach var="req" items="${header}">
+        		<li>
+            		<c:out value="${req.key}"/> &nbsp; = &nbsp; <c:out value="${req.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+    	<hr />
+    	<ul>
+    		<c:forEach var="currentCookie" items="${cookie}">
+        		<li>
+            		<c:out value="${currentCookie.key}"/> &nbsp; = &nbsp; <c:out value="${currentCookie.value.value}"/>
+        		</li>
+    		</c:forEach>
+    	</ul>
+		</div>
+	</body>
+	
+</html>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java b/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
new file mode 100644
index 0000000..c2a4bf2
--- /dev/null
+++ b/metron-streaming/Metron-DataServices/src/test/java/com/apache/metron/pcapservice/CellTimestampComparatorTest.java
@@ -0,0 +1,92 @@
+package com.opensoc.pcapservice;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hbase.Cell;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.opensoc.pcapservice.CellTimestampComparator;
+
+/**
+ * The Class CellTimestampComparatorTest.
+ */
+public class CellTimestampComparatorTest {
+
+  /**
+   * Sets the up.
+   * 
+   * @throws Exception
+   *           the exception
+   */
+  @Before
+  public void setUp() throws Exception {
+  }
+
+  /**
+   * Tear down.
+   * 
+   * @throws Exception
+   *           the exception
+   */
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  /**
+   * Test_less.
+   */
+  @Test
+  public void test_less() {
+    // mocking
+    Cell cell1 = Mockito.mock(Cell.class);
+    Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
+    Cell cell2 = Mockito.mock(Cell.class);
+    Mockito.when(cell2.getTimestamp()).thenReturn(13845345808L);
+
+    CellTimestampComparator comparator = new CellTimestampComparator();
+
+    // actual call and verify
+    Assert.assertTrue(comparator.compare(cell2, cell1) == -1);
+
+  }
+
+  /**
+   * Test_greater.
+   */
+  @Test
+  public void test_greater() {
+    // mocking
+    Cell cell1 = Mockito.mock(Cell.class);
+    Mockito.when(cell1.getTimestamp()).thenReturn(13745345808L);
+    Cell cell2 = Mockito.mock(Cell.class);
+    Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
+
+    CellTimestampComparator comparator = new CellTimestampComparator();
+
+    // actual call and verify
+    Assert.assertTrue(comparator.compare(cell2, cell1) == 1);
+
+  }
+
+  /**
+   * Test_equal.
+   */
+  @Test
+  public void test_equal() {
+    // mocking
+    Cell cell1 = Mockito.mock(Cell.class);
+    Mockito.when(cell1.getTimestamp()).thenReturn(13945345808L);
+    Cell cell2 = Mockito.mock(Cell.class);
+    Mockito.when(cell2.getTimestamp()).thenReturn(13945345808L);
+
+    CellTimestampComparator comparator = new CellTimestampComparator();
+
+    // actual call and verify
+    Assert.assertTrue(comparator.compare(cell2, cell1) == 0);
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/90cda3ff/metron-streaming/Metron-EnrichmentAdapters/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-EnrichmentAdapters/pom.xml b/metron-streaming/Metron-EnrichmentAdapters/pom.xml
new file mode 100644
index 0000000..fb21130
--- /dev/null
+++ b/metron-streaming/Metron-EnrichmentAdapters/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software 
+	Foundation (ASF) under one or more contributor license agreements. See the 
+	NOTICE file distributed with this work for additional information regarding 
+	copyright ownership. The ASF licenses this file to You under the Apache License, 
+	Version 2.0 (the "License"); you may not use this file except in compliance 
+	with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+	Unless required by applicable law or agreed to in writing, software distributed 
+	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES 
+	OR CONDITIONS OF ANY KIND, either express or implied. See the License for 
+	the specific language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.opensoc</groupId>
+		<artifactId>OpenSOC-Streaming</artifactId>
+		<version>0.6BETA</version>
+	</parent>
+	<artifactId>OpenSOC-EnrichmentAdapters</artifactId>
+
+	<properties>
+       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>		
+		<mysql.version>5.1.31</mysql.version>
+		<slf4j.version>1.7.7</slf4j.version>
+		<hbase.client.version>0.96.1-hadoop2</hbase.client.version>
+		<storm.hdfs.version>0.1.2</storm.hdfs.version>
+		<guava.version>17.0</guava.version>
+	</properties>
+	<dependencies>
+		<dependency>
+			<groupId>com.opensoc</groupId>
+			<artifactId>OpenSOC-Common</artifactId>
+			<version>${project.parent.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>${slf4j.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>mysql</groupId>
+			<artifactId>mysql-connector-java</artifactId>
+			<version>${mysql.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-client</artifactId>
+			<version>${hbase.client.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-log4j12</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>log4j</groupId>
+					<artifactId>log4j</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+			<version>${global_hadoop_version}</version>
+			  <exclusions>
+				<exclusion>
+				   <artifactId>servlet-api</artifactId>
+				   <groupId>javax.servlet</groupId>
+				  </exclusion>
+		    </exclusions>					
+		</dependency>
+		<dependency>
+			<groupId>org.apache.storm</groupId>
+			<artifactId>storm-core</artifactId>
+			<version>${global_storm_version}</version>
+			<scope>provided</scope>
+			  <exclusions>
+				<exclusion>
+				   <artifactId>servlet-api</artifactId>
+				   <groupId>javax.servlet</groupId>
+				  </exclusion>
+		    </exclusions>					
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>${guava.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+			<version>${global_hadoop_version}</version>
+			<exclusions>
+				<exclusion>
+				   <artifactId>servlet-api</artifactId>
+				   <groupId>javax.servlet</groupId>
+				  </exclusion>
+		    </exclusions>			
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${global_junit_version}</version>
+		</dependency>
+		<dependency>
+			<groupId>commons-validator</groupId>
+			<artifactId>commons-validator</artifactId>
+			<version>1.4.0</version>
+		</dependency>
+
+	</dependencies>
+	<reporting>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<systemProperties>
+						<property>
+							<name>mode</name>
+							<value>global</value>
+						</property>
+					</systemProperties>
+				</configuration>
+			</plugin>
+			<!-- Normally, dependency report takes time, skip it -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-project-info-reports-plugin</artifactId>
+				<version>2.7</version>
+
+				<configuration>
+					<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>emma-maven-plugin</artifactId>
+				<version>1.0-alpha-3</version>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-pmd-plugin</artifactId>
+				<configuration>
+					<targetJdk>1.7</targetJdk>
+				</configuration>
+			</plugin>
+		</plugins>
+	</reporting>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.1</version>
+				<configuration>
+					<source>1.7</source>
+					<target>1.7</target>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+</project>


Mime
View raw message