metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [42/51] [partial] incubator-metron git commit: Initial import of code from https://github.com/OpenSOC/opensoc at ac0b00373f8f56dfae03a8109af5feb373ea598e.
Date Tue, 08 Dec 2015 06:38:06 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/IseSchema.json
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/IseSchema.json b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/IseSchema.json
new file mode 100644
index 0000000..1c3f670
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/IseSchema.json
@@ -0,0 +1,21 @@
+{
+	"title": "Pcap Schema",
+	"type": "object",
+	"properties": {
+		"ip_src_addr": {
+			"type": "string"
+		},
+		"ip_dst_addr": {
+			"type": "string"
+		},
+		"ip_src_port": {
+			"type": "string"
+		},
+		"ip_dst_port": {
+			"type": "string"
+		},
+		"protocol": {
+			"type": "string"
+		}
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/LancopeSchema.json
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/LancopeSchema.json b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/LancopeSchema.json
new file mode 100644
index 0000000..12f326f
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/LancopeSchema.json
@@ -0,0 +1,22 @@
+{
+	"title": "Lancope Schema",
+	"type": "object",
+	"properties": {
+		"ip_src_addr": {
+			"type": "string"
+		},
+		"ip_dst_addr": {
+			"type": "string"
+		},
+		"ip_src_port": {
+			"type": "string"
+		},
+		"ip_dst_port": {
+			"type": "string"
+		},
+		"protocol": {
+			"type": "string"
+		}
+	},
+	"required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port","protocol"]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/PcapSchema.json
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/PcapSchema.json b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/PcapSchema.json
new file mode 100644
index 0000000..761396e
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/PcapSchema.json
@@ -0,0 +1,22 @@
+{
+	"title": "Pcap Schema",
+	"type": "object",
+	"properties": {
+		"ip_src_addr": {
+			"type": "string"
+		},
+		"ip_dst_addr": {
+			"type": "string"
+		},
+		"ip_src_port": {
+			"type": "string"
+		},
+		"ip_dst_port": {
+			"type": "string"
+		},
+		"protocol": {
+			"type": "string"
+		}
+	},
+	"required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port","protocol"]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/SourcefireSchema.json
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/SourcefireSchema.json b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/SourcefireSchema.json
new file mode 100644
index 0000000..3984b00
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-MessageParsers/src/test/resources/TestSchemas/SourcefireSchema.json
@@ -0,0 +1,22 @@
+{
+	"title": "Sourcefire Schema",
+	"type": "object",
+	"properties": {
+		"ip_src_addr": {
+			"type": "string"
+		},
+		"ip_dst_addr": {
+			"type": "string"
+		},
+		"ip_src_port": {
+			"type": "string"
+		},
+		"ip_dst_port": {
+			"type": "string"
+		},
+		"protocol": {
+			"type": "string"
+		}
+	},
+	"required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port","protocol"]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/.pmd
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/.pmd b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/.pmd
new file mode 100644
index 0000000..b4dd643
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/.pmd
@@ -0,0 +1,1262 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pmd>
+    <useProjectRuleSet>false</useProjectRuleSet>
+    <ruleSetFile>.ruleset</ruleSetFile>
+    <rules>
+        <rule>
+            <name>IfStmtsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>IfElseStmtsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>WhileLoopsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>ForLoopsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryConstructor</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>NullAssignment</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>OnlyOneReturn</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AssignmentInOperand</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AtLeastOneConstructor</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>DontImportSun</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>SuspiciousOctalEscape</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>CallSuperInConstructor</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryParentheses</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>DefaultPackage</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>BooleanInversion</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>DataflowAnomalyAnalysis</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidFinalLocalVariable</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidUsingShortType</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidUsingVolatile</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidUsingNativeCode</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidAccessibilityAlteration</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotCallGarbageCollectionExplicitly</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>OneDeclarationPerLine</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidPrefixingMethodParameters</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidLiteralsInIfCondition</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UseObjectForClearerAPI</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UseConcurrentHashMap</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedPrivateField</name>
+            <ruleset>Unused Code</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedLocalVariable</name>
+            <ruleset>Unused Code</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedPrivateMethod</name>
+            <ruleset>Unused Code</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedFormalParameter</name>
+            <ruleset>Unused Code</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedModifier</name>
+            <ruleset>Unused Code</ruleset>
+        </rule>
+        <rule>
+            <name>MethodReturnsInternalArray</name>
+            <ruleset>Security Code Guidelines</ruleset>
+        </rule>
+        <rule>
+            <name>ArrayIsStoredDirectly</name>
+            <ruleset>Security Code Guidelines</ruleset>
+        </rule>
+        <rule>
+            <name>ProperCloneImplementation</name>
+            <ruleset>Clone Implementation</ruleset>
+        </rule>
+        <rule>
+            <name>CloneThrowsCloneNotSupportedException</name>
+            <ruleset>Clone Implementation</ruleset>
+        </rule>
+        <rule>
+            <name>CloneMethodMustImplementCloneable</name>
+            <ruleset>Clone Implementation</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitStaticSuite</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitSpelling</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitAssertionsShouldIncludeMessage</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitTestsShouldIncludeAssert</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>TestClassWithoutTestCases</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryBooleanAssertion</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>UseAssertEqualsInsteadOfAssertTrue</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>UseAssertSameInsteadOfAssertTrue</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>UseAssertNullInsteadOfAssertTrue</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>SimplifyBooleanAssertion</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitTestContainsTooManyAsserts</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>UseAssertTrueInsteadOfAssertEquals</name>
+            <ruleset>JUnit</ruleset>
+        </rule>
+        <rule>
+            <name>CommentRequired</name>
+            <ruleset>Comments</ruleset>
+        </rule>
+        <rule>
+            <name>CommentSize</name>
+            <ruleset>Comments</ruleset>
+        </rule>
+        <rule>
+            <name>CommentContent</name>
+            <ruleset>Comments</ruleset>
+        </rule>
+        <rule>
+            <name>ShortVariable</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>LongVariable</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>ShortMethodName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>VariableNamingConventions</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>MethodNamingConventions</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>ClassNamingConventions</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>AbstractNaming</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidDollarSigns</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>MethodWithSameNameAsEnclosingClass</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>SuspiciousHashcodeMethodName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>SuspiciousConstantFieldName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>SuspiciousEqualsMethodName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidFieldNameMatchingTypeName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidFieldNameMatchingMethodName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>NoPackage</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>PackageCase</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>MisleadingVariableName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>BooleanGetMethodName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>ShortClassName</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>GenericsNaming</name>
+            <ruleset>Naming</ruleset>
+        </rule>
+        <rule>
+            <name>DuplicateImports</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>DontImportJavaLang</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedImports</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>ImportFromSamePackage</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>TooManyStaticImports</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryFullyQualifiedName</name>
+            <ruleset>Import Statements</ruleset>
+        </rule>
+        <rule>
+            <name>ReplaceVectorWithList</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>ReplaceHashtableWithMap</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>ReplaceEnumerationWithIterator</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidEnumAsIdentifier</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidAssertAsIdentifier</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>IntegerInstantiation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>ByteInstantiation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>ShortInstantiation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>LongInstantiation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>JUnit4TestShouldUseBeforeAnnotation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>JUnit4TestShouldUseAfterAnnotation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>JUnit4TestShouldUseTestAnnotation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>JUnit4SuitesShouldUseSuiteAnnotation</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>JUnitUseExpected</name>
+            <ruleset>Migration</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryParentheses</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryBlock</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>DontNestJsfInJstlIteration</name>
+            <ruleset>Basic JSF</ruleset>
+        </rule>
+        <rule>
+            <name>MistypedCDATASection</name>
+            <ruleset>Basic XML</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyCatchBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyIfStmt</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyWhileStmt</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyTryBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyFinallyBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptySwitchStatements</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptySynchronizedBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStatementNotInLoop</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyInitializer</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStatementBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStaticInitializer</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>CallSuperFirst</name>
+            <ruleset>Android</ruleset>
+        </rule>
+        <rule>
+            <name>CallSuperLast</name>
+            <ruleset>Android</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotHardCodeSDCard</name>
+            <ruleset>Android</ruleset>
+        </rule>
+        <rule>
+            <name>JumbledIncrementer</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>ForLoopShouldBeWhileLoop</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>OverrideBothEqualsAndHashcode</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>DoubleCheckedLocking</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>ReturnFromFinallyBlock</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>UnconditionalIfStatement</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>BooleanInstantiation</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>CollapsibleIfStatements</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>ClassCastExceptionWithToArray</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidDecimalLiteralsInBigDecimalConstructor</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>MisplacedNullCheck</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidThreadGroup</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>BrokenNullCheck</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>BigIntegerInstantiation</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidUsingOctalValues</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidUsingHardCodedIP</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>CheckResultSet</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidMultipleUnaryOperators</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>ExtendsObject</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>CheckSkipResult</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidBranchingStatementAsLastInLoop</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>DontCallThreadRun</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>DontUseFloatTypeForLoopIndices</name>
+            <ruleset>Basic</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyCatchBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyIfStmt</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyWhileStmt</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyTryBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyFinallyBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptySwitchStatements</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptySynchronizedBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStatementNotInLoop</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyInitializer</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStatementBlock</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyStaticInitializer</name>
+            <ruleset>Empty Code</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryConversionTemporary</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryReturn</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryFinalModifier</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessOverridingMethod</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessOperationOnImmutable</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedNullCheckInEquals</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessParentheses</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>CouplingBetweenObjects</name>
+            <ruleset>Coupling</ruleset>
+        </rule>
+        <rule>
+            <name>ExcessiveImports</name>
+            <ruleset>Coupling</ruleset>
+        </rule>
+        <rule>
+            <name>LooseCoupling</name>
+            <ruleset>Coupling</ruleset>
+        </rule>
+        <rule>
+            <name>LoosePackageCoupling</name>
+            <ruleset>Coupling</ruleset>
+        </rule>
+        <rule>
+            <name>LawOfDemeter</name>
+            <ruleset>Coupling</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryConversionTemporary</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryReturn</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryFinalModifier</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessOverridingMethod</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessOperationOnImmutable</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UnusedNullCheckInEquals</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>UselessParentheses</name>
+            <ruleset>Unnecessary</ruleset>
+        </rule>
+        <rule>
+            <name>NoLongScripts</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoScriptlets</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoInlineStyleInformation</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoClassAttribute</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoJspForward</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>IframeMissingSrcAttribute</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoHtmlComments</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>DuplicateJspImports</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>JspEncoding</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>NoInlineScript</name>
+            <ruleset>Basic JSP</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidCatchingThrowable</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>SignatureDeclareThrowsException</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>ExceptionAsFlowControl</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidCatchingNPE</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidThrowingRawExceptionTypes</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidThrowingNullPointerException</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidRethrowingException</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotExtendJavaLangError</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotThrowExceptionInFinally</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidThrowingNewInstanceOfSameException</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidCatchingGenericException</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidLosingExceptionInformation</name>
+            <ruleset>Strict Exceptions</ruleset>
+        </rule>
+        <rule>
+            <name>UseCorrectExceptionLogging</name>
+            <ruleset>Jakarta Commons Logging</ruleset>
+        </rule>
+        <rule>
+            <name>ProperLogger</name>
+            <ruleset>Jakarta Commons Logging</ruleset>
+        </rule>
+        <rule>
+            <name>GuardDebugLogging</name>
+            <ruleset>Jakarta Commons Logging</ruleset>
+        </rule>
+        <rule>
+            <name>LocalVariableCouldBeFinal</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>MethodArgumentCouldBeFinal</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidInstantiatingObjectsInLoops</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>UseArrayListInsteadOfVector</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>SimplifyStartsWith</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>UseStringBufferForStringAppends</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>UseArraysAsList</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidArrayLoops</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryWrapperObjectCreation</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>AddEmptyString</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>RedundantFieldInitializer</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>PrematureDeclaration</name>
+            <ruleset>Optimization</ruleset>
+        </rule>
+        <rule>
+            <name>UseProperClassLoader</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>MDBAndSessionBeanNamingConvention</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>RemoteSessionInterfaceNamingConvention</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>LocalInterfaceSessionNamingConvention</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>LocalHomeNamingConvention</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>RemoteInterfaceNamingConvention</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotCallSystemExit</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>StaticEJBFieldShouldBeFinal</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>DoNotUseThreads</name>
+            <ruleset>J2EE</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidDuplicateLiterals</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>StringInstantiation</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>StringToString</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>InefficientStringBuffering</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryCaseChange</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>UseStringBufferLength</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>AppendCharacterWithChar</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>ConsecutiveLiteralAppends</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>UseIndexOfChar</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>InefficientEmptyStringCheck</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>InsufficientStringBufferDeclaration</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>UselessStringValueOf</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>StringBufferInstantiationWithChar</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>UseEqualsToCompareStrings</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidStringBufferField</name>
+            <ruleset>String and StringBuffer</ruleset>
+        </rule>
+        <rule>
+            <name>MoreThanOneLogger</name>
+            <ruleset>Java Logging</ruleset>
+        </rule>
+        <rule>
+            <name>LoggerIsNotStaticFinal</name>
+            <ruleset>Java Logging</ruleset>
+        </rule>
+        <rule>
+            <name>SystemPrintln</name>
+            <ruleset>Java Logging</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidPrintStackTrace</name>
+            <ruleset>Java Logging</ruleset>
+        </rule>
+        <rule>
+            <name>UseConcatOnce</name>
+            <ruleset>XPath in XSL</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidAxisNavigation</name>
+            <ruleset>XPath in XSL</ruleset>
+        </rule>
+        <rule>
+            <name>AssignmentInOperand</name>
+            <ruleset>Controversial</ruleset>
+        </rule>
+        <rule>
+            <name>UnreachableCode</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>InnaccurateNumericLiteral</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>ConsistentReturn</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>ScopeForInVariable</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>EqualComparison</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>GlobalVariable</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidTrailingComma</name>
+            <ruleset>Basic Ecmascript</ruleset>
+        </rule>
+        <rule>
+            <name>IfStmtsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>WhileLoopsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>IfElseStmtsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>ForLoopsMustUseBraces</name>
+            <ruleset>Braces</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyFinalizer</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>FinalizeOnlyCallsSuperFinalize</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>FinalizeOverloaded</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>FinalizeDoesNotCallSuperFinalize</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>FinalizeShouldBeProtected</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidCallingFinalize</name>
+            <ruleset>Finalizer</ruleset>
+        </rule>
+        <rule>
+            <name>UseSingleton</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SimplifyBooleanReturns</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SimplifyBooleanExpressions</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SwitchStmtsShouldHaveDefault</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidDeeplyNestedIfStmts</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidReassigningParameters</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SwitchDensity</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>ConstructorCallsOverridableMethod</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AccessorClassGeneration</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>FinalFieldCouldBeStatic</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>CloseResource</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>NonStaticInitializer</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>DefaultLabelNotLastInSwitchStmt</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>NonCaseLabelInSwitchStatement</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>OptimizableToArrayCall</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>BadComparison</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>EqualsNull</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>ConfusingTernary</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>InstantiationToGetClass</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>IdempotentOperations</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SimpleDateFormatNeedsLocale</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>ImmutableField</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UseLocaleWithCaseConversions</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidProtectedFieldInFinalClass</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AssignmentToNonFinalStatic</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>MissingStaticMethodInNonInstantiatableClass</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidSynchronizedAtMethodLevel</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>MissingBreakInSwitch</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UseNotifyAllInsteadOfNotify</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidInstanceofChecksInCatchClause</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AbstractClassWithoutAbstractMethod</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SimplifyConditional</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>CompareObjectsWithEquals</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>PositionLiteralsFirstInComparisons</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UnnecessaryLocalBeforeReturn</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>NonThreadSafeSingleton</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UncommentedEmptyMethod</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UncommentedEmptyConstructor</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AvoidConstantsInterface</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UnsynchronizedStaticDateFormatter</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>PreserveStackTrace</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UseCollectionIsEmpty</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>ClassWithOnlyPrivateConstructorsShouldBeFinal</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>EmptyMethodInAbstractClassShouldBeAbstract</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>SingularField</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>ReturnEmptyArrayRatherThanNull</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>AbstractClassWithoutAnyMethod</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>TooFewBranchesForASwitchStatement</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>LogicInversion</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>UseVarargs</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>FieldDeclarationsShouldBeAtStartOfClass</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>GodClass</name>
+            <ruleset>Design</ruleset>
+        </rule>
+        <rule>
+            <name>NPathComplexity</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>ExcessiveMethodLength</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>ExcessiveParameterList</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>ExcessiveClassLength</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>CyclomaticComplexity</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>ExcessivePublicCount</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>TooManyFields</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>NcssMethodCount</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>NcssTypeCount</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>NcssConstructorCount</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>TooManyMethods</name>
+            <ruleset>Code Size</ruleset>
+        </rule>
+        <rule>
+            <name>BeanMembersShouldSerialize</name>
+            <ruleset>JavaBeans</ruleset>
+        </rule>
+        <rule>
+            <name>MissingSerialVersionUID</name>
+            <ruleset>JavaBeans</ruleset>
+        </rule>
+    </rules>
+    <includeDerivedFiles>false</includeDerivedFiles>
+    <violationsAsErrors>true</violationsAsErrors>
+    <fullBuildEnabled>true</fullBuildEnabled>
+</pmd>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/README.txt
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/README.txt b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/README.txt
new file mode 100644
index 0000000..8aba23e
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/README.txt
@@ -0,0 +1,16 @@
+'hbase' module of 'opensoc' project contains the code to communicate with HBase. This module has several APIs ( refer IPcapGetter.java, IPcapScanner.java files ) 
+to fetch pcaps from HBase. Following APIs have been created under this module implementation.
+
+APIs ( in IPcapGetter.java) to get pcaps using keys :
+ 1. public PcapsResponse getPcaps(List<String> keys, String lastRowKey, long startTime, long endTime, boolean includeReverseTraffic, boolean includeDuplicateLastRow, long maxResultSize) throws IOException;
+ 2. public PcapsResponse getPcaps(String key, long startTime, long endTime, boolean includeReverseTraffic) throws IOException;
+ 3. public PcapsResponse getPcaps(List<String> keys) throws IOException;
+ 4. public PcapsResponse getPcaps(String key) throws IOException;
+
+APIs ( in IPcapScanner.java) to get pcaps using key range :
+ 1. public byte[] getPcaps(String startKey, String endKey, long maxResponseSize, long startTime, long endTime) throws IOException;
+ 2. public byte[] getPcaps(String startKey, String endKey) throws IOException;
+ 
+ 
+Refer the wiki documentation for further details : https://hwcsco.atlassian.net/wiki/pages/viewpage.action?pageId=5242892
+ 	
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/dependency-reduced-pom.xml b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/dependency-reduced-pom.xml
new file mode 100644
index 0000000..11efb2f
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/dependency-reduced-pom.xml
@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <parent>
+    <artifactId>managed-threat</artifactId>
+    <groupId>cisco</groupId>
+    <version>0.0.4-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>cisco-hbase</groupId>
+  <artifactId>cisco-hbase</artifactId>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-client</artifactId>
+      <version>0.96.0.2.0.6.0-76-hadoop2</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>hbase-common</artifactId>
+          <groupId>org.apache.hbase</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hbase-protocol</artifactId>
+          <groupId>org.apache.hbase</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-codec</artifactId>
+          <groupId>commons-codec</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-io</artifactId>
+          <groupId>commons-io</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang</artifactId>
+          <groupId>commons-lang</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-logging</artifactId>
+          <groupId>commons-logging</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>guava</artifactId>
+          <groupId>com.google.guava</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>protobuf-java</artifactId>
+          <groupId>com.google.protobuf</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>netty</artifactId>
+          <groupId>io.netty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>zookeeper</artifactId>
+          <groupId>org.apache.zookeeper</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>htrace-core</artifactId>
+          <groupId>org.cloudera.htrace</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>org.codehaus.jackson</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-auth</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>findbugs-annotations</artifactId>
+          <groupId>com.github.stephenc.findbugs</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>junit</artifactId>
+          <groupId>junit</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>2.2.0.2.0.6.0-76</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>commons-cli</artifactId>
+          <groupId>commons-cli</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-math</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>xmlenc</artifactId>
+          <groupId>xmlenc</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-httpclient</artifactId>
+          <groupId>commons-httpclient</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-net</artifactId>
+          <groupId>commons-net</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>servlet-api</artifactId>
+          <groupId>javax.servlet</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty</artifactId>
+          <groupId>org.mortbay.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jetty-util</artifactId>
+          <groupId>org.mortbay.jetty</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-core</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-json</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-server</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jasper-compiler</artifactId>
+          <groupId>tomcat</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jasper-runtime</artifactId>
+          <groupId>tomcat</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsp-api</artifactId>
+          <groupId>javax.servlet.jsp</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-el</artifactId>
+          <groupId>commons-el</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jets3t</artifactId>
+          <groupId>net.java.dev.jets3t</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-configuration</artifactId>
+          <groupId>commons-configuration</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>slf4j-api</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-core-asl</artifactId>
+          <groupId>org.codehaus.jackson</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>avro</artifactId>
+          <groupId>org.apache.avro</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jsch</artifactId>
+          <groupId>com.jcraft</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-compress</artifactId>
+          <groupId>org.apache.commons</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-annotations</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>guava</artifactId>
+          <groupId>com.google.guava</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-codec</artifactId>
+          <groupId>commons-codec</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-io</artifactId>
+          <groupId>commons-io</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-logging</artifactId>
+          <groupId>commons-logging</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>commons-lang</artifactId>
+          <groupId>commons-lang</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>org.codehaus.jackson</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>protobuf-java</artifactId>
+          <groupId>com.google.protobuf</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-auth</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>zookeeper</artifactId>
+          <groupId>org.apache.zookeeper</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+</project>
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/pom.xml b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/pom.xml
new file mode 100644
index 0000000..f1f471f
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/pom.xml
@@ -0,0 +1,137 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.cisco.opensoc</groupId>
+		<artifactId>opensoc</artifactId>
+		<version>0.1.3-SNAPSHOT</version>
+		<relativePath>../../../pom.xml</relativePath>
+	</parent>
+	<artifactId>opensoc-hbase</artifactId>
+
+	<dependencies>
+		<dependency>
+			<groupId>commons-beanutils</groupId>
+			<artifactId>commons-beanutils</artifactId>
+			<version>${commons-beanutils.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-jexl</artifactId>
+			<version>${commons-jexl.version}</version>
+		</dependency>
+
+		<dependency>
+			<artifactId>commons-configuration</artifactId>
+			<groupId>commons-configuration</groupId>
+			<version>${commons-configuration.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${junit.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.powermock</groupId>
+			<artifactId>powermock-api-mockito</artifactId>
+			<version>1.5</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.powermock</groupId>
+			<artifactId>powermock-core</artifactId>
+			<version>1.5</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.powermock</groupId>
+			<artifactId>powermock-module-junit4</artifactId>
+			<version>1.5</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>joda-time</groupId>
+			<artifactId>joda-time</artifactId>
+			<version>2.3</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-client</artifactId>
+			<version>${hbase.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-testing-util</artifactId>
+			<version>${hbase.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+			<version>${hadoop.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+			<version>${hadoop.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.cisco.opensoc</groupId>
+			<artifactId>opensoc-common</artifactId>
+			<version>${project.parent.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.cisco.opensoc</groupId>
+			<artifactId>opensoc-pcap</artifactId>
+			<version>${project.parent.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-mapreduce-client-common</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-common</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.hadoop</groupId>
+					<artifactId>hadoop-core</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.integration</groupId>
+			<artifactId>spring-integration-http</artifactId>
+			<version>${spring.integration.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-webmvc</artifactId>
+			<version>${spring.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>${logger.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>com.sun.jmx</groupId>
+					<artifactId>jmxri</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jdmk</groupId>
+					<artifactId>jmxtools</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>javax.jms</groupId>
+					<artifactId>jms</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+
+	</dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/CellTimestampComparator.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/CellTimestampComparator.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/CellTimestampComparator.java
new file mode 100644
index 0000000..18bf0e5
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/CellTimestampComparator.java
@@ -0,0 +1,23 @@
+package com.cisco.opensoc.hbase.client;
+
+import java.util.Comparator;
+
+import org.apache.hadoop.hbase.Cell;
+
+/**
+ * Comparator created for sorting pcaps cells based on the timestamp (dsc).
+ * 
+ * @author Sayi
+ */
+public class CellTimestampComparator implements Comparator<Cell> {
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
+   */
+  @Override
+  public int compare(Cell o1, Cell o2) {
+    return Long.valueOf(o2.getTimestamp()).compareTo(o1.getTimestamp());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/ConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/ConfigurationUtil.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/ConfigurationUtil.java
new file mode 100644
index 0000000..7a1d486
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/ConfigurationUtil.java
@@ -0,0 +1,267 @@
+package com.cisco.opensoc.hbase.client;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.springframework.util.Assert;
+
+import com.cisco.opensoc.common.config.ConfigurationManager;
+
+/**
+ * utility class for this module which loads commons configuration to fetch
+ * properties from underlying resources to communicate with hbase.
+ * 
+ * @author Sayi
+ */
+public class ConfigurationUtil {
+
+	/** Configuration definition file name for fetching pcaps from hbase */
+	private static final String configDefFileName = "config-definition-hbase.xml";
+	
+	/** property configuration. */
+	private static Configuration propConfiguration = null;
+
+
+	/**
+	 * The Enum SizeUnit.
+	 */
+	public enum SizeUnit {
+
+		/** The kb. */
+		KB,
+		/** The mb. */
+		MB
+	};
+
+	/** The Constant DEFAULT_HCONNECTION_RETRY_LIMIT. */
+	private static final int DEFAULT_HCONNECTION_RETRY_LIMIT = 0;
+
+	/**
+	 * Loads configuration resources 
+	 * @return Configuration
+	 */
+	public static Configuration getConfiguration() {
+		if(propConfiguration == null){
+			propConfiguration =  ConfigurationManager.getConfiguration(configDefFileName);
+		}
+		return propConfiguration;
+	}
+
+	/**
+	 * Returns the configured default result size in bytes, if the user input is
+	 * null; otherwise, returns the user input after validating with the
+	 * configured max value. Throws IllegalArgumentException if : 1. input is
+	 * less than or equals to 0 OR 2. input is greater than configured
+	 * {hbase.scan.max.result.size} value
+	 * 
+	 * @param input
+	 *            the input
+	 * @return long
+	 */
+	public static long validateMaxResultSize(String input) {
+		if (input == null) {
+			return getDefaultResultSize();
+		}
+		// validate the user input
+		long value = convertToBytes(Long.parseLong(input), getResultSizeUnit());
+		Assert.isTrue(
+				isAllowableResultSize(value),
+				"'maxResponseSize' param value must be positive and less than {hbase.scan.max.result.size} value");
+		return convertToBytes(value, getResultSizeUnit());
+	}
+
+	/**
+	 * Checks if is allowable result size.
+	 * 
+	 * @param input
+	 *            the input
+	 * @return true, if is allowable result size
+	 */
+	public static boolean isAllowableResultSize(long input) {
+		if (input <= 0 || input > getMaxResultSize()) {
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Returns the configured default result size in bytes.
+	 * 
+	 * @return long
+	 */
+	public static long getDefaultResultSize() {
+		float value = ConfigurationUtil.getConfiguration().getFloat(
+				"hbase.scan.default.result.size");
+		return convertToBytes(value, getResultSizeUnit());
+	}
+
+	/**
+	 * Returns the configured max result size in bytes.
+	 * 
+	 * @return long
+	 */
+	public static long getMaxResultSize() {
+		float value = ConfigurationUtil.getConfiguration().getFloat(
+				"hbase.scan.max.result.size");
+		return convertToBytes(value, getResultSizeUnit());
+	}
+
+	/**
+	 * Returns the configured max row size in bytes.
+	 * 
+	 * @return long
+	 */
+	public static long getMaxRowSize() {
+		float maxRowSize = ConfigurationUtil.getConfiguration().getFloat(
+				"hbase.table.max.row.size");
+		return convertToBytes(maxRowSize, getRowSizeUnit());
+	}
+
+	/**
+	 * Gets the result size unit.
+	 * 
+	 * @return the result size unit
+	 */
+	public static SizeUnit getResultSizeUnit() {
+		return SizeUnit.valueOf(ConfigurationUtil.getConfiguration()
+				.getString("hbase.scan.result.size.unit"));
+	}
+
+	/**
+	 * Gets the row size unit.
+	 * 
+	 * @return the row size unit
+	 */
+	public static SizeUnit getRowSizeUnit() {
+		return SizeUnit.valueOf(ConfigurationUtil.getConfiguration()
+				.getString("hbase.table.row.size.unit"));
+	}
+
+	/**
+	 * Gets the connection retry limit.
+	 * 
+	 * @return the connection retry limit
+	 */
+	public static int getConnectionRetryLimit() {
+		return ConfigurationUtil.getConfiguration().getInt(
+				"hbase.hconnection.retries.number",
+				DEFAULT_HCONNECTION_RETRY_LIMIT);
+	}
+
+	/**
+	 * Checks if is default include reverse traffic.
+	 * 
+	 * @return true, if is default include reverse traffic
+	 */
+	public static boolean isDefaultIncludeReverseTraffic() {
+		return ConfigurationUtil.getConfiguration().getBoolean(
+				"pcaps.include.reverse.traffic");
+	}
+
+	/**
+	 * Gets the table name.
+	 * 
+	 * @return the table name
+	 */
+	public static byte[] getTableName() {
+		return Bytes.toBytes(ConfigurationUtil.getConfiguration().getString(
+				"hbase.table.name"));
+	}
+
+	/**
+	 * Gets the column family.
+	 * 
+	 * @return the column family
+	 */
+	public static byte[] getColumnFamily() {
+		return Bytes.toBytes(ConfigurationUtil.getConfiguration().getString(
+				"hbase.table.column.family"));
+	}
+
+	/**
+	 * Gets the column qualifier.
+	 * 
+	 * @return the column qualifier
+	 */
+	public static byte[] getColumnQualifier() {
+		return Bytes.toBytes(ConfigurationUtil.getConfiguration().getString(
+				"hbase.table.column.qualifier"));
+	}
+
+	/**
+	 * Gets the max versions.
+	 * 
+	 * @return the max versions
+	 */
+	public static int getMaxVersions() {
+		return ConfigurationUtil.getConfiguration().getInt(
+				"hbase.table.column.maxVersions");
+	}
+
+	/**
+	 * Gets the configured tokens in rowkey.
+	 * 
+	 * @return the configured tokens in rowkey
+	 */
+	public static int getConfiguredTokensInRowkey() {
+		return ConfigurationUtil.getConfiguration().getInt(
+				"hbase.table.row.key.tokens");
+	}
+
+	/**
+	 * Gets the minimum tokens in inputkey.
+	 * 
+	 * @return the minimum tokens in inputkey
+	 */
+	public static int getMinimumTokensInInputkey() {
+		return ConfigurationUtil.getConfiguration().getInt(
+				"rest.api.input.key.min.tokens");
+	}
+
+	/**
+	 * Gets the appending token digits.
+	 * 
+	 * @return the appending token digits
+	 */
+	public static int getAppendingTokenDigits() {
+		return ConfigurationUtil.getConfiguration().getInt(
+				"hbase.table.row.key.token.appending.digits");
+	}
+
+	/**
+	 * Convert to bytes.
+	 * 
+	 * @param value
+	 *            the value
+	 * @param unit
+	 *            the unit
+	 * @return the long
+	 */
+	public static long convertToBytes(float value, SizeUnit unit) {
+		if (SizeUnit.KB == unit) {
+			return (long) (value * 1024);
+		}
+		if (SizeUnit.MB == unit) {
+			return (long) (value * 1024 * 1024);
+		}
+		return (long) value;
+	}
+
+	/**
+	 * The main method.
+	 * 
+	 * @param args
+	 *            the arguments
+	 */
+	public static void main(String[] args) {
+		long r1 = getMaxRowSize();
+		System.out.println("getMaxRowSizeInBytes = " + r1);
+		long r2 = getMaxResultSize();
+		System.out.println("getMaxAllowableResultSizeInBytes = " + r2);
+
+		SizeUnit u1 = getRowSizeUnit();
+		System.out.println("getMaxRowSizeUnit = " + u1.toString());
+		SizeUnit u2 = getResultSizeUnit();
+		System.out.println("getMaxAllowableResultsSizeUnit = " + u2.toString());
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigConstants.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigConstants.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigConstants.java
new file mode 100644
index 0000000..826bdda
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigConstants.java
@@ -0,0 +1,40 @@
+package com.cisco.opensoc.hbase.client;
+
+/**
+ * HBase configuration properties.
+ * 
+ * @author Sayi
+ */
+public class HBaseConfigConstants {
+
+  /** The Constant HBASE_ZOOKEEPER_QUORUM. */
+  public static final String HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+
+  /** The Constant HBASE_ZOOKEEPER_CLIENT_PORT. */
+  public static final String HBASE_ZOOKEEPER_CLIENT_PORT = "hbase.zookeeper.clientPort";
+
+  /** The Constant HBASE_ZOOKEEPER_SESSION_TIMEOUT. */
+  public static final String HBASE_ZOOKEEPER_SESSION_TIMEOUT = "zookeeper.session.timeout";
+
+  /** The Constant HBASE_ZOOKEEPER_RECOVERY_RETRY. */
+  public static final String HBASE_ZOOKEEPER_RECOVERY_RETRY = "zookeeper.recovery.retry";
+
+  /** The Constant HBASE_CLIENT_RETRIES_NUMBER. */
+  public static final String HBASE_CLIENT_RETRIES_NUMBER = "hbase.client.retries.number";
+
+  /** The delimeter. */
+  String delimeter = "-";
+
+  /** The regex. */
+  String regex = "\\-";
+
+  /** The Constant PCAP_KEY_DELIMETER. */
+  public static final String PCAP_KEY_DELIMETER = "-";
+
+  /** The Constant START_KEY. */
+  public static final String START_KEY = "startKey";
+
+  /** The Constant END_KEY. */
+  public static final String END_KEY = "endKey";
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigurationUtil.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigurationUtil.java
new file mode 100644
index 0000000..c92a3e4
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/HBaseConfigurationUtil.java
@@ -0,0 +1,165 @@
+/**
+ * 
+ */
+package com.cisco.opensoc.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.log4j.Logger;
+import org.mortbay.log.Log;
+
+/**
+ * Utility class which creates HConnection instance when the first request is
+ * received and registers a shut down hook which closes the connection when the
+ * JVM exits. Creates new connection to the cluster only if the existing
+ * connection is closed for unknown reasons. Also creates Configuration with
+ * HBase resources using configuration properties.
+ * 
+ * @author Sayi
+ * 
+ */
+public class HBaseConfigurationUtil {
+
+  /** The Constant LOGGER. */
+  private static final Logger LOGGER = Logger
+      .getLogger(HBaseConfigurationUtil.class);
+
+  /** Configuration which holds all HBase properties. */
+  private static Configuration config;
+
+  /**
+   * A cluster connection which knows how to find master node and locate regions
+   * on the cluster.
+   */
+  private static HConnection clusterConnection = null;
+
+  /**
+   * Creates HConnection instance when the first request is received and returns
+   * the same instance for all subsequent requests if the connection is still
+   * open.
+   * 
+   * @return HConnection instance
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public static HConnection getConnection() throws IOException {
+    if (!connectionAvailable()) {
+      synchronized (HBaseConfigurationUtil.class) {
+        createClusterConncetion();
+      }
+    }
+    return clusterConnection;
+  }
+
+  /**
+   * Creates the cluster conncetion.
+   * 
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  private static void createClusterConncetion() throws IOException {
+    try {
+      if (connectionAvailable()) {
+        return;
+      }
+      clusterConnection = HConnectionManager.createConnection(read());
+      addShutdownHook();
+      System.out.println("Created HConnection and added shutDownHook");
+    } catch (IOException e) {
+      LOGGER
+          .error(
+              "Exception occurred while creating HConnection using HConnectionManager",
+              e);
+      throw e;
+    }
+  }
+
+  /**
+   * Connection available.
+   * 
+   * @return true, if successful
+   */
+  private static boolean connectionAvailable() {
+    if (clusterConnection == null) {
+      System.out.println("clusterConnection=" + clusterConnection);
+      return false;
+    }
+    System.out.println("clusterConnection.isClosed()="
+        + clusterConnection.isClosed());
+    return clusterConnection != null && !clusterConnection.isClosed();
+  }
+
+  /**
+   * Adds the shutdown hook.
+   */
+  private static void addShutdownHook() {
+    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+      public void run() {
+        System.out
+            .println("Executing ShutdownHook HBaseConfigurationUtil : Closing HConnection");
+        try {
+          clusterConnection.close();
+        } catch (IOException e) {
+          Log.debug("Caught ignorable exception ", e);
+        }
+      }
+    }, "HBaseConfigurationUtilShutDown"));
+  }
+
+  /**
+   * Closes the underlying connection to cluster; ignores if any exception is
+   * thrown.
+   */
+  public static void closeConnection() {
+    if (clusterConnection != null) {
+      try {
+        clusterConnection.close();
+      } catch (IOException e) {
+        Log.debug("Caught ignorable exception ", e);
+      }
+    }
+  }
+
+  /**
+   * This method creates Configuration with HBase resources using configuration
+   * properties. The same Configuration object will be used to communicate with
+   * all HBase tables;
+   * 
+   * @return Configuration object
+   */
+  public static Configuration read() {
+    if (config == null) {
+      synchronized (HBaseConfigurationUtil.class) {
+        if (config == null) {
+          config = HBaseConfiguration.create();
+
+          config.set(
+              HBaseConfigConstants.HBASE_ZOOKEEPER_QUORUM,
+              ConfigurationUtil.getConfiguration().getString(
+                  "hbase.zookeeper.quorum"));
+          config.set(
+              HBaseConfigConstants.HBASE_ZOOKEEPER_CLIENT_PORT,
+              ConfigurationUtil.getConfiguration().getString(
+                  "hbase.zookeeper.clientPort"));
+          config.set(
+              HBaseConfigConstants.HBASE_CLIENT_RETRIES_NUMBER,
+              ConfigurationUtil.getConfiguration().getString(
+                  "hbase.client.retries.number"));
+          config.set(
+              HBaseConfigConstants.HBASE_ZOOKEEPER_SESSION_TIMEOUT,
+              ConfigurationUtil.getConfiguration().getString(
+                  "zookeeper.session.timeout"));
+          config.set(
+              HBaseConfigConstants.HBASE_ZOOKEEPER_RECOVERY_RETRY,
+              ConfigurationUtil.getConfiguration().getString(
+                  "zookeeper.recovery.retry"));
+        }
+      }
+    }
+    return config;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapGetter.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapGetter.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapGetter.java
new file mode 100644
index 0000000..7dd9c1e
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapGetter.java
@@ -0,0 +1,88 @@
+/**
+ * 
+ */
+package com.cisco.opensoc.hbase.client;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * interface to all 'keys' based pcaps fetching methods.
+ * 
+ * @author Sayi
+ */
+public interface IPcapGetter {
+
+  /**
+   * Gets the pcaps for the input list of keys and lastRowKey.
+   * 
+   * @param keys
+   *          the list of keys for which pcaps are to be retrieved
+   * @param lastRowKey
+   *          last row key from the previous partial response
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps. The value is set to '0' if the caller sends negative value
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps. The value is set to Long.MAX_VALUE if the caller sends
+   *          negative value. 'endTime' must be greater than the 'startTime'.
+   * @param includeReverseTraffic
+   *          indicates whether or not to include pcaps from the reverse traffic
+   * @param includeDuplicateLastRow
+   *          indicates whether or not to include the last row from the previous
+   *          partial response
+   * @param maxResultSize
+   *          the max result size
+   * @return PcapsResponse with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public PcapsResponse getPcaps(List<String> keys, String lastRowKey,
+      long startTime, long endTime, boolean includeReverseTraffic,
+      boolean includeDuplicateLastRow, long maxResultSize) throws IOException;
+
+  /**
+   * Gets the pcaps for the input key.
+   * 
+   * @param key
+   *          the key for which pcaps is to be retrieved.
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps. The value is set to '0' if the caller sends negative value
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps.The value is set to Long.MAX_VALUE if the caller sends
+   *          negative value. 'endTime' must be greater than the 'startTime'.
+   * @param includeReverseTraffic
+   *          indicates whether or not to include pcaps from the reverse traffic
+   * @return PcapsResponse with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public PcapsResponse getPcaps(String key, long startTime, long endTime,
+      boolean includeReverseTraffic) throws IOException;
+
+  /**
+   * Gets the pcaps for the input list of keys.
+   * 
+   * @param keys
+   *          the list of keys for which pcaps are to be retrieved.
+   * @return PcapsResponse with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public PcapsResponse getPcaps(List<String> keys) throws IOException;
+
+  /**
+   * Gets the pcaps for the input key.
+   * 
+   * @param key
+   *          the key for which pcaps is to be retrieved.
+   * @return PcapsResponse with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public PcapsResponse getPcaps(String key) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapReceiver.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapReceiver.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapReceiver.java
new file mode 100644
index 0000000..a06ba6e
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapReceiver.java
@@ -0,0 +1,109 @@
+package com.cisco.opensoc.hbase.client;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestParam;
+
+/**
+ * Single point of entry for all REST calls. Exposes methods to fetch pcaps for
+ * the given list of keys or range of keys and optional start time and end time.
+ * If the caller doesn't provide start time and end time, all pcaps from
+ * beginning of the time to until now are returned.
+ * 
+ * @author Sayi
+ * 
+ */
+public interface IPcapReceiver {
+
+  /**
+   * Gets the pcaps for the given list of keys and optional startTime and
+   * endTime.
+   * 
+   * @param keys
+   *          the list of keys for which pcaps are to be retrieved
+   * @param lastRowKey
+   *          last row key from the previous partial response
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps.
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps. The default value is set to Long.MAX_VALUE. 'endTime' must
+   *          be greater than the 'startTime'.
+   * @param includeReverseTraffic
+   *          indicates whether or not to include pcaps from the reverse traffic
+   * @param includeDuplicateLastRow
+   *          indicates whether or not to include the last row from the previous
+   *          partial response
+   * @param maxResponseSize
+   *          indicates the maximum response size in MegaBytes. User needs to
+   *          pass positive value and must be less than 60 (MB)
+   * @return byte array with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public ResponseEntity<byte[]> getPcapsByKeys(@RequestParam List<String> keys,
+      @RequestParam String lastRowKey, @RequestParam long startTime,
+      @RequestParam long endTime, @RequestParam boolean includeReverseTraffic,
+      @RequestParam boolean includeDuplicateLastRow,
+      @RequestParam String maxResponseSize) throws IOException;
+
+  /**
+   * get pcaps for a given key range.
+   * 
+   * @param startKey
+   *          the start key of a key range for which pcaps are to be retrieved
+   * @param endKey
+   *          the end key of a key range for which pcaps are to be retrieved
+   * @param maxResponseSize
+   *          indicates the maximum response size in MegaBytes. User needs to
+   *          pass positive value and must be less than 60 (MB)
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps.
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps. 'endTime' must be greater than the 'startTime'.
+   * @return byte array with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public ResponseEntity<byte[]> getPcapsByKeyRange(
+      @RequestParam String startKey, @RequestParam String endKey,
+      @RequestParam String maxResponseSize, @RequestParam long startTime,
+      @RequestParam long endTime) throws IOException;
+
+  /**
+   * get pcaps for the given identifiers.
+   * 
+   * @param srcIp
+   *          source ip address
+   * @param destIp
+   *          destination ip address
+   * @param protocol
+   *          network protocol
+   * @param srcPort
+   *          source port
+   * @param destPort
+   *          destination port
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps.
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps. 'endTime' must be greater than the 'startTime'.
+   * @param includeReverseTraffic
+   *          indicates whether or not to include pcaps from the reverse traffic
+   * @return byte array with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public ResponseEntity<byte[]> getPcapsByIdentifiers(
+      @RequestParam String srcIp, @RequestParam String destIp,
+      @RequestParam String protocol, @RequestParam String srcPort,
+      @RequestParam String destPort, @RequestParam long startTime,
+      @RequestParam long endTime, @RequestParam boolean includeReverseTraffic)
+      throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapScanner.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapScanner.java b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapScanner.java
new file mode 100644
index 0000000..c8c19ef
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-PCAP_Reconstruction/hbase/src/main/java/com/cisco/opensoc/hbase/client/IPcapScanner.java
@@ -0,0 +1,49 @@
+package com.cisco.opensoc.hbase.client;
+
+import java.io.IOException;
+
+/**
+ * The Interface for all pcaps fetching methods based on key range.
+ */
+public interface IPcapScanner {
+
+  /**
+   * Gets the pcaps for between startKey (inclusive) and endKey (exclusive).
+   * 
+   * @param startKey
+   *          the start key of a key range for which pcaps is to be retrieved.
+   * @param endKey
+   *          the end key of a key range for which pcaps is to be retrieved.
+   * @param maxResponseSize
+   *          indicates the maximum response size in MegaBytes(MB). User needs
+   *          to pass positive value and must be less than 60 (MB)
+   * @param startTime
+   *          the start time in system milliseconds to be used to filter the
+   *          pcaps. The value is set to '0' if the caller sends negative value
+   * @param endTime
+   *          the end time in system milliseconds to be used to filter the
+   *          pcaps. The value is set Long.MAX_VALUE if the caller sends
+   *          negative value
+   * @return byte array with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public byte[] getPcaps(String startKey, String endKey, long maxResponseSize,
+      long startTime, long endTime) throws IOException;
+
+  /**
+   * Gets the pcaps for between startKey (inclusive) and endKey (exclusive).
+   * 
+   * @param startKey
+   *          the start key (inclusive) of a key range for which pcaps is to be
+   *          retrieved.
+   * @param endKey
+   *          the end key (exclusive) of a key range for which pcaps is to be
+   *          retrieved.
+   * @return byte array with all matching pcaps merged together
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  public byte[] getPcaps(String startKey, String endKey) throws IOException;
+
+}


Mime
View raw message