metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [23/26] incubator-metron git commit: replace opensoc-steaming version 0.4BETA with 0.6BETA 8e7a6b4ad9febbc4ea47ba7810c42cc94d4dee37
Date Thu, 17 Dec 2015 20:46:25 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractConfigTest.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractConfigTest.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractConfigTest.java
new file mode 100644
index 0000000..7484e16
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractConfigTest.java
@@ -0,0 +1,299 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.opensoc.helpers.topology.SettingsLoader;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractConfigTest</code> is
+ * an abstract base class for implementing JUnit tests that need to use
+ * config to connect to ZooKeeper and HBase. The <code>setup</code> method will
attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Oct 10, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractConfigTest  extends AbstractTestContext{
+         /**
+         * The configPath.
+         */
+        protected String configPath=null;   
+        
+        /**
+        * The configName.
+        */
+       protected String configName=null;           
+
+        /**
+         * The config.
+         */
+        private Configuration config=null;
+        
+         /**
+         * The settings.
+         */
+        Map<String, String> settings=null;       
+
+        /**
+         * The schemaJsonString.
+         */
+        private String schemaJsonString = null;
+        /**
+         * Any Object for mavenMode
+         * @parameter
+         *   expression="${mode}"
+         *   default-value="local"
+         */
+         private Object mode="local";        
+
+        /**
+         * Constructs a new <code>AbstractConfigTest</code> instance.
+         * @throws Exception 
+         */
+        public AbstractConfigTest() throws Exception {
+            super.setUp();
+        }
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @param name the name of the test case.
+         */
+        public AbstractConfigTest(String name) {
+            super(name);
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#setUp()
+         */
+        protected void setUp(String configName) throws Exception {
+            super.setUp();
+            this.setConfigPath("src/test/resources/config/"+getClass().getSimpleName()+".config");
+            try {
+                this.setConfig(new PropertiesConfiguration(this.getConfigPath()));
+               
+                Map configOptions= SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config,
configName+"=");
+                this.setSettings(SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config,
configName + "."));
+                this.getSettings().put(configName, (String) configOptions.get(configName));
+            } catch (ConfigurationException e) {
+                fail("Config not found !!"+e);
+                e.printStackTrace();
+            }               
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#tearDown()
+         */
+        @Override
+        protected void tearDown() throws Exception {
+
+        }
+
+        
+         /**
+         * validateJsonData
+         * @param jsonSchema
+         * @param jsonData
+         * @return
+         * @throws Exception
+         */
+         
+        protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+            throws Exception {
+    
+            final JsonNode d = JsonLoader.fromString(jsonData);
+            final JsonNode s = JsonLoader.fromString(jsonSchema);
+    
+            final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+            JsonValidator v = factory.getValidator();
+    
+            ProcessingReport report = v.validate(s, d);
+            System.out.println(report);
+            
+            return report.toString().contains("success");
+        }
+        
+        protected String readSchemaFromFile(URL schema_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    schema_url.getFile()));
+            String line;
+            StringBuilder sb = new StringBuilder();
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                sb.append(line);
+            }
+            br.close();
+
+            String schema_string = sb.toString().replaceAll("\n", "");
+            schema_string = schema_string.replaceAll(" ", "");
+
+            System.out.println("Read in schema: " + schema_string);
+
+            return schema_string;
+        }        
+  
+        protected String[] readTestDataFromFile(String test_data_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    new File(test_data_url)));
+            ArrayList<String> inputDataLines = new ArrayList<String>();
+           
+            String line;
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                inputDataLines.add(line.toString().replaceAll("\n", ""));
+            }
+            br.close();
+            String[] inputData = new String[inputDataLines.size()];
+            inputData = inputDataLines.toArray(inputData);
+
+            return inputData;
+        }          
+       /**
+        * Skip Tests
+        */
+       public boolean skipTests(Object mode){
+           if(mode.toString().equals("local")){
+               return true;
+           }else {
+               return false;
+           }
+       }
+       
+       /**
+        * Returns the mode.
+        * @return the mode.
+        */
+       
+       public Object getMode() {
+           return mode;
+       }
+
+       /**
+        * Sets the mode.
+        * @param mode the mode.
+        */
+       
+       public void setMode(Object mode) {
+       
+           this.mode = mode;
+       }
+
+    
+         /**
+         * @param readSchemaFromFile
+         */
+        public void setSchemaJsonString(String schemaJsonString) {
+            this.schemaJsonString=schemaJsonString;
+        }
+
+    
+         /**
+         * @return
+         */
+        public String getSchemaJsonString() {
+           return this.schemaJsonString;
+        }
+        
+        /**
+        * Returns the configPath.
+        * @return the configPath.
+        */
+       public String getConfigPath() {
+           return configPath;
+       }
+    
+       /**
+        * Sets the configPath.
+        * @param configPath the configPath.
+        */
+       public void setConfigPath(String configPath) {
+           this.configPath = configPath;
+       }    
+       /**
+        * Returns the config.
+        * @return the config.
+        */
+       
+       public Configuration getConfig() {
+           return config;
+       }
+    
+       /**
+        * Sets the config.
+        * @param config the config.
+        */
+       
+       public void setConfig(Configuration config) {
+       
+           this.config = config;
+       }  
+       /**
+        * Returns the settings.
+        * @return the settings.
+        */
+       
+       public Map<String, String> getSettings() {
+           return settings;
+       }
+
+       /**
+        * Sets the settings.
+        * @param settings the settings.
+        */
+       
+       public void setSettings(Map<String, String> settings) {
+           this.settings = settings;
+       }   
+       /**
+       * Returns the configName.
+       * @return the configName.
+       */
+      public String getConfigName() {
+          return configName;
+      }
+
+      /**
+       * Sets the configName.
+       * @param configName the configName.
+       */
+      public void setConfigName(String configName) {  
+          this.configName = configName;
+      }       
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractSchemaTest.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractSchemaTest.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractSchemaTest.java
new file mode 100644
index 0000000..670d7f9
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractSchemaTest.java
@@ -0,0 +1,198 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.test;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.net.URL;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+
+ /**
+ * <ul>
+ * <li>Title: </li>
+ * <li>Description: The class <code>AbstractSchemaTest</code> is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * Json Schema. The <code>setup</code> method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.</li>
+ * <li>Created: Aug 7, 2014</li>
+ * </ul>
+ * @version $Revision: 1.1 $
+ */
+public class AbstractSchemaTest  extends AbstractConfigTest{
+        
+        
+         /**
+         * The schemaJsonString.
+         */
+        private String schemaJsonString = null;
+        /**
+         * Any Object for mavenMode
+         * @parameter
+         *   expression="${mode}"
+         *   default-value="local"
+         */
+         private Object mode="local";        
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @throws Exception 
+         */
+        public AbstractSchemaTest() throws Exception {
+            super.setUp();
+        }
+
+        /**
+         * Constructs a new <code>AbstractTestContext</code> instance.
+         * @param name the name of the test case.
+         */
+        public AbstractSchemaTest(String name) {
+            super(name);
+            try{
+                if(System.getProperty("mode")!=null){
+                    setMode(System.getProperty("mode") );                
+                }else
+                {
+                    setMode("local");
+                }
+            }catch(Exception ex){
+                setMode("local");
+            }            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#setUp()
+         */
+        @Override
+        protected void setUp() throws Exception {
+            super.setUp();
+            
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see junit.framework.TestCase#tearDown()
+         */
+        @Override
+        protected void tearDown() throws Exception {
+
+        }
+
+        
+         /**
+         * validateJsonData
+         * @param jsonSchema
+         * @param jsonData
+         * @return
+         * @throws Exception
+         */
+         
+        protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+            throws Exception {
+    
+            final JsonNode d = JsonLoader.fromString(jsonData);
+            final JsonNode s = JsonLoader.fromString(jsonSchema);
+    
+            final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+            JsonValidator v = factory.getValidator();
+    
+            ProcessingReport report = v.validate(s, d);
+            System.out.println(report);
+            
+            return report.toString().contains("success");
+        }
+        
+        protected String readSchemaFromFile(URL schema_url) throws Exception {
+            BufferedReader br = new BufferedReader(new FileReader(
+                    schema_url.getFile()));
+            String line;
+            StringBuilder sb = new StringBuilder();
+            while ((line = br.readLine()) != null) {
+                System.out.println(line);
+                sb.append(line);
+            }
+            br.close();
+
+            String schema_string = sb.toString().replaceAll("\n", "");
+            schema_string = schema_string.replaceAll(" ", "");
+
+            System.out.println("Read in schema: " + schema_string);
+
+            return schema_string;
+
+        }        
+        
+       /**
+        * Skip Tests
+        */
+       public boolean skipTests(Object mode){
+           if(mode.toString().equals("local")){
+               return true;
+           }else {
+               return false;
+           }
+       }
+       
+       /**
+        * Returns the mode.
+        * @return the mode.
+        */
+       
+       public Object getMode() {
+           return mode;
+       }
+
+       /**
+        * Sets the mode.
+        * @param mode the mode.
+        */
+       
+       public void setMode(Object mode) {
+       
+           this.mode = mode;
+       }
+
+    
+     /**
+     
+     * @param readSchemaFromFile
+     */
+     
+    public void setSchemaJsonString(String schemaJsonString) {
+        this.schemaJsonString=schemaJsonString;
+    }
+
+    
+     /**
+     
+     * @return
+     */
+     
+    public String getSchemaJsonString() {
+       return this.schemaJsonString;
+    }
+     
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
index 7f7f34a..ea5b04f 100644
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
@@ -52,7 +52,7 @@ public class AbstractTestContext  extends TestCase{
          * Any Object for mavenMode
          * @parameter
          *   expression="${mode}"
-         *   default-value="local"
+         *   default-value="global"
          */
          private Object mode="local";        
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
deleted file mode 100644
index 47061b6..0000000
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.opensoc.test;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.json.simple.JSONObject;
-
-import com.opensoc.ise.parser.ISEParser;
-import com.opensoc.ise.parser.ParseException;
-
-public class ISEParserTest {
-
-	public static void main(String[] args) throws ParseException, IOException {
-	}
-
-}
-
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
new file mode 100644
index 0000000..2dbcd95
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
@@ -0,0 +1,137 @@
+package com.opensoc.tldextractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class BasicTldExtractor implements Serializable {
+	private static final long serialVersionUID = -7440226111118873815L;
+	private StringBuilder sb = new StringBuilder();
+
+    private Pattern pattern;
+    
+    /**
+    * The inputFile.
+    */
+   private String inputFile ="effective_tld_names.dat";
+   
+   public BasicTldExtractor(String filePath) {
+       this.inputFile=filePath;
+       this.init();
+   }
+   
+	public BasicTldExtractor() {
+      this.init();
+	}
+
+	private void init(){
+	       try {
+	            ArrayList<String> terms = new ArrayList<String>();
+
+	            
+	            BufferedReader br = new BufferedReader(new InputStreamReader(
+	                    getClass().getClassLoader().getResourceAsStream(inputFile)));
+	            String s = null;
+	            while ((s = br.readLine()) != null) {
+	                s = s.trim();
+	                if (s.length() == 0 || s.startsWith("//") || s.startsWith("!"))
+	                    continue;
+	                terms.add(s);
+	            }
+	            Collections.sort(terms, new StringLengthComparator());
+	            for (String t : terms)
+	                add(t);
+	            compile();
+	            br.close();
+	        } catch (IOException e) {
+	            throw new IllegalStateException(e);
+	        }
+	}
+	protected void add(String s) {
+		s = s.replace(".", "\\.");
+		s = "\\." + s;
+		if (s.startsWith("*")) {
+			s = s.replace("*", ".+");
+			sb.append(s).append("|");
+		} else {
+			sb.append(s).append("|");
+		}
+	}
+
+	public void compile() {
+		if (sb.length() > 0)
+			sb.deleteCharAt(sb.length() - 1);
+		sb.insert(0, "[^.]+?(");
+		sb.append(")$");
+		pattern = Pattern.compile(sb.toString());
+		sb = null;
+	}
+
+	public String extract2LD(String host) {
+		Matcher m = pattern.matcher(host);
+		if (m.find()) {
+			return m.group(0);
+		}
+		return null;
+	}
+
+	public String extractTLD(String host) {
+		Matcher m = pattern.matcher(host);
+		if (m.find()) {
+			return m.group(1);
+		}
+		return null;
+	}
+
+	public static class StringLengthComparator implements Comparator<String> {
+		public int compare(String s1, String s2) {
+			if (s1.length() > s2.length())
+				return -1;
+			if (s1.length() < s2.length())
+				return 1;
+			return 0;
+		}
+	}
+    /**
+     * Returns the sb.
+     * @return the sb.
+     */
+    
+    public StringBuilder getSb() {
+        return sb;
+    }
+
+    /**
+     * Sets the sb.
+     * @param sb the sb.
+     */
+    
+    public void setSb(StringBuilder sb) {
+    
+        this.sb = sb;
+    }
+    /**
+     * Returns the inputFile.
+     * @return the inputFile.
+     */
+    
+    public String getInputFile() {
+        return inputFile;
+    }
+
+    /**
+     * Sets the inputFile.
+     * @param inputFile the inputFile.
+     */
+    
+    public void setInputFile(String inputFile) {
+    
+        this.inputFile = inputFile;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/test/BasicTldExtractorTest.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/test/BasicTldExtractorTest.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/test/BasicTldExtractorTest.java
new file mode 100644
index 0000000..03cc065
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/test/BasicTldExtractorTest.java
@@ -0,0 +1,125 @@
+
+ 
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.tldextractor.test;
+
+import com.opensoc.test.AbstractConfigTest;
+import com.opensoc.tldextractor.BasicTldExtractor;
+
+
+ /**
+ * <ul>
+ * <li>Title: Basic TLD Extractor Test</li>
+ * <li>Description: Basic TLD Extractor class test</li>
+ * <li>Created: Feb 26, 2015</li>
+ * </ul>
+ * @author $Author:  $
+ * @version $Revision: 1.1 $
+ */
+public class BasicTldExtractorTest extends AbstractConfigTest {
+     /**
+     * The tldExtractor.
+     */
+     
+    private BasicTldExtractor tldExtractor=null;
+
+    /**
+     * Constructs a new <code>BasicTldExtractorTest</code> instance.
+     * @param name
+     */
+
+    public BasicTldExtractorTest(String name) {
+        super(name);
+    }
+
+    /**
+     
+     * @throws java.lang.Exception
+     */
+    protected static void setUpBeforeClass() throws Exception {
+    }
+
+    /**
+     
+     * @throws java.lang.Exception
+     */
+    protected static void tearDownAfterClass() throws Exception {
+    }
+
+    /* 
+     * (non-Javadoc)
+     * @see junit.framework.TestCase#setUp()
+     */
+
+    protected void setUp() throws Exception {
+        super.setUp("com.opensoc.tldextractor.test.BasicTldExtractorTest");
+        this.tldExtractor=new BasicTldExtractor(this.getConfig().getString("logFile"));
+    } 
+
+    /* 
+     * (non-Javadoc)
+     * @see junit.framework.TestCase#tearDown()
+     */
+
+    protected void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#BasicTldExtractor()}.
+     */
+    public void testBasicTldExtractor() {
+        assertNotNull(this.tldExtractor);
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extract2LD(java.lang.String)}.
+     */
+    public void testExtract2LD() {
+        //fail("Not yet implemented");
+    }
+
+    /**
+     * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extractTLD(java.lang.String)}.
+     */
+    public void testExtractTLD() 
+    {
+        String result = this.tldExtractor.extractTLD("cisco.com");
+        System.out.println("result ="+result);
+    }
+    /**
+     * Returns the tldExtractor.
+     * @return the tldExtractor.
+     */
+    
+    public BasicTldExtractor getTldExtractor() {
+        return tldExtractor;
+    }
+
+    /**
+     * Sets the tldExtractor.
+     * @param tldExtractor the tldExtractor.
+     */
+    
+    public void setTldExtractor(BasicTldExtractor tldExtractor) {
+    
+        this.tldExtractor = tldExtractor;
+    }
+    
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
deleted file mode 100644
index 9f8bae3..0000000
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
+++ /dev/null
@@ -1,186 +0,0 @@
-package com.opensoc.topologyhelpers;
-
-import java.io.File;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-
-public class Cli {
-
-	private String[] args = null;
-	private Options options = new Options();
-
-	private String path = null;
-	private boolean debug = true;
-	private boolean local_mode = true;
-	private boolean generator_spout = false;
-
-	public boolean isGenerator_spout() {
-		return generator_spout;
-	}
-
-	public void setGenerator_spout(boolean generator_spout) {
-		this.generator_spout = generator_spout;
-	}
-
-	public String getPath() {
-		return path;
-	}
-
-	public void setPath(String path) {
-		this.path = path;
-	}
-
-	public boolean isDebug() {
-		return debug;
-	}
-
-	public void setDebug(boolean debug) {
-		this.debug = debug;
-	}
-
-	public boolean isLocal_mode() {
-		return local_mode;
-	}
-
-	public void setLocal_mode(boolean local_mode) {
-		this.local_mode = local_mode;
-	}
-
-	public Cli(String[] args) {
-
-		this.args = args;
-
-		Option help = new Option("h", "Display help menue");
-		options.addOption(help);
-		options.addOption(
-				"config_path",
-				true,
-				"OPTIONAL ARGUMENT [/path/to/configs] Path to configuration folder. If not provided topology
will initialize with default configs");
-		options.addOption(
-				"local_mode",
-				true,
-				"REQUIRED ARGUMENT [true|false] Local mode or cluster mode.  If set to true the topology
will run in local mode.  If set to false the topology will be deployed to Storm nimbus");
-		options.addOption(
-				"debug",
-				true,
-				"OPTIONAL ARGUMENT [true|false] Storm debugging enabled.  Default value is true");
-		options.addOption(
-				"generator_spout",
-				true,
-				"REQUIRED ARGUMENT [true|false] Turn on test generator spout.  Default is set to false.
 If test generator spout is turned on then kafka spout is turned off.  Instead the generator
spout will read telemetry from file and ingest it into a topology");
-	}
-
-	public void parse() {
-		CommandLineParser parser = new BasicParser();
-
-		CommandLine cmd = null;
-		try {
-			cmd = parser.parse(options, args);
-
-			if (cmd.hasOption("h"))
-				help();
-
-			if (cmd.hasOption("local_mode")) {
-
-				String local_value = cmd.getOptionValue("local_mode").trim()
-						.toLowerCase();
-
-				if (local_value.equals("true"))
-					local_mode = true;
-
-				else if (local_value.equals("false"))
-					local_mode = false;
-				else {
-					System.out
-							.println("[OpenSOC] ERROR: Invalid value for local mode");
-					System.out
-							.println("[OpenSOC] ERROR: Using cli argument -local_mode="
-									+ cmd.getOptionValue("local_mode"));
-					help();
-				}
-			} else {
-				System.out
-						.println("[OpenSOC] ERROR: Invalid value for local mode");
-				help();
-			}
-			if (cmd.hasOption("generator_spout")) {
-
-				String local_value = cmd.getOptionValue("generator_spout").trim()
-						.toLowerCase();
-
-				if (local_value.equals("true"))
-					generator_spout = true;
-
-				else if (local_value.equals("false"))
-					generator_spout = false;
-				else {
-					System.out
-							.println("[OpenSOC] ERROR: Invalid value for local generator_spout");
-					System.out
-							.println("[OpenSOC] ERROR: Using cli argument -generator_spout="
-									+ cmd.getOptionValue("generator_spout"));
-					help();
-				}
-			} else {
-				System.out
-						.println("[OpenSOC] ERROR: Invalid value for generator_spout");
-				help();
-			}
-			if (cmd.hasOption("config_path")) {
-
-				path = cmd.getOptionValue("config_path").trim();
-
-				File file = new File(path);
-
-				if (!file.isDirectory() || !file.exists()) {
-					System.out
-							.println("[OpenSOC] ERROR: Invalid settings directory name given");
-					System.out
-							.println("[OpenSOC] ERROR: Using cli argument -config_path="
-									+ cmd.getOptionValue("config_path"));
-					help();
-				}
-			}
-
-			if (cmd.hasOption("debug")) {
-				String debug_value = cmd.getOptionValue("debug");
-
-				if (debug_value.equals("true"))
-					debug = true;
-				else if (debug_value.equals("false"))
-					debug = false;
-				else {
-					System.out
-							.println("[OpenSOC] ERROR: Invalid value for debug_value");
-					System.out
-							.println("[OpenSOC] ERROR: Using cli argument -debug_value="
-									+ cmd.getOptionValue("debug_value"));
-					help();
-				}
-			}
-
-		} catch (ParseException e) {
-			System.out
-					.println("[OpenSOC] ERROR: Failed to parse command line arguments");
-			help();
-		}
-	}
-
-	private void help() {
-		// This prints out some help
-		HelpFormatter formater = new HelpFormatter();
-
-		formater.printHelp("Topology Options:", options);
-
-		System.out
-				.println("[OpenSOC] Example usage: \n storm jar OpenSOC-Topologies-0.3BETA-SNAPSHOT.jar
com.opensoc.topology.Bro -local_mode true -config_path OpenSOC_Configs/ -generator_spout true");
-
-		System.exit(0);
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
deleted file mode 100644
index c21205e..0000000
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.opensoc.topologyhelpers;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-import org.json.simple.JSONObject;
-
-public class ErrorGenerator {
-
-	public static JSONObject generateErrorMessage(String message, String exception)
-	{
-		JSONObject error_message = new JSONObject();
-		
-		error_message.put("time", System.currentTimeMillis());
-		try {
-			error_message.put("hostname", InetAddress.getLocalHost().getHostName());
-		} catch (UnknownHostException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		
-		error_message.put("message", message);
-		error_message.put("exception", exception);
-		
-		return error_message;
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
deleted file mode 100644
index bb2a460..0000000
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
+++ /dev/null
@@ -1,118 +0,0 @@
-package com.opensoc.topologyhelpers;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.configuration.XMLConfiguration;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-
-public class SettingsLoader {
-
-	public static JSONObject loadEnvironmentIdnetifier(String config_path)
-			throws ConfigurationException {
-		Configuration config = new PropertiesConfiguration(config_path);
-
-		String customer = config.getString("customer.id", "unknown");
-		String datacenter = config.getString("datacenter.id", "unknown");
-		String instance = config.getString("instance.id", "unknown");
-
-		JSONObject identifier = new JSONObject();
-		identifier.put("customer", customer);
-		identifier.put("datacenter", datacenter);
-		identifier.put("instance", instance);
-
-		return identifier;
-	}
-
-	public static JSONObject loadTopologyIdnetifier(String config_path)
-			throws ConfigurationException {
-		Configuration config = new PropertiesConfiguration(config_path);
-
-		String topology = config.getString("topology.id", "unknown");
-		String instance = config.getString("instance.id", "unknown");
-
-		JSONObject identifier = new JSONObject();
-		identifier.put("topology", topology);
-		identifier.put("topology_instance", instance);
-
-		return identifier;
-	}
-	
-
-	public static String generateTopologyName(JSONObject env, JSONObject topo) {
-
-		return (env.get("customer") + "_" + env.get("datacenter") + "_"
-				+ env.get("instance") + "_" + topo.get("topology") + "_" + topo.get("topology_instance"));
-	}
-	
-	public static JSONObject generateAlertsIdentifier(JSONObject env, JSONObject topo)
-	{
-		JSONObject identifier = new JSONObject();
-		identifier.put("environment", env);
-		identifier.put("topology", topo);
-		
-		return identifier;
-	}
-
-	public static Map<String, JSONObject> loadRegexAlerts(String config_path)
-			throws ConfigurationException, ParseException {
-		XMLConfiguration alert_rules = new XMLConfiguration();
-		alert_rules.setDelimiterParsingDisabled(true);
-		alert_rules.load(config_path);
-
-		int number_of_rules = alert_rules.getList("rule.pattern").size();
-
-		String[] patterns = alert_rules.getStringArray("rule.pattern");
-		String[] alerts = alert_rules.getStringArray("rule.alert");
-
-		JSONParser pr = new JSONParser();
-		Map<String, JSONObject> rules = new HashMap<String, JSONObject>();
-
-		for (int i = 0; i < patterns.length; i++)
-			rules.put(patterns[i], (JSONObject) pr.parse(alerts[i]));
-
-		return rules;
-	}
-
-	public static Map<String, JSONObject> loadKnownHosts(String config_path)
-			throws ConfigurationException, ParseException {
-		Configuration hosts = new PropertiesConfiguration(config_path);
-
-		Iterator<String> keys = hosts.getKeys();
-		Map<String, JSONObject> known_hosts = new HashMap<String, JSONObject>();
-		JSONParser parser = new JSONParser();
-
-		while (keys.hasNext()) {
-			String key = keys.next().trim();
-			JSONArray value = (JSONArray) parser.parse(hosts.getProperty(key)
-					.toString());
-			known_hosts.put(key, (JSONObject) value.get(0));
-		}
-
-		return known_hosts;
-	}
-
-	public static void printConfigOptions(PropertiesConfiguration config, String path_fragment)
-	{
-		Iterator<String> itr = config.getKeys();
-		
-		while(itr.hasNext())
-		{
-			String key = itr.next();
-			
-			if(key.contains(path_fragment))
-			{
-				
-				System.out.println("[OpenSOC] Key: " + key + " -> " + config.getString(key));
-			}
-		}
-
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/a919cc19/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
b/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
new file mode 100644
index 0000000..6b3dc05
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
@@ -0,0 +1,2 @@
+#BasicTldExtractorConfig
+logFile=effective_tld_names.dat



Mime
View raw message