metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [1/7] incubator-metron git commit: METRON-427 Create Ambari Management Pack for Metron Installation closes apache/incubator-metron#266
Date Mon, 26 Sep 2016 14:05:04 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 6834e146d -> 125dbef1e


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
index 05291f1..18d3b8d 100644
--- a/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
@@ -48,7 +48,7 @@ configurations:
       dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
       dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode")
}}'
   - hadoop-env:
-      namenode_heapsize: 1024
+      namenode_heapsize: 2048
       dtnode_heapsize: 1024
   - hbase-env:
       hbase_regionserver_heapsize: 1024

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
index 8b24735..429981e 100644
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -39,7 +39,7 @@ configurations:
       dataDir: '{{ zookeeper_data_dir }}'
   - hadoop-env:
       hadoop_heapsize: 1024
-      namenode_heapsize: 512
+      namenode_heapsize: 2048
       dtnode_heapsize: 512
       namenode_opt_permsize: 128m
   - hbase-env:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
index 72b2bba..a976d00 100644
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
@@ -42,7 +42,7 @@ configurations:
   - zoo.cfg:
       dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
   - hadoop-env:
-      namenode_heapsize: 1024
+      namenode_heapsize: 2048
       dtnode_heapsize: 1024
   - hbase-env:
       hbase_regionserver_heapsize: 1024

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
index 4358370..b412ed8 100644
--- a/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
+++ b/metron-platform/metron-enrichment/src/main/assembly/assembly.xml
@@ -43,6 +43,18 @@
       <filtered>true</filtered>
     </fileSet>
     <fileSet>
+      <directory>${project.basedir}/src/main/resources/ddl</directory>
+      <outputDirectory>/ddl</outputDirectory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>**/*.formatted</exclude>
+        <exclude>**/*.filtered</exclude>
+      </excludes>
+      <fileMode>0644</fileMode>
+      <lineEnding>unix</lineEnding>
+      <filtered>true</filtered>
+    </fileSet>
+    <fileSet>
       <directory>${project.basedir}/src/main/flux</directory>
       <outputDirectory>/flux</outputDirectory>
       <useDefaultExcludes>true</useDefaultExcludes>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
index 5b1bf9b..e10af77 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
@@ -45,7 +45,7 @@ public class GeoAdapter extends JdbcAdapter {
   public JSONObject enrich(CacheKey value) {
     JSONObject enriched = new JSONObject();
     if(!resetConnectionIfNecessary()) {
-      _LOG.error("Enrichment failure, cannot maintain a connection to JDBC.  Please check
connection.  In the meantime, I'm not enriching.");
+      _LOG.error("GEO Enrichment failure, cannot maintain a connection to JDBC.  Please check
connection.  In the meantime, I'm not enriching.");
       return enriched;
     }
     try {
@@ -78,9 +78,10 @@ public class GeoAdapter extends JdbcAdapter {
       }
       resultSet.close();
     } catch (Exception e) {
-      _LOG.error("Enrichment failure: " + e.getMessage(), e);
+      _LOG.error("GEO Enrichment failure: " + e.getMessage(), e);
       return new JSONObject();
     }
+    _LOG.trace("GEO Enrichment success: ", enriched);
     return enriched;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
index d2af3cc..240531c 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/simplehbase/SimpleHBaseAdapter.java
@@ -99,6 +99,7 @@ public class SimpleHBaseAdapter implements EnrichmentAdapter<CacheKey>,Serializa
         throw new RuntimeException("Unable to retrieve value: " + e.getMessage(), e);
       }
     }
+    _LOG.trace("SimpleHBaseAdapter succeeded:", enriched);
     return enriched;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
index 9fa7363..4950ddb 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java
@@ -27,6 +27,8 @@ import org.apache.metron.common.stellar.StellarProcessor;
 import org.apache.metron.enrichment.bolt.CacheKey;
 import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
 import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
 import java.util.Map;
@@ -35,6 +37,7 @@ import java.util.function.Function;
 import static org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.STELLAR_CONTEXT_CONF;
 
 public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable {
+  protected static final Logger _LOG = LoggerFactory.getLogger(StellarAdapter.class);
 
   private enum EnrichmentType implements Function<SensorEnrichmentConfig, ConfigHandler>{
     ENRICHMENT(config -> config.getEnrichment().getEnrichmentConfigs().get("stellar"))
@@ -78,6 +81,7 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
     Map<String, Object> globalConfig = value.getConfig().getConfiguration();
     Map<String, Object> sensorConfig = value.getConfig().getEnrichment().getConfig();
     if(handler == null) {
+        _LOG.trace("Stellar ConfigHandler is null.");
       return new JSONObject();
     }
     Map<String, Object> message = value.getValue(Map.class);
@@ -102,7 +106,9 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable
         }
       }
     }
-    return new JSONObject(message);
+    JSONObject enriched = new JSONObject(message);
+    _LOG.trace("Stellar Enrichment Success: " + enriched);
+    return enriched;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
index cb7ebe0..3f16b55 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/threatintel/ThreatIntelAdapter.java
@@ -89,16 +89,17 @@ public class ThreatIntelAdapter implements EnrichmentAdapter<CacheKey>,Serializa
           String enrichmentType = enrichmentTypes.get(i++);
           if (isThreat) {
             enriched.put(enrichmentType, "alert");
-            _LOG.trace("Enriched value => " + enriched);
+            _LOG.trace("Theat Intel Enriched value => ", enriched);
           }
         }
       }
       catch(IOException e) {
         _LOG.error("Unable to retrieve value: " + e.getMessage(), e);
         initializeAdapter();
-        throw new RuntimeException("Unable to retrieve value", e);
+        throw new RuntimeException("Theat Intel Unable to retrieve value", e);
       }
     }
+    _LOG.trace("Threat Intel Enrichment Success:", enriched);
     return enriched;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
index 2eb4b0a..e0c96a2 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/JoinBolt.java
@@ -115,6 +115,7 @@ public abstract class JoinBolt<V> extends ConfiguredEnrichmentBolt
{
                       );
         cache.invalidate(key);
         collector.ack(tuple);
+        LOG.trace("Emitted message for key: {}", key);
       } else {
         cache.put(key, streamMessageMap);
         if(LOG.isDebugEnabled()) {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
index 7ff8739..bb0bee5 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelJoinBolt.java
@@ -89,6 +89,7 @@ public class ThreatIntelJoinBolt extends EnrichmentJoinBolt {
   @Override
   public JSONObject joinMessages(Map<String, JSONObject> streamMessageMap) {
     JSONObject ret = super.joinMessages(streamMessageMap);
+    LOG.trace("Received joined messages: {}", ret);
     boolean isAlert = ret.containsKey("is_alert");
     if(!isAlert) {
       for (Object key : ret.keySet()) {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/resources/ddl/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE  `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned
NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY  (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE  `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region`
char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude`
double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default
NULL,PRIMARY KEY
+  (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv'  into table `blocks`  fields
terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv'  into table `location` 
fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+  BEGIN
+    DECLARE ipn INTEGER UNSIGNED;
+    DECLARE locID_var INTEGER;
+    IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+    END IF;
+    SET ipn = INET_ATON(ip);
+    SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start
FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >=
ipn;
+    RETURN locID_var;
+  END
+$$
+DELIMITER ;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
index 91e8446..c4f4150 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/bolt/BulkMessageWriterBolt.java
@@ -101,6 +101,7 @@ public class BulkMessageWriterBolt extends ConfiguredEnrichmentBolt {
                            , bulkMessageWriter
                            , configurationTransformation.apply(new EnrichmentWriterConfiguration(getConfigurations()))
                            );
+      LOG.trace("Writing enrichment message: {}", message);
     }
     catch(Exception e) {
       throw new RuntimeException("This should have been caught in the writerComponent.  If
you see this, file a JIRA", e);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/125dbef1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 659a467..ce0b177 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,6 +27,7 @@
     <modules>
             <module>metron-analytics</module>
             <module>metron-platform</module>
+            <module>metron-deployment</module>
     </modules>
     <properties>
         <twitter>@ApacheMetron</twitter>
@@ -201,8 +202,9 @@
                         <exclude>metron-ui/lib/public/css/font-awesome.min.css</exclude>
                         <exclude>metron-ui/lib/public/font/**</exclude>
                         <exclude>metron-ui/node_modules/**</exclude>
-			<!-- pickle file - binary format -->
+                        <!-- pickle file - binary format -->
                         <exclude>**/packaging/ambari/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/*.p</exclude>
+                        <exclude>**/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard/dashboard.p</exclude>
                     </excludes>
                 </configuration>
             </plugin>


Mime
View raw message