eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject [2/5] incubator-eagle git commit: enable storm configuration for Eagle
Date Mon, 30 Nov 2015 02:48:59 GMT
enable storm configuration for Eagle


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/86e3d6d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/86e3d6d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/86e3d6d0

Branch: refs/heads/master
Commit: 86e3d6d0d9684c96f5f9a145e96514eb65bc79e3
Parents: 4329ef1
Author: Zhao, Qingwen <qingwzhao@ebay.com>
Authored: Wed Nov 25 20:55:51 2015 +0800
Committer: Zhao, Qingwen <qingwzhao@ebay.com>
Committed: Wed Nov 25 20:55:51 2015 +0800

----------------------------------------------------------------------
 .../datastream/StormTopologyExecutorImpl.scala     | 17 +++++++++++++++++
 .../security/hbase/HbaseAuditLogProcessorMain.java |  4 +---
 .../auditlog/HdfsAuditLogProcessorMain.java        |  4 +---
 .../securitylog/HDFSSecurityLogProcessorMain.java  |  4 +---
 .../jobrunning/HiveJobRunningMonitoringMain.java   |  6 +++---
 .../src/assembly/eagle-topology-assembly.xml       |  4 ++--
 6 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
index 737fe44..48ee9c4 100644
--- a/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
+++ b/eagle-core/eagle-data-process/eagle-stream-process-api/src/main/scala/org/apache/eagle/datastream/StormTopologyExecutorImpl.scala
@@ -16,9 +16,12 @@
  */
 package org.apache.eagle.datastream
 
+import java.io.{FileInputStream, File}
+
 import backtype.storm.generated.StormTopology
 import backtype.storm.utils.Utils
 import backtype.storm.{Config, LocalCluster, StormSubmitter}
+import org.yaml.snakeyaml.Yaml
 import storm.trident.spout.RichSpoutBatchExecutor
 
 case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesafe.config.Config)
extends AbstractTopologyExecutor {
@@ -32,6 +35,20 @@ case class StormTopologyExecutorImpl(topology: StormTopology, config: com.typesa
     conf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384))
     conf.put(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384))
 
+    if(config.hasPath("envContextConfig.stormConfigFile")) {
+      //val inputFileStream = {
+      //  StormTopologyExecutorImpl.getClass.getClassLoader.getResourceAsStream(config.getString("envContextConfig.stormConfigFile"))
+      //}
+      val file = new File(config.getString("envContextConfig.stormConfigFile"))
+      if(file.exists()) {
+        val inputFileStream = new FileInputStream(file)
+        val yaml = new Yaml()
+        val stormConf = yaml.load(inputFileStream).asInstanceOf[java.util.LinkedHashMap[String,
Object]]
+        conf.putAll(stormConf)
+        inputFileStream.close()
+      }
+    }
+
     val topologyName = config.getString("envContextConfig.topologyName")
     if (!localMode) {
       StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology)

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
index 5d6d9f3..1a90cbd 100644
--- a/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hbase-securitylog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HbaseAuditLogProcessorMain {
     private static final Logger LOG = LoggerFactory.getLogger(HbaseAuditLogProcessorMain.class);
 
     public static void main(String[] args) throws Exception{
-        new ConfigOptionParser().load(args);
-        System.setProperty("config.trace", "loads");
-        Config config = ConfigFactory.load();
+        Config config = new ConfigOptionParser().load(args);
 
         LOG.info("Config class: " + config.getClass().getCanonicalName());
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
index ce0c6be..fa712d4 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/java/org/apache/eagle/security/auditlog/HdfsAuditLogProcessorMain.java
@@ -38,9 +38,7 @@ public class HdfsAuditLogProcessorMain {
 	private static final Logger LOG = LoggerFactory.getLogger(HdfsAuditLogProcessorMain.class);
 
 	public static void main(String[] args) throws Exception{
-        new ConfigOptionParser().load(args);
-        System.setProperty("config.trace", "loads");
-        Config config = ConfigFactory.load();
+        Config config = new ConfigOptionParser().load(args);
 
         LOG.info("Config class: " + config.getClass().getCanonicalName());
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
index b514a7c..9238cad 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -33,9 +33,7 @@ public class HDFSSecurityLogProcessorMain {
     private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
 
     public static void main(String[] args) throws Exception{
-        new ConfigOptionParser().load(args);
-        System.setProperty("config.trace", "loads");
-        Config config = ConfigFactory.load();
+        Config config = new ConfigOptionParser().load(args);
 
         LOG.info("Config class: " + config.getClass().getCanonicalName());
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
index 983e871..c3e192a 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/jobrunning/HiveJobRunningMonitoringMain.java
@@ -33,9 +33,9 @@ public class HiveJobRunningMonitoringMain {
 	private static final Logger LOG = LoggerFactory.getLogger(HiveJobRunningMonitoringMain.class);
 
 	public static void main(String[] args) throws Exception{
-        new ConfigOptionParser().load(args);
-        System.setProperty("config.trace", "loads");
-        Config config = ConfigFactory.load();
+        Config config = new ConfigOptionParser().load(args);
+        //System.setProperty("config.trace", "loads");
+        //Config config = ConfigFactory.load();
 
         LOG.info("Config class: " + config.getClass().getCanonicalName());
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/86e3d6d0/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
index 2e34c9b..db8563f 100644
--- a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
+++ b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
@@ -16,8 +16,8 @@
                 <excludes>
                     <exclude>**/application.conf</exclude>
                     <exclude>**/defaults.yaml</exclude>
-                    <exclude>**/storm.yaml</exclude>
-                    <exclude>**/storm.yaml.1</exclude>
+                    <exclude>**/*storm.yaml</exclude>
+                    <exclude>**/*storm.yaml.1</exclude>
                     <exclude>**/log4j.properties</exclude>
                 </excludes>
             </unpackOptions>


Mime
View raw message