eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m.@apache.org
Subject incubator-eagle git commit: [EAGLE-685] adjust hadoop-queue application configuration parameters according to classified style
Date Wed, 26 Oct 2016 08:46:51 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 8e00cdfe0 -> 278b0961c


[EAGLE-685] adjust hadoop-queue application configuration parameters according to classified
style

application configuration has been divided into a few sections based on parameter type.
need to adjust code to fit for the change.

Author: anyway1021 <mw@apache.org>

Closes #565 from anyway1021/adjust-params.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/278b0961
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/278b0961
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/278b0961

Branch: refs/heads/master
Commit: 278b0961c9c198ce68dc0596b7f9b71ffd89b038
Parents: 8e00cdf
Author: anyway1021 <mw@apache.org>
Authored: Wed Oct 26 16:46:24 2016 +0800
Committer: anyway1021 <mw@apache.org>
Committed: Wed Oct 26 16:46:24 2016 +0800

----------------------------------------------------------------------
 .../common/config/EagleConfigConstants.java     |  3 ++
 .../service/client/EagleServiceConnector.java   | 22 +++++++---
 .../queue/HadoopQueueRunningAppConfig.java      | 14 +++---
 .../queue/common/HadoopClusterConstants.java    |  1 +
 .../crawler/ClusterMetricsParseListener.java    |  2 +-
 .../storm/HadoopQueueMetricPersistBolt.java     |  4 +-
 ...doop.queue.HadoopQueueRunningAppProvider.xml | 46 ++++++--------------
 .../src/main/resources/application.conf         | 18 +++-----
 .../src/test/resources/application.conf         | 18 +++-----
 9 files changed, 55 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
index 693041d..7c0c257 100644
--- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
@@ -18,8 +18,11 @@ package org.apache.eagle.common.config;
 
 public final class EagleConfigConstants {
     public static final String SERVICE_ENV = "service.env";
+    public static final String SERVICE = "service";
     public static final String SERVICE_HOST = "service.host";
     public static final String SERVICE_PORT = "service.port";
+    public static final String SERVICE_USERNAME = "service.username";
+    public static final String SERVICE_PASSWORD = "service.password";
     public static final String SERVICE_HBASE_ZOOKEEPER_QUORUM = "storage.hbase.zookeeperQuorum";
     public static final String SERVICE_HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT = "storage.hbase.zookeeperPropertyClientPort";
     public static final String SERVICE_ZOOKEEPER_ZNODE_PARENT = "storage.hbase.zookeeperZnodeParent";

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
index 2b2b1ac..a34f552 100644
--- a/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
+++ b/eagle-core/eagle-query/eagle-client-base/src/main/java/org/apache/eagle/service/client/EagleServiceConnector.java
@@ -22,6 +22,8 @@ import org.apache.eagle.common.config.EagleConfigConstants;
 
 import java.io.Serializable;
 
+import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_ENV;
+
 /**
  * Some common codes to enable DAO through eagle service including service host/post, credential
population etc.
  */
@@ -59,12 +61,20 @@ public class EagleServiceConnector implements Serializable{
     }
 
     public EagleServiceConnector(Config config){
-        this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "." +
EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
-        this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.PORT);
-        if (config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.USERNAME) &&
-                config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.PASSWORD)) {
-            this.username = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.USERNAME);
-            this.password = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.PASSWORD);
+        if (config.hasPath(EagleConfigConstants.SERVICE)) {
+            this.eagleServiceHost = config.getString(EagleConfigConstants.SERVICE_HOST);
+            this.eagleServicePort = config.getInt(EagleConfigConstants.SERVICE_PORT);
+            if (config.hasPath(EagleConfigConstants.SERVICE_USERNAME) && config.hasPath(EagleConfigConstants.SERVICE_PASSWORD))
{
+                this.username = config.getString(EagleConfigConstants.SERVICE_USERNAME);
+                this.password = config.getString(EagleConfigConstants.SERVICE_PASSWORD);
+            }
+        } else {
+            this.eagleServiceHost = config.getString(EagleConfigConstants.EAGLE_PROPS + "."
+ EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.HOST);
+            this.eagleServicePort = config.getInt(EagleConfigConstants.EAGLE_PROPS + "."
+ EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PORT);
+            if (config.hasPath(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.EAGLE_SERVICE
+ "." + EagleConfigConstants.USERNAME) && config.hasPath(EagleConfigConstants.EAGLE_PROPS
+ "." + EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD)) {
+                this.username = config.getString(EagleConfigConstants.EAGLE_PROPS + "." +
EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.USERNAME);
+                this.password = config.getString(EagleConfigConstants.EAGLE_PROPS + "." +
EagleConfigConstants.EAGLE_SERVICE + "." + EagleConfigConstants.PASSWORD);
+            }
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
index 5d5d736..e370ce9 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppConfig.java
@@ -37,9 +37,7 @@ public class HadoopQueueRunningAppConfig implements Serializable {
     }
 
     public static class Topology implements Serializable {
-        public boolean localMode;
         public int numOfParserTasks;
-        public String name;
     }
 
     public static class DataSourceConfig implements Serializable {
@@ -82,17 +80,15 @@ public class HadoopQueueRunningAppConfig implements Serializable {
     private void init(Config config) {
         this.config = config;
 
-        this.topology.localMode = config.getBoolean("topology.localMode");
         this.topology.numOfParserTasks = config.getInt("topology.numOfParserTasks");
-        this.topology.name = config.getString("topology.name");
 
         this.dataSourceConfig.rMEndPoints = config.getString("dataSourceConfig.rMEndPoints");
         this.dataSourceConfig.fetchIntervalSec = config.getString("dataSourceConfig.fetchIntervalSec");
 
-        this.eagleProps.site = config.getString("eagleProps.site");
-        this.eagleProps.eagleService.host = config.getString("eagleProps.eagleService.host");
-        this.eagleProps.eagleService.port = config.getInt("eagleProps.eagleService.port");
-        this.eagleProps.eagleService.username = config.getString("eagleProps.eagleService.username");
-        this.eagleProps.eagleService.password = config.getString("eagleProps.eagleService.password");
+        this.eagleProps.site = config.getString("siteId");
+        this.eagleProps.eagleService.host = config.getString("service.host");
+        this.eagleProps.eagleService.port = config.getInt("service.port");
+        this.eagleProps.eagleService.username = config.getString("service.username");
+        this.eagleProps.eagleService.password = config.getString("service.password");
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
index c924455..4a63343 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/common/HadoopClusterConstants.java
@@ -45,6 +45,7 @@ public class HadoopClusterConstants {
         public static final String HADOOP_CLUSTER_AVAILABLE_MEMORY = "hadoop.cluster.availablememory";
         public static final String HADOOP_CLUSTER_RESERVED_MEMORY = "hadoop.cluster.reservedmemory";
         public static final String HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES = "hadoop.cluster.totalvirtualcores";
+        public static final String HADOOP_CLUSTER_ALLOCATED_VIRTUAL_CORES = "hadoop.cluster.allocatedvcores";
 
         // metrics from scheduler info
         public static final String HADOOP_CLUSTER_CAPACITY = "hadoop.cluster.capacity";

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
index ed78465..d26e295 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/crawler/ClusterMetricsParseListener.java
@@ -83,7 +83,7 @@ public class ClusterMetricsParseListener {
         createMetric(MetricName.HADOOP_CLUSTER_TOTAL_MEMORY, currentTimestamp, metrics.getTotalMB(),
AggregateFunc.MAX);
         createMetric(MetricName.HADOOP_CLUSTER_AVAILABLE_MEMORY, currentTimestamp, metrics.getAvailableMB(),
AggregateFunc.AVG);
         createMetric(MetricName.HADOOP_CLUSTER_RESERVED_MEMORY, currentTimestamp, metrics.getReservedMB(),
AggregateFunc.AVG);
-        createMetric(MetricName.HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES, currentTimestamp, metrics.getTotalVirtualCores(),
AggregateFunc.MAX);
+        createMetric(MetricName.HADOOP_CLUSTER_TOTAL_VIRTUAL_CORES, currentTimestamp, metrics.getAllocatedVirtualCores(),
AggregateFunc.MAX);
     }
 
     public void flush() {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
index 4edf27d..3609184 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
@@ -28,7 +28,6 @@ import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
 import org.apache.eagle.hadoop.queue.model.scheduler.RunningQueueAPIEntity;
 import org.apache.eagle.log.entity.GenericMetricEntity;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
-import org.apache.eagle.service.client.EagleServiceConnector;
 import org.apache.eagle.service.client.IEagleServiceClient;
 import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
 import org.slf4j.Logger;
@@ -51,7 +50,8 @@ public class HadoopQueueMetricPersistBolt extends BaseRichBolt {
 
     @Override
     public void prepare(Map stormConf, TopologyContext context, OutputCollector collector)
{
-        this.client = new EagleServiceClientImpl(new EagleServiceConnector(this.config.getConfig()));
+        HadoopQueueRunningAppConfig.EagleProps.EagleService eagleService = config.eagleProps.eagleService;
+        this.client = new EagleServiceClientImpl(eagleService.host, eagleService.port, eagleService.username,
eagleService.password);
         this.collector = collector;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index c5e9ff9..43092d2 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -23,50 +23,30 @@
   <configuration>
     <!-- org.apache.eagle.hadoop.queue.HadoopQueueRunningAppConfig -->
     <property>
-      <name>workers</name>
-      <displayName>storm worker number</displayName>
-      <value>4</value>
+      <name>dataSourceConfig.rMEndPoints</name>
+      <displayName>Resource Manager End Point</displayName>
+      <description>end point of resource manager, e.g. http://sandbox.hortonworks.com:8088/</description>
+      <value>http://sandbox.hortonworks.com:8088/</value>
+      <required>true</required>
     </property>
     <property>
-      <name>topology.localMode</name>
-      <value>true</value>
+      <name>workers</name>
+      <displayName>Storm Worker Number</displayName>
+      <description>the number of storm worker</description>
+      <value>1</value>
     </property>
     <property>
       <name>topology.numOfParserTasks</name>
+      <displayName>Parallel Tasks Per Bolt</displayName>
+      <description>the number of tasks that should be assigned to execute a bolt</description>
       <value>2</value>
     </property>
     <property>
-      <name>topology.name</name>
-      <value>sandbox-running-queue-topology</value>
-    </property>
-    <property>
-      <name>dataSourceConfig.rMEndPoints</name>
-      <value>http://sandbox.hortonworks.com:8088/</value>
-    </property>
-    <property>
       <name>dataSourceConfig.fetchIntervalSec</name>
+      <displayName>Fetch Interval in Seconds</displayName>
+      <description>fetching interval in seconds</description>
       <value>10</value>
     </property>
-    <property>
-      <name>eagleProps.site</name>
-      <value>sandbox</value>
-    </property>
-    <property>
-      <name>eagleProps.eagleService.host</name>
-      <value>localhost</value>
-    </property>
-    <property>
-      <name>eagleProps.eagleService.port</name>
-      <value>9099</value>
-    </property>
-    <property>
-      <name>eagleProps.eagleService.username</name>
-      <value>admin</value>
-    </property>
-    <property>
-      <name>eagleProps.eagleService.password</name>
-      <value>secret</value>
-    </property>
   </configuration>
   <docs>
     <install>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
index 807bd5b..e5c9b81 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
@@ -15,25 +15,21 @@
 
 {
   "topology" : {
-    "localMode" : true,
     "numOfParserTasks" : 2,
-    "name" : "sandbox-running-queue-topology",
   },
   "dataSourceConfig": {
     "rMEndPoints" : "http://sandbox.hortonworks.com:8088/",
     "fetchIntervalSec": "10"
   },
-  "eagleProps" : {
-    "site": "sandbox",
-    "eagleService": {
-      "host": "localhost",
-      "port": 9099
-      "username": "admin",
-      "password": "secret"
-    }
+  "siteId": "sandbox"
+  "service": {
+    "host": "localhost"
+    "port": "9099"
+    "username": "admin"
+    "password": "secret"
   }
   "appId":"hadoopQueueMonitorJob",
   "mode":"LOCAL",
   application.storm.nimbusHost=localhost,
-  "workers":4,
+  "workers":1,
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/278b0961/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
index 807bd5b..e5c9b81 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
@@ -15,25 +15,21 @@
 
 {
   "topology" : {
-    "localMode" : true,
     "numOfParserTasks" : 2,
-    "name" : "sandbox-running-queue-topology",
   },
   "dataSourceConfig": {
     "rMEndPoints" : "http://sandbox.hortonworks.com:8088/",
     "fetchIntervalSec": "10"
   },
-  "eagleProps" : {
-    "site": "sandbox",
-    "eagleService": {
-      "host": "localhost",
-      "port": 9099
-      "username": "admin",
-      "password": "secret"
-    }
+  "siteId": "sandbox"
+  "service": {
+    "host": "localhost"
+    "port": "9099"
+    "username": "admin"
+    "password": "secret"
   }
   "appId":"hadoopQueueMonitorJob",
   "mode":"LOCAL",
   application.storm.nimbusHost=localhost,
-  "workers":4,
+  "workers":1,
 }


Mime
View raw message