eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject eagle git commit: [EAGLE-1016] fix keyword "group" conflict in JDBC storage
Date Fri, 19 May 2017 05:55:35 GMT
Repository: eagle
Updated Branches:
  refs/heads/master 7d39863f4 -> af9b056dc


[EAGLE-1016] fix keyword "group" conflict in JDBC storage

https://issues.apache.org/jira/browse/EAGLE-1016

If you meet the same issue after applying this patch, please drop the old table first.

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #933 from qingwen220/EAGLE-1016.


Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/af9b056d
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/af9b056d
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/af9b056d

Branch: refs/heads/master
Commit: af9b056dc453f46c6a2aeb92d8dd6fe503f1b577
Parents: 7d39863
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Fri May 19 13:55:27 2017 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Fri May 19 13:55:27 2017 +0800

----------------------------------------------------------------------
 eagle-assembly/src/main/conf/eagle.conf         | 26 +++++++++++---------
 .../metadata/model/MetricSchemaEntity.java      |  6 ++---
 .../jdbc/conn/ConnectionConfigFactory.java      | 22 +++++++++++------
 .../system_metric_collector.py                  |  2 +-
 .../scheduler/QueueStructureAPIEntity.java      |  1 +
 .../model/scheduler/RunningQueueAPIEntity.java  |  1 +
 .../mr/historyentity/JobExecutionAPIEntity.java |  1 +
 .../eagle/metric/HadoopMetricMonitorApp.java    |  5 ++--
 .../app/dev/public/js/ctrls/metricCtrl.js       |  2 +-
 .../entity/HBaseServiceTopologyAPIEntity.java   |  1 +
 .../entity/HdfsServiceTopologyAPIEntity.java    |  2 +-
 .../entity/MRServiceTopologyAPIEntity.java      |  1 +
 .../entity/SystemServiceTopologyAPIEntity.java  |  1 +
 13 files changed, 43 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-assembly/src/main/conf/eagle.conf
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/conf/eagle.conf b/eagle-assembly/src/main/conf/eagle.conf
index 496e3d0..3441fe6 100644
--- a/eagle-assembly/src/main/conf/eagle.conf
+++ b/eagle-assembly/src/main/conf/eagle.conf
@@ -40,32 +40,34 @@ zookeeper {
 
 storage {
   # storage type: ["hbase","jdbc"]
-  # default is "hbase"
+  # "hbase" is preferred
   type = "hbase"
 
+  ## for prod env
   hbase {
     autoCreateTable = true
-
     # hbase configuration: hbase.zookeeper.quorum
-    # default is "localhost"
     zookeeperQuorum = "localhost"
-
     # hbase configuration: hbase.zookeeper.property.clientPort
-    # default is 2181
     zookeeperPropertyClientPort = 2181
-
     # hbase configuration: zookeeper.znode.parent
-    # default is "/hbase"
     zookeeperZnodeParent = "/hbase-unsecure"
-
-    # eagle web login profile: [sandbox, default]
-    # default is sandbox
     tableNamePrefixedWithEnvironment = false
-
     # eagle coprocessor enabled or not: [true, false]
-    # default is false
     coprocessorEnabled = false
   }
+
+  ## for dev env, in which JPM is NOT supported
+  jdbc {
+    adapter = "mysql"
+    username = "root"
+    password = ""
+    database = "eagle"
+    driverClass = "com.mysql.jdbc.Driver"
+    connectionUrl = "jdbc:mysql://localhost:3306/eagle"
+    connectionProps = "encoding=UTF-8"
+    connectionMax = 8
+  }
 }
 
 # ---------------------------------------------

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
index 3bd5825..36eef9d 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
@@ -30,12 +30,12 @@ import java.util.List;
 @Service(MetricSchemaEntity.METRIC_SCHEMA_SERVICE)
 @JsonIgnoreProperties(ignoreUnknown = true)
 @TimeSeries(false)
-@Tags({"site","site","group"})
+@Tags({"site","metricName","metricGroup"})
 public class MetricSchemaEntity extends TaggedLogAPIEntity {
     static final String METRIC_SCHEMA_SERVICE = "MetricSchemaService";
-    public static final String METRIC_NAME_TAG = "name";
     public static final String METRIC_SITE_TAG = "site";
-    public static final String METRIC_GROUP_TAG = "group";
+    public static final String METRIC_NAME_TAG = "metricName";
+    public static final String METRIC_GROUP_TAG = "metricGroup";
 
     @Column("a")
     private List<String> dimensionFields;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
index 71f19aa..bb9e5f1 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
@@ -32,21 +32,29 @@ public class ConnectionConfigFactory {
         String username = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_USERNAME);
         String password = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_PASSWORD);
         String connUrl = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_URL);
-        String connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS);
         String adapter = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_ADAPTER);
         String databaseName = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DATABASE);
         String driverClass = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DRIVER_CLASS);
-        String connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE);
+
+
+        String connProps = null;
+        if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_PROPS))
{
+            connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS);
+        }
+
+        String connMaxSize = null;
+        if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_MAX_SIZE))
{
+            connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE);
+        }
 
         ConnectionConfig config = new ConnectionConfig();
         if(username != null) config.setUserName(username);
         if(password != null)config.setPassword(password);
         if(connUrl != null) config.setConnectionUrl(connUrl);
-        if(connProps!=null) config.setConnectionProperties(connProps);
-        if(adapter!=null) config.setAdapter(adapter);
-        if(databaseName!=null) config.setDatabaseName(databaseName);
-        if(driverClass!=null) config.setDriverClassName(driverClass);
-
+        if(connProps != null) config.setConnectionProperties(connProps);
+        if(adapter != null) config.setAdapter(adapter);
+        if(databaseName != null) config.setDatabaseName(databaseName);
+        if(driverClass != null) config.setDriverClassName(driverClass);
 
         if(connMaxSize!=null){
             config.setConnectionMaxActive(Integer.parseInt(connMaxSize));

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-external/hadoop_jmx_collector/system_metric_collector.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/system_metric_collector.py b/eagle-external/hadoop_jmx_collector/system_metric_collector.py
index fba06f2..501798b 100644
--- a/eagle-external/hadoop_jmx_collector/system_metric_collector.py
+++ b/eagle-external/hadoop_jmx_collector/system_metric_collector.py
@@ -224,7 +224,7 @@ class SystemMetricCollector(MetricCollector):
         output = os.popen('sudo ipmitool sdr | grep Temp | grep CPU').readlines()
         for item in output:
             items = re.split("^(CPU\d+)\sTemp\.\s+\|\s+(\d+|\d+\.\d+)\s", item.rstrip())
-            event = self.new_metric("System.CPU")
+            event = self.new_metric("system.cpu")
             event["timestamp"] = int(round(time.time() * 1000))
             event["metric"] = DATA_TYPE + "." + 'cpu.temp'
             event["value"] = items[2]

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
index 72f67bc..fde709d 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
@@ -31,6 +31,7 @@ import java.util.List;
 @Service(HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME)
 @TimeSeries(false)
 @Partition( {"site"})
+@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE})
 public class QueueStructureAPIEntity extends TaggedLogAPIEntity {
     @Column("a")
     private List<String> subQueues;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
index be4ef31..a93bad0 100755
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
@@ -32,6 +32,7 @@ import java.util.List;
 @Service(HadoopClusterConstants.RUNNING_QUEUE_SERVICE_NAME)
 @TimeSeries(true)
 @Partition( {"site"})
+@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE})
 public class RunningQueueAPIEntity extends TaggedLogAPIEntity {
     @Column("a")
     private String state;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
index 746843d..7a649b7 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
@@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
     @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = true),
     @Index(name = "Index_2_jobDefId", columns = { "jobDefId" }, unique = false)
     })
+@Tags({"site", "jobId", "jobName", "jobDefId", "jobType", "user", "queue"})
 public class JobExecutionAPIEntity extends JobBaseAPIEntity {
     @Column("a")
     private String currentState;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
index aa1753f..1761aba 100644
--- a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
+++ b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
@@ -51,15 +51,14 @@ public class HadoopMetricMonitorApp extends StormApplication {
             .siteAs(AppConfigUtils.getSiteId(config))
             .namedByField("metric")
             .eventTimeByField("timestamp")
-            .dimensionFields("host", "group", "site", "device")
+            .dimensionFields("host", "site", "device")
             .granularity(Calendar.SECOND)
             .valueField("value");
         return environment.newApp(config)
             .fromStream("HADOOP_JMX_METRIC_STREAM").transformBy(new CounterToRateFunction(hadoopMetricDescriptor,3,
TimeUnit.SECONDS, ClockWithOffset.INSTANCE))
             .saveAsMetric(hadoopMetricDescriptor)
             .fromStream("SYSTEM_METRIC_STREAM").transformBy(new CounterToRateFunction(systemMetricDescriptor,3,
TimeUnit.SECONDS, ClockWithOffset.INSTANCE))
-            .saveAsMetric(systemMetricDescriptor
-            )
+            .saveAsMetric(systemMetricDescriptor)
             .toTopology();
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
index b310738..e0f0f15 100644
--- a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
+++ b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
@@ -40,7 +40,7 @@
 		$scope.metricList = [$scope.metricName];
 		CompatibleEntity.groups({
 			query: 'MetricSchemaService',
-			groups: 'name',
+			groups: 'metricName',
 			fields: 'count',
 			limit: 9999,
 		})._promise.then(function (res) {

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
index 9d7233e..ce829ed 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 @Prefix("hbaseservicestatus")
 @Service(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME)
 @TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG,
TopologyConstants.ROLE_TAG})
 public class HBaseServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
     @Column("a")
     private String status;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
index 4054cba..1362736 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
@@ -22,13 +22,13 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import org.apache.eagle.log.entity.meta.*;
 import org.apache.eagle.topology.TopologyConstants;
 
-
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 @Table("hadoop_topology")
 @ColumnFamily("f")
 @Prefix("hdfsservicestatus")
 @Service(TopologyConstants.HDFS_INSTANCE_SERVICE_NAME)
 @TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG,
TopologyConstants.ROLE_TAG})
 public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
     @Column("a")
     private String status;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
index fa7e1a4..4c3040e 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 @Prefix("mrservicestatus")
 @Service(TopologyConstants.MR_INSTANCE_SERVICE_NAME)
 @TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG,
TopologyConstants.ROLE_TAG})
 public class MRServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
     @Column("a")
     private String status;

http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
index 119cb46..e441741 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
@@ -29,6 +29,7 @@ import org.apache.eagle.topology.TopologyConstants;
 @Prefix("systemservicestatus")
 @Service(TopologyConstants.SYSTEM_INSTANCE_SERVICE_NAME)
 @TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG,
TopologyConstants.ROLE_TAG})
 public class SystemServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
     @Column("a")
     private String status;


Mime
View raw message