eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m.@apache.org
Subject [25/47] incubator-eagle git commit: [EAGLE-305] 306, 308 & 313: Fix several application manager minor bugs & update the doc
Date Mon, 25 Jul 2016 09:37:02 GMT
[EAGLE-305] 306, 308 & 313: Fix several application manager minor bugs & update the
doc

https://issues.apache.org/jira/browse/EAGLE-305
https://issues.apache.org/jira/browse/EAGLE-306
https://issues.apache.org/jira/browse/EAGLE-308
https://issues.apache.org/jira/browse/EAGLE-313

Author: Zhao, Qingwen <qingwzhao@ebay.com>
Author: jiljiang <jiljiang@ebay.com>
Author: Daniel <Daniel.Zhou@dataguise.com>

Closes #206 from qingwen220/EAGLE-305.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ffd5fddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ffd5fddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ffd5fddd

Branch: refs/heads/master
Commit: ffd5fddd357effae7aaf2384a2a9b6d479f9c930
Parents: fd39e6e
Author: Zhao, Qingwen <qingwzhao@ebay.com>
Authored: Mon May 30 14:53:03 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Mon May 30 14:53:03 2016 +0800

----------------------------------------------------------------------
 .../src/main/bin/eagle-topology-init.sh         |  2 +
 .../ApplicationManagementResource.java          | 16 +++--
 .../application/dao/ApplicationManagerDAO.java  |  1 +
 .../dao/ApplicationManagerDaoImpl.java          | 18 +++++-
 .../impl/StormExecutionPlatform.scala           |  2 +-
 .../tutorial/application_manager_tutorial.md    | 68 +++++++++++---------
 .../app/public/feature/topology/controller.js   |  2 +-
 .../src/main/webapp/app/public/js/app.config.js |  3 +-
 8 files changed, 73 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-assembly/src/main/bin/eagle-topology-init.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh
index be38a0d..dc9dd59 100755
--- a/eagle-assembly/src/main/bin/eagle-topology-init.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh
@@ -64,6 +64,8 @@ curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H
'Conten
 
 curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"
-d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"metrics"},"description":"Metrics dashboard","version":"v0.3.0"}]'
 
+curl -silent -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=FeatureDescService"
-d '[{"prefix":"eagleFeatureDesc","tags":{"feature":"topology"},"description":"Application
topology management feature","version":"v0.4.0"}]'
+
 
 ## AlertStreamService: alert streams generated from data source
 echo ""

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
index 779fbe5..6e4521d 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/ApplicationManagementResource.java
@@ -21,6 +21,7 @@ package org.apache.eagle.service.application;
 
 import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.policy.common.Constants;
 import org.apache.eagle.service.application.dao.ApplicationManagerDAO;
 import org.apache.eagle.service.application.dao.ApplicationManagerDaoImpl;
 import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
@@ -28,10 +29,7 @@ import org.apache.eagle.service.application.entity.TopologyOperationEntity;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.type.TypeFactory;
 
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
+import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 import java.io.IOException;
 import java.io.InputStream;
@@ -98,4 +96,14 @@ public class ApplicationManagementResource {
         ObjectMapper objectMapper = new ObjectMapper();
         return objectMapper.readValue(inputStream, TypeFactory.defaultInstance().constructCollectionType(LinkedList.class,
TopologyOperationEntity.class));
     }
+
+    @Path("topology")
+    @DELETE
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public GenericServiceAPIResponseEntity deleteTopology(@QueryParam("topology") String
topology) {
+        return dao.deleteTopology(topology);
+    }
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
index 189370b..dfa261b 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDAO.java
@@ -29,4 +29,5 @@ public interface ApplicationManagerDAO {
     String loadTopologyExecutionStatus(String site, String application, String topology);
     int loadTopologyOperationsInRunning(String site, String application, String topology)
throws Exception;
     GenericServiceAPIResponseEntity createOperation(List<TopologyOperationEntity> entities)
throws Exception;
+    GenericServiceAPIResponseEntity deleteTopology(String topology);
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
index e21f624..4881cf4 100644
--- a/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
+++ b/eagle-core/eagle-application-management/eagle-application-service/src/main/java/org/apache/eagle/service/application/dao/ApplicationManagerDaoImpl.java
@@ -18,11 +18,9 @@
 
 package org.apache.eagle.service.application.dao;
 
-
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
 import org.apache.eagle.policy.common.Constants;
 import org.apache.eagle.service.application.entity.TopologyExecutionEntity;
-import org.apache.eagle.service.application.entity.TopologyExecutionStatus;
 import org.apache.eagle.service.application.entity.TopologyOperationEntity;
 import org.apache.eagle.service.generic.GenericEntityServiceResource;
 import org.slf4j.Logger;
@@ -73,5 +71,21 @@ public class ApplicationManagerDaoImpl implements ApplicationManagerDAO
{
         return response;
     }
 
+    @Override
+    public GenericServiceAPIResponseEntity deleteTopology(String topology) {
+        String topologyQuery = Constants.TOPOLOGY_DESCRIPTION_SERVICE_ENDPOINT_NAME+ "[@topology=\""
+ topology + "\"]{*}";
+        String executionQuery = Constants.TOPOLOGY_EXECUTION_SERVICE_ENDPOINT_NAME + "[@topology=\""
+ topology + "\"]{*}";
+        int pageSize = Integer.MAX_VALUE;
+
+        GenericServiceAPIResponseEntity response = resource.deleteByQuery(topologyQuery,
null, null, pageSize, null, false, false, 0L, 0, true, 0, null, false);
+        if(response.isSuccess()) {
+            response = resource.deleteByQuery(executionQuery, null, null, pageSize, null,
false, false, 0L, 0, true, 0, null, false);
+        }
+        if(!response.isSuccess()) {
+            LOG.error(response.getException());
+        }
+        return response;
+    }
+
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
b/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
index 5b1bb48..af4cafa 100644
--- a/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
+++ b/eagle-core/eagle-application-management/eagle-stream-application-manager/src/main/scala/org/apache/eagle/stream/application/impl/StormExecutionPlatform.scala
@@ -166,7 +166,7 @@ class StormExecutionPlatform extends ExecutionPlatform {
         case None =>
           topologyExecution.setStatus(TopologyExecutionStatus.STOPPED)
           topologyExecution.setUrl("")
-          topologyExecution.setDescription(s"Fail to find topology: $name")
+          topologyExecution.setDescription("")
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-docs/tutorial/application_manager_tutorial.md
----------------------------------------------------------------------
diff --git a/eagle-docs/tutorial/application_manager_tutorial.md b/eagle-docs/tutorial/application_manager_tutorial.md
index 4690428..4eccfa4 100644
--- a/eagle-docs/tutorial/application_manager_tutorial.md
+++ b/eagle-docs/tutorial/application_manager_tutorial.md
@@ -15,55 +15,63 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 
-> Application manager aims to manage topology status on EAGLE UI. Users can easily start/start
topologies remotely or locally without shell commands.
+> Application manager aims to manage topology status on EAGLE UI. Users can easily start/start
topologies remotely or locally without any shell commands. At the same, it should be capable
to sync the latest status of topologies on the execution platform (e.g., storm cluster). 
 
-This tutorial will go through all parts of appliaction manager and then give an example to
use it. 
+This tutorial will go through all parts of application manager and then give an example to
use it. 
 
 ### Design
-Briefly speaking, Application manager consists of a deamon scheduler and the application
manager. The scheduler loads user operations(start/stop), and the applicaiton manager is responsible
to execute these operations. For more details, please refer to [here](https://cwiki.apache.org/confluence/display/EAG/Application+Management)
+Application manager consists of a daemon scheduler and an execution module. The scheduler
periodically loads user operations(start/stop) from database, and the execution module executes
these operations. For more details, please refer to [here](https://cwiki.apache.org/confluence/display/EAG/Application+Management)
 
-### Manual
+### Configurations
+The configuration file `eagle-scheduler.conf` defines scheduler parameters, execution platform
settings and parts of default topology configuration.
 
-#### Step 1: configure the scheduler
-The configuration file `eagle-scheduler.conf` defines scheduler parameters, topology execution
platform settings and parts of topology settings. Here are some important ones:
+* **Scheduler properties**
 
-* `envContextConfig.env`
+    **appCommandLoaderEnabled**: enable application manager. **TODO**: change it to true.
<br />
+    **appCommandLoaderIntervalSecs**: defines the interval of the scheduler loads commands.
Default value is 1 second.  <br />
+    **appHealthCheckIntervalSecs**: define the interval of health check, which tries to sync
the topology execution status to Eagle. <br /><br />
 
-   application execution platform. Default value is storm
+* **Execution platform properties**
    
-* `envContextConfig.url`
-   
-   execution platform http url. Default is "http://sandbox.hortonworks.com:8744"
+    **envContextConfig.env**: application execution platform. Default value is storm.  <br
/>
+    **envContextConfig.url**: execution platform http url. Default is "http://sandbox.hortonworks.com:8744".
 <br />
+    **envContextConfig.nimbusHost**: storm nimbus host. Default is "sandbox.hortonworks.com".
 <br />
+    **envContextConfig.nimbusThriftPort**: default is 6627.  
+    **envContextConfig.jarFile**: storm fat jar path. **TODO**: change "/dir-to-jar/eagle-topology-0.3.0-incubating-assembly.jar"
to your own jar path. <br /><br />
+
+* **Topology default properties**
+    
+    Some default topology properties are defined here. 
+    
+Note: these configurations can be overridden in the topology configurations, which is shown
below. The only difference is to add a prefix `.app`. For example, 'app.envContextConfig.jarFile'
is to override 'envContextConfig.jarFile' in eagle-schedule.conf
    
-* `envContextConfig.nimbusHost`
   
-   Storm nimbus host. Default is "sandbox.hortonworks.com"
-   
-* `envContextConfig.nimbusThriftPort`
-   
-   Default is 6627
-   
-* `envContextConfig.jarFile`
+#### Manual
+
+1. Editing eagle-scheduler.conf, and start Eagle service
 
-   Storm fat jar path. The only setting users must specify "/dir-to-jar/eagle-topology-0.3.0-incubating-assembly.jar"
+        # enable application manager         
+        appCommandLoaderEnabled=true
+        # provide jar path
+        envContextConfig.jarFile=/path-to-jar
    
-After the configuration is ready, start Eagle service `bin/eagle-service.sh start`. 
-  
-#### Step 2: add topologies on UI
-1. First of all, go to admin page 
+    For more configurations, please back to **Configuration**. <br />
+    After the configuration is ready, start Eagle service `bin/eagle-service.sh start`. 
+   
+2. Go to admin page 
    ![admin-page](/images/appManager/admin-page.png)
    ![topology-monitor](/images/appManager/topology-monitor.png)
     
-2. Go to management page, and create a topology description. There are three required fields
+3. Go to management page, and create a topology description. There are three required fields
     * name: topology name
     * type: topology type [CLASS, DYNAMIC]
-    * execution entry: either the class which implement interface TopologyExecutable or eagle
[DSL](https://github.com/apache/incubator-eagle/blob/master/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf)
based topology definition
+    * execution entry: either the class which implements interface TopologyExecutable or
eagle [DSL](https://github.com/apache/incubator-eagle/blob/master/eagle-assembly/src/main/conf/sandbox-hadoopjmx-pipeline.conf)
based topology definition
    ![topology-description](/images/appManager/topology-description.png)
    
-3. Back to monitoring page, and choose the site/application to deploy the topology 
+4. Back to monitoring page, and choose the site/application to deploy the topology 
    ![topology-execution](/images/appManager/topology-execution.png)
    
-4. Go to site page, and edit site->application and add some new configurations. Blow are
some example configurations for [site=sandbox, applicatoin=hbaseSecurityLog]
+5. Go to site page, and edit site->application and add some new configurations. Blow are
some example configurations for [site=sandbox, applicatoin=hbaseSecurityLog]
    `These configurations have a higher priority than those in eagle-scheduler.conf`
    
            classification.hbase.zookeeper.property.clientPort=2181
@@ -97,11 +105,11 @@ After the configuration is ready, start Eagle service `bin/eagle-service.sh
star
    ![topology-configuration-1](/images/appManager/topology-configuration-1.png)
    ![topology-configuration-2](/images/appManager/topology-configuration-2.png)
    
-5. Go to monitoring page, and start topologies
+6. Go to monitoring page, and start topologies
    ![start-topology-1](/images/appManager/start-topology-1.png)
    ![start-topology-2](/images/appManager/start-topology-2.png)
    
-6. stop topologies on monitoring page
+7. stop topologies on monitoring page
    ![stop-topology-1](/images/appManager/stop-topology-1.png)
    ![stop-topology-2](/images/appManager/stop-topology-2.png)
    ![stop-topology-3](/images/appManager/stop-topology-3.png)

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js b/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
index 5504a77..94886c9 100644
--- a/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
+++ b/eagle-webservice/src/main/webapp/app/public/feature/topology/controller.js
@@ -247,7 +247,7 @@
 
 		$scope.deleteTopology = function (topology) {
 			UI.deleteConfirm(topology.tags.topology).then(null, null, function(holder) {
-				Entities.deleteEntities("TopologyDescriptionService", {topology: topology.tags.topology})._promise.then(function()
{
+				Entities.delete("TopologyDescriptionService", {topology: topology.tags.topology})._promise.then(function()
{
 					holder.closeFunc();
 					common.array.remove(topology, $scope.topologyList);
 				});

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ffd5fddd/eagle-webservice/src/main/webapp/app/public/js/app.config.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/js/app.config.js b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
index bd6d62c..ec30ccd 100644
--- a/eagle-webservice/src/main/webapp/app/public/js/app.config.js
+++ b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
@@ -43,7 +43,8 @@
 			DELETE_HOOK: {
 				FeatureDescService: 'rest/module/feature?feature=${feature}',
 				ApplicationDescService: 'rest/module/application?application=${application}',
-				SiteDescService: 'rest/module/site?site=${site}'
+				SiteDescService: 'rest/module/site?site=${site}',
+				TopologyDescriptionService: 'rest/app/topology?topology=${topology}'
 			},
 			UPDATE_HOOK: {
 				SiteDescService: 'rest/module/siteApplication'


Mime
View raw message