helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swaroop...@apache.org
Subject git commit: added documentation. removed unused imports
Date Wed, 08 May 2013 21:58:05 GMT
Updated Branches:
  refs/heads/master ada408c8e -> 356fd28b4


added documentation. removed unused imports


Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/356fd28b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/356fd28b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/356fd28b

Branch: refs/heads/master
Commit: 356fd28b45127f14fe97cc40ab568e01f339a9b6
Parents: ada408c
Author: Swaroop Jagadish <sjagadis@sjagadis-ld.linkedin.biz>
Authored: Wed May 8 14:57:42 2013 -0700
Committer: Swaroop Jagadish <sjagadis@sjagadis-ld.linkedin.biz>
Committed: Wed May 8 14:57:42 2013 -0700

----------------------------------------------------------------------
 .../apache/helix/taskexecution/TaskCluster.java    |    2 -
 .../helix/taskexecution/TaskExecutionDemo.java     |   26 ++++++++++++--
 2 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/356fd28b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskCluster.java
----------------------------------------------------------------------
diff --git a/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskCluster.java
b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskCluster.java
index 1156038..c49659e 100644
--- a/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskCluster.java
+++ b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskCluster.java
@@ -20,8 +20,6 @@ package org.apache.helix.taskexecution;
  */
 
 import org.apache.helix.ConfigAccessor;
-import org.apache.helix.model.ConfigScope;
-import org.apache.helix.model.builder.ConfigScopeBuilder;
 import org.apache.helix.manager.zk.ZKHelixAdmin;
 import org.apache.helix.manager.zk.ZNRecordSerializer;
 import org.apache.helix.manager.zk.ZkClient;

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/356fd28b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskExecutionDemo.java
----------------------------------------------------------------------
diff --git a/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskExecutionDemo.java
b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskExecutionDemo.java
index b9c9d0d..d63ef63 100644
--- a/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskExecutionDemo.java
+++ b/recipes/task-execution/src/main/java/org/apache/helix/taskexecution/TaskExecutionDemo.java
@@ -32,11 +32,29 @@ import org.apache.helix.HelixManager;
 import org.apache.helix.controller.HelixControllerMain;
 import org.apache.helix.taskexecution.Dag.Node;
 /**
- * 
- *
+ * Demo for execution of task Dag using primitives provided by Helix. This demo sets up a
Dag of tasks for
+ * providing analytics for impression and click events. Each node on the dag has an id and
declares the desired parallelism and 
+ * IDs of the nodes it depends on. When we submit this dag for execution using submitDag(){@link
TaskCluster}, we create a Helix resource for 
+ * each node with number of partitions based on desired parallelism. We use a "OnlineOffline"
state model.
+ * The demo starts NUM_WORKERS workers. Each worker is given a {@link TaskFactory} and {@link
TaskResultStore}. Each worker is assigned a bunch of task 
+ * partitions by Helix and gets state transition messages for the task partitions it is assigned.
When the worker gets a state transition message 
+ * for a task partition, it first checks if all upstream dependencies are satisfied by making
sure that corresponding partitions have transitioned to 
+ * "Online" state. It then creates a Task object using TaskFactory based on resource name
(since task IDs have been mapped to resource names)
+ * The demo has the following steps
+ * <OL>
+ * <LI>Start zookeeper</LI>
+ * <LI>Setup task cluster {@link TaskCluster}</LI>
+ * <LI>Start Helix controller</LI>
+ * <LI> Populate dummy impression and click data </LI>
+ * <LI> Start workers </LI>
+ * <LI> Submit dag </LI>
  */
 public class TaskExecutionDemo {
 
+	private static final int NUM_WORKERS = 10;
+	private static final int NUM_IMP_EVENTS = 10000;
+
+
 	public static void main(String[] args) throws Exception {
 		if (args.length != 3) {
 			System.err
@@ -97,7 +115,7 @@ public class TaskExecutionDemo {
 	private static void populateDummyData(TaskResultStore taskResultStore) throws Exception
{
 		float fraudProbability = 0.01f;
 		float clickProbability = 0.01f;
-		int numImps = 10000; 
+		int numImps = NUM_IMP_EVENTS; 
 		Random rand = new Random();
 		String[] countries = {"US", "CANADA", "UK", "CHINA", "UNKNOWN"};
 		String[] genders = {"M", "F", "UNKNOWN"};
@@ -139,7 +157,7 @@ public class TaskExecutionDemo {
 
 	private static void startWorkers(String zkAddr, String clusterName,
 			TaskFactory taskFactory, TaskResultStore taskResultStore) {
-		int numWorkers = 10;
+		int numWorkers = NUM_WORKERS;
 		Executor executor = Executors.newFixedThreadPool(numWorkers);
 
 		for (int i = 0; i < numWorkers; i++) {


Mime
View raw message