eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject [1/2] incubator-eagle git commit: [EAGLE-350] Running queue metrics monitoring
Date Tue, 12 Jul 2016 03:33:25 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/develop f3e7687e3 -> 9f6fea4ab


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Queues.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Queues.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Queues.java
new file mode 100644
index 0000000..09b4eb0
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Queues.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Queues {
+	public List<Queue> getQueue() {
+		return queue;
+	}
+
+	public void setQueue(List<Queue> queue) {
+		this.queue = queue;
+	}
+
+	private List<Queue> queue;
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/ResourcesUsed.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/ResourcesUsed.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/ResourcesUsed.java
new file mode 100644
index 0000000..cb80ff6
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/ResourcesUsed.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ResourcesUsed {
+
+	private long memory;
+
+	private long vCores;
+
+	public long getvCores() {
+		return vCores;
+	}
+
+	public void setvCores(long vCores) {
+		this.vCores = vCores;
+	}
+
+	public long getMemory() {
+		return memory;
+	}
+
+	public void setMemory(long memory) {
+		this.memory = memory;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
new file mode 100755
index 0000000..a9b9b3a
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.meta.*;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@Table("running_queue")
+@ColumnFamily("f")
+@Prefix("rqueue")
+@Service(HadoopClusterConstants.RUNNING_QUEUE_SERVICE_NAME)
+@TimeSeries(true)
+@Partition({"site"})
+public class RunningQueueAPIEntity extends TaggedLogAPIEntity {
+	@Column("a")
+	private String state;
+	@Column("b")
+	private double absoluteCapacity;
+	@Column("c")
+	private double absoluteMaxCapacity;
+	@Column("d")
+	private double absoluteUsedCapacity;
+	@Column("e")
+	private long memory;
+	@Column("f")
+	private long vcores;
+	@Column("g")
+	private int numActiveApplications;
+	@Column("h")
+	private int numPendingApplications;
+	@Column("i")
+	private int maxActiveApplications;
+	@Column("j")
+	private String scheduler;
+	@Column("k")
+	private List<UserWrapper> users;
+
+	public String getScheduler() {
+		return scheduler;
+	}
+
+	public void setScheduler(String scheduler) {
+		this.scheduler = scheduler;
+		valueChanged("scheduler");
+	}
+
+	public int getMaxActiveApplications() {
+		return maxActiveApplications;
+	}
+
+	public void setMaxActiveApplications(int maxActiveApplications) {
+		this.maxActiveApplications = maxActiveApplications;
+		valueChanged("maxActiveApplications");
+	}
+
+	public String getState() {
+		return state;
+	}
+
+	public void setState(String state) {
+		this.state = state;
+		valueChanged("state");
+
+	}
+
+	public double getAbsoluteCapacity() {
+		return absoluteCapacity;
+	}
+
+	public void setAbsoluteCapacity(double absoluteCapacity) {
+		this.absoluteCapacity = absoluteCapacity;
+		valueChanged("absoluteCapacity");
+	}
+
+	public double getAbsoluteMaxCapacity() {
+		return absoluteMaxCapacity;
+	}
+
+	public void setAbsoluteMaxCapacity(double absoluteMaxCapacity) {
+		this.absoluteMaxCapacity = absoluteMaxCapacity;
+		valueChanged("absoluteMaxCapacity");
+	}
+
+	public double getAbsoluteUsedCapacity() {
+		return absoluteUsedCapacity;
+	}
+
+	public void setAbsoluteUsedCapacity(double absoluteUsedCapacity) {
+		this.absoluteUsedCapacity = absoluteUsedCapacity;
+		valueChanged("absoluteUsedCapacity");
+	}
+
+	public long getMemory() {
+		return memory;
+	}
+
+	public void setMemory(long memory) {
+		this.memory = memory;
+		valueChanged("memory");
+	}
+
+	public long getVcores() {
+		return vcores;
+	}
+
+	public void setVcores(long vcores) {
+		this.vcores = vcores;
+		valueChanged("vcores");
+	}
+
+	public int getNumActiveApplications() {
+		return numActiveApplications;
+	}
+
+	public void setNumActiveApplications(int numActiveApplications) {
+		this.numActiveApplications = numActiveApplications;
+		valueChanged("numActiveApplications");
+	}
+
+	public int getNumPendingApplications() {
+		return numPendingApplications;
+	}
+
+	public void setNumPendingApplications(int numPendingApplications) {
+		this.numPendingApplications = numPendingApplications;
+		valueChanged("numPendingApplications");
+	}
+
+	public List<UserWrapper> getUsers() {
+		return users;
+	}
+
+	public void setUsers(List<UserWrapper> users) {
+		this.users = users;
+		valueChanged("numPendingApplications");
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Scheduler.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Scheduler.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Scheduler.java
new file mode 100644
index 0000000..3d26d9a
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Scheduler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Scheduler {
+	public SchedulerInfo getSchedulerInfo() {
+		return schedulerInfo;
+	}
+
+	public void setSchedulerInfo(SchedulerInfo schedulerInfo) {
+		this.schedulerInfo = schedulerInfo;
+	}
+
+	private SchedulerInfo schedulerInfo;
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerInfo.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerInfo.java
new file mode 100644
index 0000000..8ed7745
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SchedulerInfo {
+	private String type;
+	private double capacity;
+	private double usedCapacity;
+	private double maxCapacity;
+	private String queueName;
+	private Queues queues;
+
+	public Queues getQueues() {
+		return queues;
+	}
+
+	public void setQueues(Queues queues) {
+		this.queues = queues;
+	}
+
+
+	public double getUsedCapacity() {
+		return usedCapacity;
+	}
+
+	public void setUsedCapacity(double usedCapacity) {
+		this.usedCapacity = usedCapacity;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public double getCapacity() {
+		return capacity;
+	}
+
+	public void setCapacity(double capacity) {
+		this.capacity = capacity;
+	}
+
+	public double getMaxCapacity() {
+		return maxCapacity;
+	}
+
+	public void setMaxCapacity(double maxCapacity) {
+		this.maxCapacity = maxCapacity;
+	}
+
+	public String getQueueName() {
+		return queueName;
+	}
+
+	public void setQueueName(String queueName) {
+		this.queueName = queueName;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerWrapper.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerWrapper.java
new file mode 100644
index 0000000..f181f2f
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/SchedulerWrapper.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SchedulerWrapper {
+	public Scheduler getScheduler() {
+		return scheduler;
+	}
+
+	public void setScheduler(Scheduler scheduler) {
+		this.scheduler = scheduler;
+	}
+
+	private Scheduler scheduler;
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/User.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/User.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/User.java
new file mode 100644
index 0000000..bac9eb2
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/User.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class User {
+	private String username;
+	private ResourcesUsed resourcesUsed;
+	private int numPendingApplications;
+	private int numActiveApplications;
+
+	public String getUsername() {
+		return username;
+	}
+
+	public void setUsername(String username) {
+		this.username = username;
+	}
+
+	public ResourcesUsed getResourcesUsed() {
+		return resourcesUsed;
+	}
+
+	public void setResourcesUsed(ResourcesUsed resourcesUsed) {
+		this.resourcesUsed = resourcesUsed;
+	}
+
+	public int getNumPendingApplications() {
+		return numPendingApplications;
+	}
+
+	public void setNumPendingApplications(int numPendingApplications) {
+		this.numPendingApplications = numPendingApplications;
+	}
+
+	public int getNumActiveApplications() {
+		return numActiveApplications;
+	}
+
+	public void setNumActiveApplications(int numActiveApplications) {
+		this.numActiveApplications = numActiveApplications;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/UserWrapper.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/UserWrapper.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/UserWrapper.java
new file mode 100644
index 0000000..9303727
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/UserWrapper.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+public class UserWrapper {
+
+    private String username;
+    private long memory;
+    private long vCores;
+    private int numPendingApplications;
+    private int numActiveApplications;
+
+    public UserWrapper(User user) {
+        this.username = user.getUsername();
+        this.memory = user.getResourcesUsed().getMemory();
+        this.vCores = user.getResourcesUsed().getvCores();
+        this.numActiveApplications = user.getNumActiveApplications();
+        this.numPendingApplications = user.getNumPendingApplications();
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public long getMemory() {
+        return memory;
+    }
+
+    public void setMemory(long memory) {
+        this.memory = memory;
+    }
+
+    public long getvCores() {
+        return vCores;
+    }
+
+    public void setvCores(long vCores) {
+        this.vCores = vCores;
+    }
+
+    public int getNumPendingApplications() {
+        return numPendingApplications;
+    }
+
+    public void setNumPendingApplications(int numPendingApplications) {
+        this.numPendingApplications = numPendingApplications;
+    }
+
+    public int getNumActiveApplications() {
+        return numActiveApplications;
+    }
+
+    public void setNumActiveApplications(int numActiveApplications) {
+        this.numActiveApplications = numActiveApplications;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Users.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Users.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Users.java
new file mode 100755
index 0000000..f2ca1e2
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/Users.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.model.scheduler;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Users {
+	private List<User> user;
+
+	public List<User> getUser() {
+		return user;
+	}
+
+	public void setUser(List<User> user) {
+		this.user = user;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMessageId.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMessageId.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMessageId.java
new file mode 100644
index 0000000..f4b9a90
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMessageId.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.storm;
+
+import com.google.common.base.Objects;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants.DataSource;
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants.DataType;
+
+public class HadoopQueueMessageId {
+    private String dataType;
+    private String dataSource;
+    private Long timestamp;
+
+    public HadoopQueueMessageId(DataType dataType, DataSource dataSource, Long timestamp) {
+        this.dataSource = dataSource.name();
+        this.dataType = dataType.name();
+        this.timestamp = timestamp;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        final HadoopQueueMessageId other = (HadoopQueueMessageId) obj;
+        return Objects.equal(this.dataType, other.dataType)
+                && Objects.equal(this.dataSource, other.dataSource)
+                && Objects.equal(this.timestamp, other.timestamp);
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder().append(dataType).append(dataSource).append(timestamp).toHashCode();
+    }
+
+    @Override
+    public String toString() {
+        return String.format("dataType=%s, dataSource=%s, timestamp=%d", dataType, dataSource, timestamp);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
new file mode 100644
index 0000000..c99ea41
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueMetricPersistBolt.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.storm;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.eagle.hadoop.queue.model.scheduler.RunningQueueAPIEntity;
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.service.client.EagleServiceConnector;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class HadoopQueueMetricPersistBolt extends BaseRichBolt {
+
+    private final static Logger LOG = LoggerFactory.getLogger(HadoopQueueMetricPersistBolt.class);
+
+    private Config config;
+    private IEagleServiceClient client;
+    private OutputCollector collector;
+
+    public HadoopQueueMetricPersistBolt(Config config) {
+        this.config = config;
+    }
+
+    @Override
+    public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+        this.client = new EagleServiceClientImpl(new EagleServiceConnector(this.config));
+        this.collector = collector;
+    }
+
+    @Override
+    public void execute(Tuple input) {
+        if (input == null) {
+            return;
+        }
+        String dataType = input.getStringByField(HadoopClusterConstants.FIELD_DATATYPE);
+        Object data = input.getValueByField(HadoopClusterConstants.FIELD_DATA);
+        if (dataType.equalsIgnoreCase(HadoopClusterConstants.DataType.METRIC.toString())) {
+            List<GenericMetricEntity> metrics = (List<GenericMetricEntity>) data;
+            writeMetrics(metrics);
+         } else if (dataType.equalsIgnoreCase(HadoopClusterConstants.DataType.ENTITY.toString())) {
+            List<RunningQueueAPIEntity> entities = (List<RunningQueueAPIEntity>) data;
+            writeEntities(entities);
+        }
+        this.collector.ack(input);
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+    }
+
+    private void writeEntities(List<RunningQueueAPIEntity> entities){
+        try {
+            GenericServiceAPIResponseEntity response = client.create(entities);
+            if(!response.isSuccess()){
+                LOG.error("Got exception from eagle service: " + response.getException());
+            }else{
+                LOG.info("Successfully wrote " + entities.size() + " RunningQueueAPIEntity entities");
+            }
+        } catch (Exception e) {
+            LOG.error("cannot create running queue entities successfully", e);
+        }
+        entities.clear();
+    }
+
+    private void writeMetrics(List<GenericMetricEntity> entities){
+        try {
+            GenericServiceAPIResponseEntity response = client.create(entities);
+            if(response.isSuccess()){
+                LOG.info("Successfully wrote " + entities.size() + " GenericMetricEntity entities");
+            }else{
+                LOG.error(response.getException());
+            }
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningExtractor.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningExtractor.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningExtractor.java
new file mode 100644
index 0000000..975e633
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningExtractor.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.storm;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import com.typesafe.config.Config;
+import org.apache.eagle.hadoop.queue.crawler.ClusterMetricsCrawler;
+import org.apache.eagle.hadoop.queue.crawler.RunningAppsCrawler;
+import org.apache.eagle.hadoop.queue.crawler.SchedulerInfoCrawler;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.apache.eagle.hadoop.queue.common.YarnClusterResourceURLBuilder;
+import org.apache.eagle.hadoop.queue.common.YarnURLSelectorImpl;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.jpm.util.resourceFetch.ha.HAURLSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+public class HadoopQueueRunningExtractor {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(HadoopQueueRunningExtractor.class);
+    private final static int MAX_NUM_THREADS = 10;
+    private final static int MAX_WAIT_TIME = 10;
+    private final static String DEFAULT_SITE = "sandbox";
+
+    private String site;
+    private String urlBases;
+
+    private HAURLSelector urlSelector;
+    private ExecutorService executorService;
+    private SpoutOutputCollector collector;
+
+    public HadoopQueueRunningExtractor(Config eagleConf, SpoutOutputCollector collector) {
+        site = HadoopYarnResourceUtils.getConfigValue(eagleConf, "eagleProps.site", DEFAULT_SITE);
+        urlBases = HadoopYarnResourceUtils.getConfigValue(eagleConf, "dataSourceConfig.RMEndPoints", "");
+        if(urlBases == null){
+            throw new IllegalArgumentException(site + ".baseurl is null");
+        }
+        String [] urls = urlBases.split(",");
+        urlSelector = new YarnURLSelectorImpl(urls, Constants.CompressionType.GZIP);
+        executorService = Executors.newFixedThreadPool(MAX_NUM_THREADS);
+        this.collector = collector;
+    }
+
+    private void checkUrl() throws IOException {
+        if (!urlSelector.checkUrl(YarnClusterResourceURLBuilder.buildRunningAppsURL(urlSelector.getSelectedUrl()))) {
+            urlSelector.reSelectUrl();
+        }
+    }
+
+    public void crawl() {
+        try {
+            checkUrl();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        String selectedUrl = urlSelector.getSelectedUrl();
+        LOGGER.info("Current RM base url is " + selectedUrl);
+        List<Future<?>> futures = new ArrayList<>();
+        futures.add(executorService.submit(new ClusterMetricsCrawler(site, selectedUrl, collector)));
+        futures.add(executorService.submit(new RunningAppsCrawler(site, selectedUrl, collector)));
+        futures.add(executorService.submit(new SchedulerInfoCrawler(site, selectedUrl, collector)));
+        futures.forEach(future -> {
+            try {
+                future.get(MAX_WAIT_TIME * 1000, TimeUnit.MILLISECONDS);
+            } catch (TimeoutException e) {
+                LOGGER.info("Caught an overtime exception with message" + e.getMessage());
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            } catch (ExecutionException e) {
+                e.printStackTrace();
+            }
+        });
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningSpout.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningSpout.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningSpout.java
new file mode 100644
index 0000000..2fc85b6
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/storm/HadoopQueueRunningSpout.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue.storm;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichSpout;
+import backtype.storm.tuple.Fields;
+import com.typesafe.config.Config;
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class HadoopQueueRunningSpout extends BaseRichSpout {
+
+    private final static Logger LOG = LoggerFactory.getLogger(HadoopQueueRunningSpout.class);
+    private final static String FETCH_INTERVAL_CONF = "dataSourceConfig.FetchIntervalSec";
+    private final static String DEFAULT_FETCH_INTERVAL_SECONDS = "10";
+
+    private long fetchIntervalSec;
+    private long lastFetchTime = 0;
+
+    private HadoopQueueRunningExtractor extractor;
+    private Config config;
+
+    public HadoopQueueRunningSpout(Config config) {
+        this.config = config;
+        fetchIntervalSec = Long.parseLong(HadoopYarnResourceUtils.getConfigValue(config, FETCH_INTERVAL_CONF, DEFAULT_FETCH_INTERVAL_SECONDS));
+    }
+
+    @Override
+    public void declareOutputFields(OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields(HadoopClusterConstants.FIELD_DATATYPE, HadoopClusterConstants.FIELD_DATA));
+    }
+
+    @Override
+    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+        extractor = new HadoopQueueRunningExtractor(config, collector);
+    }
+
+    @Override
+    public void nextTuple() {
+        try {
+            long fetchTime = System.currentTimeMillis();
+            if (fetchTime > this.fetchIntervalSec * 1000 + this.lastFetchTime) {
+                extractor.crawl();
+                lastFetchTime = fetchTime;
+            }
+        } catch (Exception ex) {
+            LOG.error("Fail crawling running queue resources and continue ...", ex);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
new file mode 100644
index 0000000..77ae8be
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/application.conf
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "topology" : {
+    "localMode" : true,
+    "numOfTotalWorkers" : 2,
+    "numOfParserTasks" : 2,
+    "name" : "sandbox-running-queue-topology",
+  },
+  "dataSourceConfig": {
+    "RMEndPoints" : "http://sandbox.hortonworks.com:8088/",
+    "FetchIntervalSec": "10"
+  },
+  "eagleProps" : {
+    "site": "sandbox",
+    "eagleService": {
+      "host": "localhost",
+      "port": 9099
+      "username": "admin",
+      "password": "secret"
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/log4j.properties b/eagle-jpm/eagle-hadoop-queue/src/main/resources/log4j.properties
new file mode 100644
index 0000000..71a5dac
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/log4j.properties
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, DRFA, stdout
+eagle.log.dir=./logs
+eagle.log.file=eagle.log
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+# Daily Rolling File Appender
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestClusterMetricsCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestClusterMetricsCrawler.java b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestClusterMetricsCrawler.java
new file mode 100644
index 0000000..371d75c
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestClusterMetricsCrawler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.apache.eagle.hadoop.queue.crawler.ClusterMetricsCrawler;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestClusterMetricsCrawler {
+
+    @Test @Ignore
+    public void test() {
+        String site = "sandbox";
+        Config config = ConfigFactory.load();
+
+        String baseUrl = HadoopYarnResourceUtils.getConfigValue(config, "dataSourceConfig.RMEndPoints", "");
+        ClusterMetricsCrawler crawler = new ClusterMetricsCrawler(site, baseUrl, null);
+        crawler.run();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestHadoopYarnResourceUtils.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestHadoopYarnResourceUtils.java b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestHadoopYarnResourceUtils.java
new file mode 100644
index 0000000..27bdc3b
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestHadoopYarnResourceUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.apache.eagle.hadoop.queue.common.YarnClusterResourceURLBuilder;
+import org.apache.eagle.hadoop.queue.model.applications.AppsWrapper;
+import org.apache.eagle.hadoop.queue.model.clusterMetrics.ClusterMetricsWrapper;
+import org.apache.eagle.hadoop.queue.model.scheduler.SchedulerWrapper;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestHadoopYarnResourceUtils {
+
+    @Test @Ignore
+    public void test() {
+        Config config = ConfigFactory.load();
+
+        String baseUrl = HadoopYarnResourceUtils.getConfigValue(config, "dataSourceConfig.RMEndPoints", "");
+        String clusterMetricUrl = YarnClusterResourceURLBuilder.buildClusterMetricsURL(baseUrl);
+        String finishedAppsUrl = YarnClusterResourceURLBuilder.buildFinishedAppsURL(baseUrl);
+        String schedulerUrl = YarnClusterResourceURLBuilder.buildSchedulerInfoURL(baseUrl);
+
+        try {
+            ClusterMetricsWrapper clusterMetrics = (ClusterMetricsWrapper) HadoopYarnResourceUtils.getObjectFromStreamWithGzip(clusterMetricUrl, ClusterMetricsWrapper.class);
+            AppsWrapper appsWrapper = (AppsWrapper) HadoopYarnResourceUtils.getObjectFromStreamWithGzip(finishedAppsUrl, AppsWrapper.class);
+            SchedulerWrapper schedulerWrapper = (SchedulerWrapper) HadoopYarnResourceUtils.getObjectFromStreamWithGzip(schedulerUrl, SchedulerWrapper.class);
+            Assert.assertTrue(appsWrapper != null);
+            Assert.assertTrue(clusterMetrics != null);
+            Assert.assertTrue(schedulerWrapper != null);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestRunningAppsCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestRunningAppsCrawler.java b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestRunningAppsCrawler.java
new file mode 100644
index 0000000..148ace0
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestRunningAppsCrawler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.apache.eagle.hadoop.queue.crawler.RunningAppsCrawler;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestRunningAppsCrawler {
+
+    @Test @Ignore
+    public void test() {
+        String site = "sandbox";
+        Config config = ConfigFactory.load();
+
+        String baseUrl = HadoopYarnResourceUtils.getConfigValue(config, "dataSourceConfig.RMEndPoints", "");
+        RunningAppsCrawler crawler = new RunningAppsCrawler(site, baseUrl, null);
+        crawler.run();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestSchedulerInfoCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestSchedulerInfoCrawler.java b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestSchedulerInfoCrawler.java
new file mode 100644
index 0000000..5f7c636
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/java/org/apache/eagle/hadoop/queue/TestSchedulerInfoCrawler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.hadoop.queue;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.hadoop.queue.common.HadoopYarnResourceUtils;
+import org.apache.eagle.hadoop.queue.crawler.SchedulerInfoCrawler;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestSchedulerInfoCrawler {
+    @Test @Ignore
+    public void test() {
+        String site = "sandbox";
+        Config config = ConfigFactory.load();
+
+        String baseUrl = HadoopYarnResourceUtils.getConfigValue(config, "dataSourceConfig.RMEndPoints", "");
+
+        SchedulerInfoCrawler crawler = new SchedulerInfoCrawler(site, baseUrl, null);
+        crawler.run();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application-bak.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application-bak.conf b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application-bak.conf
new file mode 100644
index 0000000..f346b7f
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application-bak.conf
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "topology" : {
+    "localMode" : true,
+    "numOfTotalWorkers" : 2,
+    "numOfParserTasks" : 2,
+    "name" : "sandbox-running-queue-topology",
+  },
+  "dataSourceConfig": {
+    "RMEndPoints" : "http://sandbox.hortonworks.com:8088/",
+    "FetchIntervalSec": "10"
+  },
+  "eagleProps" : {
+    "site": "sandbox",
+    "eagleService": {
+      "host": "localhost",
+      "port": 9098
+      "username": "admin",
+      "password": "secret"
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
new file mode 100644
index 0000000..77ae8be
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/resources/application.conf
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  "topology" : {
+    "localMode" : true,
+    "numOfTotalWorkers" : 2,
+    "numOfParserTasks" : 2,
+    "name" : "sandbox-running-queue-topology",
+  },
+  "dataSourceConfig": {
+    "RMEndPoints" : "http://sandbox.hortonworks.com:8088/",
+    "FetchIntervalSec": "10"
+  },
+  "eagleProps" : {
+    "site": "sandbox",
+    "eagleService": {
+      "host": "localhost",
+      "port": 9099
+      "username": "admin",
+      "password": "secret"
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-hadoop-queue/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/test/resources/log4j.properties b/eagle-jpm/eagle-hadoop-queue/src/test/resources/log4j.properties
new file mode 100644
index 0000000..71a5dac
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/test/resources/log4j.properties
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, DRFA, stdout
+eagle.log.dir=./logs
+eagle.log.file=eagle.log
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+# Daily Rolling File Appender
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ha/AbstractURLSelector.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ha/AbstractURLSelector.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ha/AbstractURLSelector.java
new file mode 100644
index 0000000..57c2902
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/resourceFetch/ha/AbstractURLSelector.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.jpm.util.resourceFetch.ha;
+
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.jpm.util.resourceFetch.connection.InputStreamUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+
+public abstract class AbstractURLSelector implements HAURLSelector {
+    private final String[] urls;
+    private volatile String selectedUrl;
+
+    private volatile boolean reselectInProgress;
+    private final Constants.CompressionType compressionType;
+
+    private static final long MAX_RETRY_TIME = 3;
+    private static final Logger LOG = LoggerFactory.getLogger(HAURLSelectorImpl.class);
+
+    public AbstractURLSelector(String[] urls, Constants.CompressionType compressionType) {
+        this.urls = urls;
+        this.compressionType = compressionType;
+    }
+
+    public boolean checkUrl(String urlString) {
+        InputStream is = null;
+        try {
+            is = InputStreamUtils.getInputStream(urlString, null, compressionType);
+        } catch (Exception ex) {
+            LOG.info("get input stream from url: " + urlString + " failed. ");
+            return false;
+        } finally {
+            if (is != null) { try {	is.close(); } catch (IOException e) {/*Do nothing*/} }
+        }
+        return true;
+    }
+
+    @Override
+    public String getSelectedUrl() {
+        if (selectedUrl == null) {
+            selectedUrl = urls[0];
+        }
+        return selectedUrl;
+    }
+
+    @Override
+    public void reSelectUrl() throws IOException {
+        if (reselectInProgress) return;
+        synchronized(this) {
+            if (reselectInProgress) return;
+            reselectInProgress = true;
+            try {
+                LOG.info("Going to reselect url");
+                for (int i = 0; i < urls.length; i++) {
+                    String urlToCheck = urls[i];
+                    LOG.info("Going to try url :" + urlToCheck);
+                    for (int time = 0; time < MAX_RETRY_TIME; time++) {
+                        if (checkUrl(buildTestURL(urlToCheck))) {
+                            selectedUrl = urls[i];
+                            LOG.info("Successfully switch to new url : " + selectedUrl);
+                            return;
+                        }
+                        LOG.info("try url " + urlToCheck + "fail for " + (time+1) + " times, sleep 5 seconds before try again. ");
+                        try {
+                            Thread.sleep(5 * 1000);
+                        }
+                        catch (InterruptedException ex) { /* Do Nothing */}
+                    }
+                }
+                throw new IOException("No alive url found: "+ StringUtils.join(";", Arrays.asList(this.urls)));
+            }
+            finally {
+                reselectInProgress = false;
+            }
+        }
+    }
+
+    protected abstract String buildTestURL(String urlToCheck);
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-jpm/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/pom.xml b/eagle-jpm/pom.xml
index c1bdaad..05c3087 100644
--- a/eagle-jpm/pom.xml
+++ b/eagle-jpm/pom.xml
@@ -35,7 +35,8 @@
         <module>eagle-jpm-spark-running</module>
         <module>eagle-jpm-spark-history</module>
         <module>eagle-jpm-mr-history</module>
-         <module>eagle-jpm-entity</module>
+        <module>eagle-jpm-entity</module>
+        <module>eagle-hadoop-queue</module>
         <module>eagle-jpm-util</module>
   </modules>
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index d454416..7e1cb46 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -28,6 +28,11 @@
     <url>http://maven.apache.org</url>
     <packaging>jar</packaging>
     <dependencies>
+        <!--<dependency>
+            <groupId>org.apache.eagle</groupId>
+            <artifactId>eagle-hadoop-queue</artifactId>
+            <version>${project.version}</version>
+        </dependency>-->
         <dependency>
             <groupId>org.apache.eagle</groupId>
             <artifactId>eagle-security-hdfs-auditlog</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9f6fea4a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d6a005f..0de71b9 100755
--- a/pom.xml
+++ b/pom.xml
@@ -133,6 +133,7 @@
         <module>eagle-examples</module>
         <module>eagle-gc</module>
         <module>eagle-hadoop-metric</module>
+        <module>eagle-jpm</module>
     </modules>
     <properties>
         <!-- General Properties -->


Mime
View raw message