ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tbeerbo...@apache.org
Subject [3/3] git commit: AMBARI-2254 - Reduce number of requests when querying for subtasks
Date Thu, 14 Nov 2013 10:39:20 GMT
AMBARI-2254 - Reduce number of requests when querying for subtasks


Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/fc427d9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/fc427d9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/fc427d9f

Branch: refs/heads/trunk
Commit: fc427d9f3e2d448a4e8746dd04199ad17390755c
Parents: 22f5fdf
Author: tbeerbower <tbeerbower@hortonworks.com>
Authored: Wed Nov 13 11:01:57 2013 -0500
Committer: tbeerbower <tbeerbower@hortonworks.com>
Committed: Thu Nov 14 05:21:19 2013 -0500

----------------------------------------------------------------------
 .../ambari/server/api/query/QueryImpl.java      | 527 ++++++++------
 .../server/api/resources/ResourceInstance.java  |   6 -
 .../resources/ResourceInstanceFactoryImpl.java  |  39 +-
 .../api/resources/ResourceInstanceImpl.java     | 142 ----
 .../ganglia/GangliaPropertyProvider.java        |   6 +-
 .../internal/ClusterControllerImpl.java         | 178 +++--
 .../controller/spi/ClusterController.java       | 172 ++---
 .../org/apache/ambari/server/api/TestSuite.java |  34 +-
 .../ambari/server/api/query/QueryImplTest.java  | 705 ++++++-------------
 .../api/resources/ResourceInstanceImplTest.java |  81 ---
 .../ganglia/GangliaPropertyProviderTest.java    |   4 +-
 .../internal/ClusterControllerImplTest.java     | 162 +++--
 12 files changed, 880 insertions(+), 1176 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
index 7eb36e9..09923dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java
@@ -18,10 +18,12 @@
 
 package org.apache.ambari.server.api.query;
 
+import org.apache.ambari.server.api.resources.ResourceDefinition;
 import org.apache.ambari.server.api.resources.ResourceInstance;
+import org.apache.ambari.server.api.resources.ResourceInstanceFactoryImpl;
+import org.apache.ambari.server.api.resources.SubResourceDefinition;
 import org.apache.ambari.server.api.services.ResultImpl;
-import org.apache.ambari.server.api.util.TreeNodeImpl;
-import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
+import org.apache.ambari.server.controller.predicate.OrPredicate;
 import org.apache.ambari.server.controller.utilities.PropertyHelper;
 import org.apache.ambari.server.controller.predicate.AndPredicate;
 import org.apache.ambari.server.controller.predicate.EqualsPredicate;
@@ -36,62 +38,64 @@ import java.util.*;
 /**
  * Default read query.
  */
-public class QueryImpl implements Query {
+public class QueryImpl implements Query, ResourceInstance {
 
   /**
-   * Resource instance.
+   * Definition for the resource type.  The definition contains all information specific to the
+   * resource type.
    */
-  private ResourceInstance m_resource;
+  private final ResourceDefinition resourceDefinition;
 
   /**
-   * Properties of the query which make up the select portion of the query.
+   * The cluster controller.
    */
-  private Set<String> m_setQueryProperties = new HashSet<String>();
+  private final ClusterController clusterController;
 
   /**
-   * Indicates that the query should include all available properties.
+   * Properties of the query which make up the select portion of the query.
    */
-  private boolean allProperties = false;
+  private final Set<String> queryPropertySet = new HashSet<String>();
 
   /**
-   * Map that associates each property set on the query to temporal data.
+   * Map that associates categories with temporal data.
    */
-  private Map<String, TemporalInfo> m_mapPropertyTemporalInfo = new HashMap<String, TemporalInfo>();
+  private final Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
 
   /**
-   * Map that associates categories with temporal data.
+   * Map of primary and foreign keys and values necessary to identify the resource.
    */
-  private Map<String, TemporalInfo> m_mapCategoryTemporalInfo = new HashMap<String, TemporalInfo>();
+  private final Map<Resource.Type, String> resourceIds = new HashMap<Resource.Type, String>();
 
   /**
    * Sub-resources of the resource which is being operated on.
    */
-  private Map<String, ResourceInstance> m_mapSubResources = new HashMap<String, ResourceInstance>();
+  private final Map<String, QueryImpl> querySubResourceSet = new HashMap<String, QueryImpl>();
 
   /**
-   * The user supplied predicate.
+   * Sub-resource instances of this resource.
+   * Map of resource name to resource instance.
    */
-  private Predicate m_userPredicate;
+  private Map<String, QueryImpl> subResourceSet;
 
   /**
-   * The user supplied page request information.
+   * Indicates that the query should include all available properties.
    */
-  private PageRequest m_pageRequest;
+  private boolean allProperties = false;
 
   /**
-   * Predicate calculated for query
+   * The user supplied predicate.
    */
-  private Predicate m_predicate;
+  private Predicate userPredicate;
 
   /**
-   * Query resources, don't clone
+   * The user supplied page request information.
    */
-  private Set<Resource> m_providerResources;
+  private PageRequest pageRequest;
 
   /**
-   * Populated query resources with applied predicate and pagination, don't clone
+   * Query resources.
    */
-  private Iterable<Resource> m_resourceIterable;
+  private Set<Resource> providerResourceSet;
 
   /**
    * The logger.
@@ -99,15 +103,27 @@ public class QueryImpl implements Query {
   private final static Logger LOG =
       LoggerFactory.getLogger(QueryImpl.class);
 
+
+  // ----- Constructor -------------------------------------------------------
+
   /**
-   * Constructor.
+   * Constructor
    *
-   * @param resource the resource being operated on
+   * @param mapIds              the map of ids
+   * @param resourceDefinition  the resource definition
+   * @param clusterController   the cluster controller
    */
-  public QueryImpl(ResourceInstance resource) {
-    m_resource = resource;
+  public QueryImpl(Map<Resource.Type, String> mapIds,
+                   ResourceDefinition resourceDefinition,
+                   ClusterController clusterController) {
+    this.resourceDefinition = resourceDefinition;
+    this.clusterController  = clusterController;
+    setIds(mapIds);
   }
 
+
+  // ----- Query -------------------------------------------------------------
+
   @Override
   public void addProperty(String category, String name, TemporalInfo temporalInfo) {
     if (category == null && name.equals("*")) {
@@ -116,17 +132,17 @@ public class QueryImpl implements Query {
     } else{
       if (addPropertyToSubResource(category, name, temporalInfo)){
         // add pk/fk properties of the resource to this query
-        Resource.Type resourceType = m_resource.getResourceDefinition().getType();
-        Schema        schema       = getClusterController().getSchema(resourceType);
+        Resource.Type resourceType = getResourceDefinition().getType();
+        Schema        schema       = clusterController.getSchema(resourceType);
 
-        for (Resource.Type type : m_resource.getIds().keySet()) {
+        for (Resource.Type type : getIds().keySet()) {
           addLocalProperty(schema.getKeyPropertyId(type));
         }
       } else {
         String propertyId = PropertyHelper.getPropertyId(category, name.equals("*") ? null : name);
         addLocalProperty(propertyId);
         if (temporalInfo != null) {
-          m_mapCategoryTemporalInfo.put(propertyId, temporalInfo);
+          temporalInfoMap.put(propertyId, temporalInfo);
         }
       }
     }
@@ -134,217 +150,304 @@ public class QueryImpl implements Query {
 
   @Override
   public void addLocalProperty(String property) {
-    m_setQueryProperties.add(property);
+    queryPropertySet.add(property);
   }
 
-  protected Set<Resource> getProviderResources() throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
-    if (m_providerResources == null) {
-      Resource.Type resourceType = m_resource.getResourceDefinition().getType();
+  @Override
+  public Result execute()
+      throws UnsupportedPropertyException,
+             SystemException,
+             NoSuchResourceException,
+             NoSuchParentResourceException {
+
+    queryForResources();
+    return getResult();
+  }
 
-      if (m_resource.getIds().get(getResourceType()) == null) {
-        addCollectionProperties(getResourceType());
-      }
+  @Override
+  public Predicate getPredicate() {
+    return createPredicate();
+  }
 
-      if (m_setQueryProperties.isEmpty() && m_mapSubResources.isEmpty()) {
-        //Add sub resource properties for default case where no fields are specified.
-        m_mapSubResources.putAll(m_resource.getSubResources());
-      }
+  @Override
+  public Set<String> getProperties() {
+    return Collections.unmodifiableSet(queryPropertySet);
+  }
 
-      if (LOG.isDebugEnabled()) {
-        //todo: include predicate info.  Need to implement toString for all predicates.
-        LOG.debug("Executing resource query: " + m_resource.getIds());
-      }
+  @Override
+  public void setUserPredicate(Predicate predicate) {
+    userPredicate = predicate;
+  }
 
-      Predicate predicate = getPredicate();
+  @Override
+  public void setPageRequest(PageRequest pageRequest) {
+    this.pageRequest = pageRequest;
+  }
 
-      Request request = createRequest();
-      m_providerResources = getClusterController().getRawResources(resourceType, request, predicate);
-    }
 
-    return m_providerResources;
-  }
+  // ----- ResourceInstance --------------------------------------------------
 
-  protected Iterable<Resource> getResourceIterable() throws NoSuchParentResourceException, UnsupportedPropertyException,
-      NoSuchResourceException, SystemException {
-    if (m_resourceIterable == null) {
-      if (m_pageRequest == null) {
-        m_resourceIterable = getClusterController()
-            .getResources(getResourceType(), m_providerResources, getPredicate());
-      } else {
-        m_resourceIterable = getClusterController()
-            .getResources(getResourceType(), m_providerResources, getPredicate(), m_pageRequest).getIterable();
-      }
-    }
-    return m_resourceIterable;
+  @Override
+  public void setIds(Map<Resource.Type, String> mapIds) {
+    resourceIds.putAll(mapIds);
   }
 
   @Override
-  public Result execute() throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
-
-    Map<Resource,Collection<Resource>> resourceTree =
-        new HashMap<Resource, Collection<Resource>>();
-    Map<Resource,String> resourceNames = new HashMap<Resource, String>();
+  public Map<Resource.Type, String> getIds() {
+    return new HashMap<Resource.Type, String>((resourceIds));
+  }
 
-    Map<QueryImpl, Resource> nextLevel = new HashMap<QueryImpl, Resource>();
-    nextLevel.put(this, null);
+  @Override
+  public Query getQuery() {
+    return this;
+  }
 
-    while (!nextLevel.isEmpty()) {
-      Map<QueryImpl, Resource> current = nextLevel;
-      Map<Resource.Type, Set<Resource>> resourceMap = new HashMap<Resource.Type, Set<Resource>>();
-      Map<Resource.Type, Predicate> predicateMap = new HashMap<Resource.Type, Predicate>();
+  @Override
+  public ResourceDefinition getResourceDefinition() {
+    return resourceDefinition;
+  }
 
-      Map<Resource.Type, Request> requestMap = new HashMap<Resource.Type, Request>();
+  @Override
+  public boolean isCollectionResource() {
+    return getIds().get(getResourceDefinition().getType()) == null;
+  }
 
-      //All resource instances in tree have same set of property ids, only resource type is valuable
-      //Request and predicate are formed using data described above
-      for (QueryImpl query : current.keySet()) {
+  @Override
+  public Map<String, ResourceInstance> getSubResources() {
+    return new HashMap<String, ResourceInstance>(ensureSubResources());
+  }
 
-        Resource.Type type = query.getResourceType();
-        if (!resourceMap.containsKey(type)) {
-          resourceMap.put(type, new HashSet<Resource>());
-          predicateMap.put(type, query.getPredicate());
-          requestMap.put(type, query.createRequest());
-        }
 
-        resourceMap.get(type).addAll(query.getProviderResources());
+  // ----- Object overrides --------------------------------------------------
 
-      }
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
 
-      getClusterController().populateResources(resourceMap, requestMap, predicateMap);
+    QueryImpl query = (QueryImpl) o;
 
-      //fill next level
-      nextLevel = new HashMap<QueryImpl, Resource>();
+    return clusterController.equals(query.clusterController) && !(pageRequest != null ?
+        !pageRequest.equals(query.pageRequest) :
+        query.pageRequest != null) && queryPropertySet.equals(query.queryPropertySet) &&
+        resourceDefinition.equals(query.resourceDefinition) &&
+        resourceIds.equals(query.resourceIds) && !(userPredicate != null ?
+        !userPredicate.equals(query.userPredicate) :
+        query.userPredicate != null);
+  }
 
-      for (Map.Entry<QueryImpl, Resource> entry : current.entrySet()) {
-        QueryImpl query = entry.getKey();
-        Resource parent = entry.getValue();
+  @Override
+  public int hashCode() {
+    int result = resourceDefinition.hashCode();
+    result = 31 * result + clusterController.hashCode();
+    result = 31 * result + queryPropertySet.hashCode();
+    result = 31 * result + resourceIds.hashCode();
+    result = 31 * result + (userPredicate != null ? userPredicate.hashCode() : 0);
+    result = 31 * result + (pageRequest != null ? pageRequest.hashCode() : 0);
+    return result;
+  }
 
-        Map<QueryImpl, Resource> subQueries = query.getSubQueries();
 
-        Collection<Resource> validResources = new ArrayList<Resource>();
+  // ----- helper methods ----------------------------------------------------
 
-        for (Resource resource : query.getResourceIterable()) {
-          validResources.add(resource);
+  /**
+   * Get the map of sub-resources.  Lazily create the map if required.  
+   */
+  private Map<String, QueryImpl> ensureSubResources() {
+    if (subResourceSet == null) {
+      subResourceSet = new HashMap<String, QueryImpl>();
+      Set<SubResourceDefinition> setSubResourceDefs =
+          getResourceDefinition().getSubResourceDefinitions();
+
+      ClusterController controller = clusterController;
+      for (SubResourceDefinition subResDef : setSubResourceDefs) {
+        Resource.Type type = subResDef.getType();
+        Map<Resource.Type, String> ids = getIds();
+        QueryImpl resource =  new QueryImpl(ids,
+            ResourceInstanceFactoryImpl.getResourceDefinition(type, ids),
+            controller);
+
+        // ensure pk is returned
+        resource.addLocalProperty(controller.getSchema(
+            type).getKeyPropertyId(type));
+        // add additionally required fk properties
+        for (Resource.Type fkType : subResDef.getAdditionalForeignKeys()) {
+          resource.addLocalProperty(controller.getSchema(type).getKeyPropertyId(fkType));
         }
 
-        resourceTree.put(parent, validResources);
+        String subResourceName = subResDef.isCollection() ?
+            resource.getResourceDefinition().getPluralName() :
+            resource.getResourceDefinition().getSingularName();
 
-        resourceNames.put(parent, query.getResultName());
-
-        nextLevel.putAll(subQueries);
+        subResourceSet.put(subResourceName, resource);
       }
-
     }
-
-    //create Result
-    Result result = createResult();
-    populateResult(result, resourceTree, resourceNames, null, null);
-    return result;
+    return subResourceSet;
   }
 
-  public String getResultName() {
-    return m_resource.isCollectionResource() ? m_resource.getResourceDefinition().getPluralName() :
-        m_resource.getResourceDefinition().getSingularName();
-  }
+  /**
+   * Query the cluster controller for the top level resources.
+   */
+  private void queryForResources()
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException {
+
+    providerResourceSet = new LinkedHashSet<Resource>();
 
-  public Resource.Type getResourceType() {
-    return m_resource.getResourceDefinition().getType();
+    // save the top level resources
+    for (Resource resource : doQuery(getPredicate())) {
+      providerResourceSet.add(resource);
+    }
+    queryForSubResources();
   }
 
-  protected void populateResult(Result result, Map<Resource, Collection<Resource>> resourceTree,
-                                  Map<Resource, String> resourceNames,
-                                  TreeNode<Resource> node, Resource parent) {
+  /**
+   * Query the cluster controller for the sub-resources associated with 
+   * the given resources.  All the sub-resources of the same type should
+   * be acquired with a single query.
+   */
+  private void queryForSubResources()
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException {
+
+    // get predicates for all of the sub resource types
+    Map<String, Predicate> predicateMap = getSubResourcePredicates(providerResourceSet);
 
-    TreeNode<Resource> tree;
-    if (node == null) {
-      tree = result.getResultTree();
-      if (m_resource.getIds().get(getResourceType()) == null) {
-        tree.setProperty("isCollection", "true");
+    for (Map.Entry<String, QueryImpl> entry : querySubResourceSet.entrySet()) {
+      QueryImpl subResource  = entry.getValue();
+
+      subResource.providerResourceSet = new LinkedHashSet<Resource>();
+
+      for (Resource resource : subResource.doQuery(predicateMap.get(entry.getKey()))) {
+        subResource.providerResourceSet.add(resource);
       }
-    } else {
-      tree = new TreeNodeImpl<Resource>(null, null, null);
+      subResource.queryForSubResources();
     }
+  }
 
-    if (node != null) {
-      tree.setProperty("isCollection", "false");
-      tree.setName(resourceNames.get(parent));
-      node.addChild(tree);
-    }
+  /**
+   * Query the cluster controller for the resources.
+   */
+  private Set<Resource> doQuery(Predicate predicate)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException {
 
-    int count = 1;
-    for (Resource resource : resourceTree.get(parent)) {
-      TreeNode<Resource> subNode = tree.addChild(resource, resource.getType() + ":" + count++);
-      if (resourceTree.containsKey(resource)) {
-        populateResult(result, resourceTree, resourceNames, subNode, resource);
-      }
+    Resource.Type resourceType1 = getResourceDefinition().getType();
+
+    if (getIds().get(resourceType1) == null) {
+      addCollectionProperties(resourceType1);
+    }
+    if (queryPropertySet.isEmpty() && querySubResourceSet.isEmpty()) {
+      //Add sub resource properties for default case where no fields are specified.
+      querySubResourceSet.putAll(ensureSubResources());
     }
 
-  }
+    Resource.Type resourceType = getResourceDefinition().getType();
+    Request       request      = createRequest();
 
-  public Map<QueryImpl, Resource> getSubQueries()
-      throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
-    return getSubQueries(getResourceIterable());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Executing resource query: " + request + " where " + predicate);
+    }
+    return clusterController.getResources(resourceType, request, predicate);
   }
 
-  protected Map<QueryImpl, Resource> getSubQueries(Iterable<Resource> resourceIterable) {
-    Map<QueryImpl, Resource> parentMap = new HashMap<QueryImpl, Resource>();
+  /**
+   * Get a map of predicates for the given resource's sub-resources keyed 
+   * by resource type.  Each predicate is a combined predicate of all 
+   * the sub resource predicates for a given type OR'd together.  This 
+   * allows for all of the sub-resources of a given type to be 
+   * acquired in a single query.
+   */
+  private Map<String, Predicate> getSubResourcePredicates(Set<Resource> resources) {
+    Map<String, Predicate> predicateMap = new HashMap<String, Predicate>();
 
-    for (Resource parentResource : resourceIterable) {
-      //Need to copy Resource Instance because of delayed query execution
-      for (ResourceInstance resourceInstanceTemplate : m_mapSubResources.values()) {
-        ResourceInstance resourceInstance;
-        resourceInstance = resourceInstanceTemplate.createCopy();
-        setParentIdsOnSubResource(parentResource, resourceInstance);
-        parentMap.put((QueryImpl) resourceInstance.getQuery(), parentResource);
+    for (Resource resource : resources) {
+      for (Map.Entry<String, QueryImpl> entry : querySubResourceSet.entrySet()) {
+        QueryImpl subResourceInstance = entry.getValue();
+        String    subResCategory      = entry.getKey();
+        Predicate predicate           = predicateMap.get(subResCategory);
 
+        setParentIdsOnSubResource(resource, subResourceInstance);
+
+        predicateMap.put(subResCategory, predicate == null ?
+            subResourceInstance.getPredicate() :
+            new OrPredicate(predicate, subResourceInstance.getPredicate()));
       }
     }
-
-    return parentMap;
+    return predicateMap;
   }
 
+  /**
+   * Get a result from this query.
+   */
+  private Result getResult()
+      throws UnsupportedPropertyException, SystemException, NoSuchResourceException, NoSuchParentResourceException {
 
-  @Override
-  public Predicate getPredicate() {
-    if (m_predicate == null) {
-      m_predicate = createPredicate(m_resource);
+    Result result = new ResultImpl(true);
+    Resource.Type resourceType = getResourceDefinition().getType();
+    if (getIds().get(resourceType) == null) {
+      result.getResultTree().setProperty("isCollection", "true");
     }
-    return m_predicate;
-  }
 
-  @Override
-  public Set<String> getProperties() {
-    return Collections.unmodifiableSet(m_setQueryProperties);
-  }
+    Predicate predicate = createPredicate();
+    Request   request   = createRequest();
 
-  @Override
-  public void setUserPredicate(Predicate predicate) {
-    m_userPredicate = predicate;
-  }
+    Iterable<Resource> iterResource;
 
-  @Override
-  public void setPageRequest(PageRequest pageRequest) {
-    m_pageRequest = pageRequest;
-  }
+    if (pageRequest == null) {
+      iterResource = clusterController.getIterable(
+          resourceType, providerResourceSet, request, predicate);
+    } else {
+      PageResponse pageResponse = clusterController.getPage(
+          resourceType, providerResourceSet, request, predicate, pageRequest);
+      iterResource = pageResponse.getIterable();
+    }
 
-  ClusterController getClusterController() {
-    return ClusterControllerHelper.getClusterController();
+    TreeNode<Resource> tree = result.getResultTree();
+
+    int count = 1;
+    for (Resource resource : iterResource) {
+      // add a child node for the resource and provide a unique name.  The name is never used.
+      TreeNode<Resource> node = tree.addChild(resource, resource.getType() + ":" + count++);
+      for (Map.Entry<String, QueryImpl> entry : querySubResourceSet.entrySet()) {
+        String    subResCategory = entry.getKey();
+        QueryImpl subResource    = entry.getValue();
+
+        setParentIdsOnSubResource(resource, subResource);
+
+        TreeNode<Resource> childResult = subResource.getResult().getResultTree();
+        childResult.setName(subResCategory);
+        childResult.setProperty("isCollection", "false");
+        node.addChild(childResult);
+      }
+    }
+    return result;
   }
 
   private void addCollectionProperties(Resource.Type resourceType) {
-    Schema schema = getClusterController().getSchema(resourceType);
+    Schema schema = clusterController.getSchema(resourceType);
+
     // add pk
     String property = schema.getKeyPropertyId(resourceType);
-    addProperty(PropertyHelper.getPropertyCategory(property), PropertyHelper.getPropertyName(property), null);
+    addProperty(PropertyHelper.getPropertyCategory(property),
+        PropertyHelper.getPropertyName(property), null);
 
-    for (Resource.Type type : m_resource.getIds().keySet()) {
+    for (Resource.Type type : getIds().keySet()) {
       // add fk's
       String keyPropertyId = schema.getKeyPropertyId(type);
-      //todo: property id can be null in some cases such as host_component queries which obtain
-      //todo: component sub-resources.  Component will not have host fk.
-      //todo: refactor so that null check is not required.
       if (keyPropertyId != null) {
-        addProperty(PropertyHelper.getPropertyCategory(keyPropertyId), PropertyHelper.getPropertyName(keyPropertyId), null);
+        addProperty(PropertyHelper.getPropertyCategory(keyPropertyId),
+            PropertyHelper.getPropertyName(keyPropertyId), null);
       }
     }
   }
@@ -352,13 +455,13 @@ public class QueryImpl implements Query {
   private void addAllProperties(TemporalInfo temporalInfo) {
     allProperties = true;
     if (temporalInfo != null) {
-      m_mapCategoryTemporalInfo.put(null, temporalInfo);
+      temporalInfoMap.put(null, temporalInfo);
     }
 
-    for (Map.Entry<String, ResourceInstance> entry : m_resource.getSubResources().entrySet()) {
+    for (Map.Entry<String, QueryImpl> entry : ensureSubResources().entrySet()) {
       String name = entry.getKey();
-      if (! m_mapSubResources.containsKey(name)) {
-        m_mapSubResources.put(name, entry.getValue());
+      if (! querySubResourceSet.containsKey(name)) {
+        querySubResourceSet.put(name, entry.getValue());
       }
     }
   }
@@ -379,10 +482,9 @@ public class QueryImpl implements Query {
     int i = path.indexOf("/");
     String p = i == -1 ? path : path.substring(0, i);
 
-    ResourceInstance subResource = m_resource.getSubResources().get(p);
+    QueryImpl subResource = ensureSubResources().get(p);
     if (subResource != null) {
-      m_mapSubResources.put(p, subResource);
-      //todo: handle case of trailing '/' (for example fields=subResource/)
+      querySubResourceSet.put(p, subResource);
 
       if (property != null || !path.equals(p)) {
         //only add if a sub property is set or if a sub category is specified
@@ -393,10 +495,10 @@ public class QueryImpl implements Query {
     return resourceAdded;
   }
 
-  private Predicate createInternalPredicate(ResourceInstance resource) {
-    Resource.Type resourceType = resource.getResourceDefinition().getType();
-    Map<Resource.Type, String> mapResourceIds = resource.getIds();
-    Schema schema = getClusterController().getSchema(resourceType);
+  private Predicate createInternalPredicate() {
+    Resource.Type resourceType = getResourceDefinition().getType();
+    Map<Resource.Type, String> mapResourceIds = getIds();
+    Schema schema = clusterController.getSchema(resourceType);
 
     Set<Predicate> setPredicates = new HashSet<Predicate>();
     for (Map.Entry<Resource.Type, String> entry : mapResourceIds.entrySet()) {
@@ -417,29 +519,28 @@ public class QueryImpl implements Query {
     }
   }
 
-  private Predicate createPredicate(ResourceInstance resource) {
+  private Predicate createPredicate() {
     Predicate predicate = null;
-    //todo: change reference type to Predicate when predicate hierarchy is fixed
-    Predicate internalPredicate = createInternalPredicate(resource);
+    Predicate internalPredicate = createInternalPredicate();
     if (internalPredicate == null) {
-      if (m_userPredicate != null) {
-        predicate = m_userPredicate;
+      if (userPredicate != null) {
+        predicate = userPredicate;
       }
     } else {
-      predicate = (m_userPredicate == null ? internalPredicate :
-          new AndPredicate(m_userPredicate, internalPredicate));
+      predicate = (userPredicate == null ? internalPredicate :
+          new AndPredicate(userPredicate, internalPredicate));
     }
     return predicate;
   }
 
-  public Request createRequest() {
+  private Request createRequest() {
     Set<String> setProperties = new HashSet<String>();
 
     Map<String, TemporalInfo> mapTemporalInfo    = new HashMap<String, TemporalInfo>();
-    TemporalInfo              globalTemporalInfo = m_mapCategoryTemporalInfo.get(null);
+    TemporalInfo              globalTemporalInfo = temporalInfoMap.get(null);
 
-    for (String group : m_setQueryProperties) {
-      TemporalInfo temporalInfo = m_mapCategoryTemporalInfo.get(group);
+    for (String group : queryPropertySet) {
+      TemporalInfo temporalInfo = temporalInfoMap.get(group);
       if (temporalInfo != null) {
         mapTemporalInfo.put(group, temporalInfo);
       } else if (globalTemporalInfo != null) {
@@ -452,44 +553,24 @@ public class QueryImpl implements Query {
   }
 
   private void setParentIdsOnSubResource(Resource resource, ResourceInstance r) {
-    Map<Resource.Type, String> mapParentIds = m_resource.getIds();
+    Map<Resource.Type, String> mapParentIds = getIds();
     Map<Resource.Type, String> mapResourceIds = new HashMap<Resource.Type, String>(mapParentIds.size());
     for (Map.Entry<Resource.Type, String> resourceIdEntry : mapParentIds.entrySet()) {
       Resource.Type type = resourceIdEntry.getKey();
       String value = resourceIdEntry.getValue();
 
       if (value == null) {
-        Object o = resource.getPropertyValue(getClusterController().getSchema(type).getKeyPropertyId(type));
+        Object o = resource.getPropertyValue(clusterController.getSchema(type).getKeyPropertyId(type));
         value = o == null ? null : o.toString();
       }
       if (value != null) {
         mapResourceIds.put(type, value);
       }
     }
-    String resourceKeyProp = getClusterController().getSchema(resource.getType()).
+    String resourceKeyProp = clusterController.getSchema(resource.getType()).
         getKeyPropertyId(resource.getType());
-    //todo: shouldn't use toString here
+
     mapResourceIds.put(resource.getType(), resource.getPropertyValue(resourceKeyProp).toString());
     r.setIds(mapResourceIds);
   }
-
-  Result createResult() {
-    return new ResultImpl(true);
-  }
-
-  /**
-   * Copy template fields to given query
-   * @param query query to which fields will be copied
-   */
-  public void cloneFields(QueryImpl query) {
-    query.m_predicate = m_predicate;
-    query.m_mapCategoryTemporalInfo = m_mapCategoryTemporalInfo;
-    query.m_mapPropertyTemporalInfo = m_mapPropertyTemporalInfo;
-    query.m_mapSubResources = m_mapSubResources;
-    query.m_setQueryProperties = m_setQueryProperties;
-    query.m_pageRequest = m_pageRequest;
-    query.m_userPredicate = m_userPredicate;
-    query.allProperties = allProperties;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java
index 454da6f..a3a8e3a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstance.java
@@ -74,10 +74,4 @@ public interface ResourceInstance {
    * @return true if the resource is a collection resource; false otherwise
    */
   public boolean isCollectionResource();
-
-  /**
-   * Create clear copy of current resource instance
-   * @return new ResourceInstance
-   */
-  ResourceInstance createCopy();
 }

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
index e71f1fc..02d081e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
@@ -19,7 +19,9 @@
 
 package org.apache.ambari.server.api.resources;
 
+import org.apache.ambari.server.api.query.QueryImpl;
 import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 
 import java.util.Map;
 
@@ -34,6 +36,20 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory {
     /**
      * The resource definition for the specified type.
      */
+    ResourceDefinition resourceDefinition = getResourceDefinition(type, mapIds);
+
+    return new QueryImpl(mapIds, resourceDefinition, ClusterControllerHelper.getClusterController());
+  }
+
+  /**
+   * Get a resource definition for the given type.
+   *
+   * @param type    the resource type
+   * @param mapIds  the map of ids
+   *
+   * @return the resource definition
+   */
+  public static ResourceDefinition getResourceDefinition(Resource.Type type, Map<Resource.Type, String> mapIds) {
     ResourceDefinition resourceDefinition;
 
     //todo: consider ResourceDependencyManager : Map<Resource.Type, ResourceDefinition>
@@ -52,7 +68,7 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory {
         break;
 
       case Component:
-        resourceDefinition = new  ComponentResourceDefinition();
+        resourceDefinition = new ComponentResourceDefinition();
         break;
 
       case HostComponent:
@@ -82,27 +98,27 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory {
       case Stack:
         resourceDefinition = new StackResourceDefinition();
         break;
-        
+
       case StackVersion:
         resourceDefinition = new StackVersionResourceDefinition();
         break;
-        
+
       case StackService:
         resourceDefinition = new StackServiceResourceDefinition();
         break;
-        
+
       case StackServiceComponent:
         resourceDefinition = new StackServiceComponentResourceDefinition();
         break;
-        
+
       case StackConfiguration:
         resourceDefinition = new StackConfigurationResourceDefinition();
         break;
-        
+
       case OperatingSystem:
         resourceDefinition = new OperatingSystemResourceDefinition();
         break;
-        
+
       case Repository:
         resourceDefinition = new RepositoryResourceDefinition();
         break;
@@ -130,15 +146,15 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory {
       case TaskAttempt:
         resourceDefinition = new TaskAttemptResourceDefinition();
         break;
-        
+
       case RootService:
         resourceDefinition = new RootServiceResourceDefinition();
         break;
-        
+
       case RootServiceComponent:
         resourceDefinition = new RootServiceComponentResourceDefinition();
         break;
-        
+
       case RootServiceHostComponent:
         resourceDefinition = new RootServiceHostComponentResourceDefinition();
         break;
@@ -150,7 +166,6 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory {
       default:
         throw new IllegalArgumentException("Unsupported resource type: " + type);
     }
-
-    return new ResourceInstanceImpl(mapIds, resourceDefinition, this);
+    return resourceDefinition;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceImpl.java
deleted file mode 100644
index a62978c..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceImpl.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.ambari.server.api.resources;
-
-import org.apache.ambari.server.api.query.Query;
-import org.apache.ambari.server.api.query.QueryImpl;
-import org.apache.ambari.server.controller.spi.ClusterController;
-import org.apache.ambari.server.controller.spi.Resource;
-import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Resource instance which contains request specific state.
- */
-public class ResourceInstanceImpl implements ResourceInstance {
-
-  /**
-   * Query associated with the resource definition.
-   */
-  private Query m_query;
-
-  /**
-   * Map of primary and foreign keys and values necessary to identify the resource.
-   */
-  private Map<Resource.Type, String> m_mapResourceIds = new HashMap<Resource.Type, String>();
-
-  /**
-   * Definition for the resource type.  The definition contains all information specific to the
-   * resource type.
-   */
-  private ResourceDefinition m_resourceDefinition;
-
-  /**
-   * Sub-resource instances of this resource.
-   * Map of resource resource name to resource instance.
-   */
-  private Map<String, ResourceInstance> m_mapSubResources;
-
-  /**
-   * Factory for creating resource instances.
-   * Used to create sub-resource instances.
-   */
-  private ResourceInstanceFactory m_resourceFactory;
-
-  /**
-   * Cluster controller reference.
-   */
-  //todo: should be injected.
-  private ClusterController m_controller = ClusterControllerHelper.getClusterController();
-
-
-  public ResourceInstanceImpl(Map<Resource.Type, String> mapIds, ResourceDefinition resourceDefinition,
-                              ResourceInstanceFactory resourceFactory) {
-
-    m_resourceDefinition = resourceDefinition;
-    m_query              = new QueryImpl(this);
-    m_resourceFactory    = resourceFactory;
-
-    setIds(mapIds);
-  }
-
-  @Override
-  public void setIds(Map<Resource.Type, String> mapIds) {
-    m_mapResourceIds.putAll(mapIds);
-  }
-
-  @Override
-  public Map<Resource.Type, String> getIds() {
-    return new HashMap<Resource.Type, String>((m_mapResourceIds));
-  }
-
-  @Override
-  public Query getQuery() {
-    return m_query;
-  }
-
-  @Override
-  public ResourceDefinition getResourceDefinition() {
-    return m_resourceDefinition;
-  }
-
-
-  @Override
-  public Map<String, ResourceInstance> getSubResources() {
-    if (m_mapSubResources == null) {
-      m_mapSubResources = new HashMap<String, ResourceInstance>();
-      Set<SubResourceDefinition> setSubResourceDefs = getResourceDefinition().getSubResourceDefinitions();
-
-      for (SubResourceDefinition subResDef : setSubResourceDefs) {
-        ResourceInstance resource = m_resourceFactory.createResource(subResDef.getType(), getIds());
-
-        // ensure pk is returned
-        resource.getQuery().addLocalProperty(m_controller.getSchema(
-            subResDef.getType()).getKeyPropertyId(subResDef.getType()));
-        // add additionally required fk properties
-        for (Resource.Type fkType : subResDef.getAdditionalForeignKeys()) {
-          resource.getQuery().addLocalProperty(m_controller.getSchema(subResDef.getType()).getKeyPropertyId(fkType));
-        }
-
-        String subResourceName = subResDef.isCollection() ? resource.getResourceDefinition().getPluralName() :
-            resource.getResourceDefinition().getSingularName();
-
-        m_mapSubResources.put(subResourceName, resource);
-      }
-    }
-    return m_mapSubResources;
-  }
-
-  @Override
-  public boolean isCollectionResource() {
-    return getIds().get(getResourceDefinition().getType()) == null;
-  }
-
-  @Override
-  public ResourceInstance createCopy() {
-    ResourceInstanceImpl resourceInstance = new ResourceInstanceImpl(m_mapResourceIds, m_resourceDefinition, m_resourceFactory);
-    resourceInstance.m_mapSubResources = m_mapSubResources;
-    ((QueryImpl) m_query).cloneFields((QueryImpl) resourceInstance.m_query);
-
-    return resourceInstance;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
index 5f7f3e0..680a07e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProvider.java
@@ -275,7 +275,7 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
 
     String clusters = getSetString(clusterSet, -1);
     String hosts    = getSetString(hostSet, 100);
-    String metrics  = getSetString(metricSet, 50);
+    String metrics  = getSetString(metricSet, 60);
 
     URIBuilder uriBuilder = new URIBuilder();
 
@@ -285,7 +285,6 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
       uriBuilder.setScheme("http");
     }
 
-    
     uriBuilder.setHost(hostProvider.getGangliaCollectorHostName(clusterName));
     
     uriBuilder.setPath("/cgi-bin/rrd.py");
@@ -299,6 +298,9 @@ public abstract class GangliaPropertyProvider extends AbstractPropertyProvider {
 
     if (metrics.length() > 0) {
       uriBuilder.setParameter("m", metrics);
+    } else {
+      // get all metrics
+      uriBuilder.setParameter("m", ".*");
     }
 
     if (temporalInfo != null) {

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java
index 3b2f9c9..1a9079f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterControllerImpl.java
@@ -91,18 +91,9 @@ public class ClusterControllerImpl implements ClusterController {
 
   // ----- ClusterController -------------------------------------------------
 
-  public Iterable<Resource> getResources(Resource.Type type, Request request, Predicate predicate)
-      throws UnsupportedPropertyException,
-             SystemException,
-             NoSuchParentResourceException,
-             NoSuchResourceException {
-    return getResources(type, request, predicate, null).getIterable();
-  }
-
   @Override
-  public Set<Resource> getRawResources(Resource.Type type, Request request, Predicate predicate)
+  public Set<Resource> getResources(Resource.Type type, Request request, Predicate predicate)
       throws UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException, SystemException {
-
     Set<Resource> resources;
 
     ResourceProvider provider = ensureResourceProvider(type);
@@ -117,21 +108,35 @@ public class ClusterControllerImpl implements ClusterController {
             + provider.getClass().getName()
             + " for request type " + type.toString());
       }
+      // make sure that the providers can satisfy the request
       checkProperties(type, request, predicate);
 
+      // get the resources
       resources = provider.getResources(request, predicate);
+
+      // populate the resources with metrics and properties.
+      populateResources(type, resources, request, predicate);
+
+      // filter the fully populated resources with the given predicate
+      Iterable<Resource> iterable = getIterable(type, resources, request, predicate);
+      resources = new LinkedHashSet<Resource>();
+      for (Resource resource : iterable){
+        resources.add(resource);
+      }
     }
     return resources;
   }
 
   @Override
-  public Iterable<Resource> getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate)
+  public Iterable<Resource> getIterable(Resource.Type type, Set<Resource> providerResources,
+                                        Request request, Predicate predicate)
       throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException {
-    return getResources(type, providerResources, predicate, null).getIterable();
+    return getPage(type, providerResources, request, predicate, null).getIterable();
   }
 
   @Override
-  public PageResponse getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate, PageRequest pageRequest)
+  public PageResponse getPage(Resource.Type type, Set<Resource> providerResources,
+                              Request request, Predicate predicate, PageRequest pageRequest)
       throws UnsupportedPropertyException,
       SystemException,
       NoSuchResourceException,
@@ -144,8 +149,6 @@ public class ClusterControllerImpl implements ClusterController {
         (ResourcePredicateEvaluator) provider : DEFAULT_RESOURCE_PREDICATE_EVALUATOR;
 
     if (!providerResources.isEmpty()) {
-
-
       Comparator<Resource> resourceComparator = pageRequest == null || pageRequest.getComparator() == null ?
           comparator : pageRequest.getComparator();
 
@@ -156,16 +159,16 @@ public class ClusterControllerImpl implements ClusterController {
         switch (pageRequest.getStartingPoint()) {
           case Beginning:
             return getPageFromOffset(pageRequest.getPageSize(), 0,
-              sortedResources, predicate, evaluator);
+              sortedResources, request, predicate, evaluator);
           case End:
             return getPageToOffset(pageRequest.getPageSize(), -1,
-              sortedResources, predicate, evaluator);
+                sortedResources, request, predicate, evaluator);
           case OffsetStart:
             return getPageFromOffset(pageRequest.getPageSize(),
-              pageRequest.getOffset(), sortedResources, predicate, evaluator);
+              pageRequest.getOffset(), sortedResources, request, predicate, evaluator);
           case OffsetEnd:
             return getPageToOffset(pageRequest.getPageSize(),
-              pageRequest.getOffset(), sortedResources, predicate, evaluator);
+              pageRequest.getOffset(), sortedResources, request, predicate, evaluator);
           // TODO : need to support the following cases for pagination
 //          case PredicateStart:
 //          case PredicateEnd:
@@ -176,21 +179,10 @@ public class ClusterControllerImpl implements ClusterController {
       resources = providerResources;
     }
 
-    return new PageResponseImpl(new ResourceIterable(resources, predicate, evaluator),
+    return new PageResponseImpl(new ResourceIterable(resources, request, predicate, evaluator),
       0, null, null);
   }
 
-  public PageResponse getResources(Resource.Type type, Request request, Predicate predicate, PageRequest pageRequest)
-      throws UnsupportedPropertyException,
-             SystemException,
-             NoSuchResourceException,
-             NoSuchParentResourceException {
-
-    Set<Resource> providerResources = getRawResources(type, request, predicate);
-    populateResources(type, providerResources, request, predicate);
-    return getResources(type, providerResources, predicate, pageRequest);
-  }
-
   @Override
   public Schema getSchema(Resource.Type type) {
     Schema schema;
@@ -267,6 +259,58 @@ public class ClusterControllerImpl implements ClusterController {
   // ----- helper methods ----------------------------------------------------
 
   /**
+   * Get an iterable set of resources filtered by the given request and
+   * predicate objects.
+   *
+   * @param type       type of resources
+   * @param request    the request
+   * @param predicate  the predicate object which filters which resources are returned
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  protected Iterable<Resource> getResourceIterable(Resource.Type type, Request request, Predicate predicate)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchParentResourceException,
+      NoSuchResourceException {
+    return getResources(type, request, predicate, null).getIterable();
+  }
+
+  /**
+   * Get a page of resources filtered by the given request, predicate objects and
+   * page request.
+   *
+   * @param type               type of resources
+   * @param request            the request
+   * @param predicate          the predicate object which filters which resources are returned
+   * @param pageRequest        the page request for a paginated response
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  protected  PageResponse getResources(Resource.Type type, Request request, Predicate predicate, PageRequest pageRequest)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException {
+
+    Set<Resource> providerResources = getResources(type, request, predicate);
+    populateResources(type, providerResources, request, predicate);
+    return getPage(type, providerResources, request, predicate, pageRequest);
+  }
+
+  /**
    * Check to make sure that all the property ids specified in the given request and
    * predicate are supported by the resource provider or property providers for the
    * given type.
@@ -343,7 +387,7 @@ public class ClusterControllerImpl implements ClusterController {
     Set<String>  keyPropertyIds = new HashSet<String>(provider.getKeyPropertyIds().values());
     Request      readRequest    = PropertyHelper.getReadRequest(keyPropertyIds);
 
-    Iterable<Resource> resources = getResources(type, readRequest, predicate);
+    Iterable<Resource> resources = getResourceIterable(type, readRequest, predicate);
 
     PredicateBuilder pb = new PredicateBuilder();
     PredicateBuilder.PredicateBuilderWithPredicate pbWithPredicate = null;
@@ -384,8 +428,7 @@ public class ClusterControllerImpl implements ClusterController {
    *
    * @throws SystemException if unable to populate the resources
    */
-  @Override
-  public Set<Resource> populateResources(Resource.Type type,
+  protected Set<Resource> populateResources(Resource.Type type,
                                          Set<Resource> resources,
                                          Request request,
                                          Predicate predicate) throws SystemException {
@@ -400,30 +443,6 @@ public class ClusterControllerImpl implements ClusterController {
   }
 
   /**
-   * Performs bulk population of the given resources from the associated property providers.  This
-   * method may filter the resources based on the predicate and return a subset
-   * of the given resources.
-   * @param resourceMap resources grouped by type
-   * @param requestMap type-request map
-   * @param predicateMap type-predicate map
-   * @return the set of resources that were successfully populated grouped by resource type
-   * @throws SystemException
-   */
-  @Override
-  public Map<Resource.Type, Set<Resource>> populateResources(Map<Resource.Type, Set<Resource>> resourceMap,
-                                Map<Resource.Type, Request> requestMap,
-                                Map<Resource.Type, Predicate> predicateMap) throws SystemException {
-
-    Map<Resource.Type, Set<Resource>> keepers = new HashMap<Resource.Type, Set<Resource>>();
-
-    for (Resource.Type type : resourceMap.keySet()) {
-      keepers.put(type, populateResources(type, resourceMap.get(type), requestMap.get(type), predicateMap.get(type)));
-    }
-
-    return keepers;
-  }
-
-  /**
    * Indicates whether or not the given property provider can service the given request.
    *
    * @param provider   the property provider
@@ -483,13 +502,14 @@ public class ClusterControllerImpl implements ClusterController {
    * @param pageSize   the page size
    * @param offset     the offset
    * @param resources  the set of resources
+   * @param request    the request
    * @param predicate  the predicate
    *
    * @return a page response containing a page of resources
    */
   private PageResponse getPageFromOffset(int pageSize, int offset,
                                          NavigableSet<Resource> resources,
-                                         Predicate predicate,
+                                         Request request, Predicate predicate,
                                          ResourcePredicateEvaluator evaluator) {
 
     int                currentOffset = 0;
@@ -509,7 +529,7 @@ public class ClusterControllerImpl implements ClusterController {
     }
 
     return new PageResponseImpl(new ResourceIterable(pageResources,
-        predicate, evaluator),
+        request, predicate, evaluator),
         currentOffset,
         previous,
         iterator.hasNext() ? iterator.next() : null);
@@ -521,13 +541,14 @@ public class ClusterControllerImpl implements ClusterController {
    * @param pageSize   the page size
    * @param offset     the offset; -1 indicates the end of the resource set
    * @param resources  the set of resources
+   * @param request    the request
    * @param predicate  the predicate
    *
    * @return a page response containing a page of resources
    */
   private PageResponse getPageToOffset(int pageSize, int offset,
                                        NavigableSet<Resource> resources,
-                                       Predicate predicate,
+                                       Request request, Predicate predicate,
                                        ResourcePredicateEvaluator evaluator) {
 
     int                currentOffset = resources.size() - 1;
@@ -550,7 +571,7 @@ public class ClusterControllerImpl implements ClusterController {
     }
 
     return new PageResponseImpl(new ResourceIterable(new
-        LinkedHashSet<Resource>(pageResources), predicate, evaluator),
+        LinkedHashSet<Resource>(pageResources), request, predicate, evaluator),
         currentOffset + 1,
         iterator.hasNext() ? iterator.next() : null,
         next);
@@ -565,7 +586,8 @@ public class ClusterControllerImpl implements ClusterController {
     return comparator;
   }
 
-// ----- ResourceIterable inner class --------------------------------------
+
+  // ----- ResourceIterable inner class --------------------------------------
 
   private static class ResourceIterable implements Iterable<Resource> {
 
@@ -575,6 +597,11 @@ public class ClusterControllerImpl implements ClusterController {
     private final Set<Resource> resources;
 
     /**
+     * The associated request.
+     */
+    private final Request request;
+
+    /**
      * The predicate used to filter the set.
      */
     private final Predicate predicate;
@@ -584,17 +611,21 @@ public class ClusterControllerImpl implements ClusterController {
      */
     private final ResourcePredicateEvaluator evaluator;
 
+
     // ----- Constructors ----------------------------------------------------
 
     /**
      * Create a ResourceIterable.
      *
      * @param resources  the set of resources to iterate over
+     * @param request    the request
      * @param predicate  the predicate used to filter the set of resources
+     * @param evaluator  the evaluator used to evaluate with the given predicate
      */
-    private ResourceIterable(Set<Resource> resources, Predicate predicate,
+    private ResourceIterable(Set<Resource> resources, Request request, Predicate predicate,
                              ResourcePredicateEvaluator evaluator) {
       this.resources = resources;
+      this.request   = request;
       this.predicate = predicate;
       this.evaluator = evaluator;
     }
@@ -603,7 +634,7 @@ public class ClusterControllerImpl implements ClusterController {
 
     @Override
     public Iterator<Resource> iterator() {
-      return new ResourceIterator(resources, predicate, evaluator);
+      return new ResourceIterator(resources, request, predicate, evaluator);
     }
   }
 
@@ -618,6 +649,11 @@ public class ClusterControllerImpl implements ClusterController {
     private final Iterator<Resource> iterator;
 
     /**
+     * The associated request.
+     */
+    private final Request request;
+
+    /**
      * The predicate used to filter the resource being iterated over.
      */
     private final Predicate predicate;
@@ -638,13 +674,16 @@ public class ClusterControllerImpl implements ClusterController {
      * Create a new ResourceIterator.
      *
      * @param resources  the set of resources to iterate over
+     * @param request    the request
      * @param predicate  the predicate used to filter the set of resources
+     * @param evaluator  the evaluator used to evaluate with the given predicate
      */
-    private ResourceIterator(Set<Resource> resources, Predicate predicate,
+    private ResourceIterator(Set<Resource> resources, Request request, Predicate predicate,
                              ResourcePredicateEvaluator evaluator) {
-      this.iterator = resources.iterator();
-      this.predicate = predicate;
-      this.evaluator = evaluator;
+      this.iterator     = resources.iterator();
+      this.request      = request;
+      this.predicate    = predicate;
+      this.evaluator    = evaluator;
       this.nextResource = getNextResource();
     }
 
@@ -684,6 +723,7 @@ public class ClusterControllerImpl implements ClusterController {
         Resource next = iterator.next();
 
         if (predicate == null || evaluator.evaluate(predicate, next)) {
+          // TODO : copy the resource and only include the requested properties.
           return next;
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java
index debe534..0ab15db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/ClusterController.java
@@ -17,8 +17,6 @@
  */
 package org.apache.ambari.server.controller.spi;
 
-
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -28,8 +26,79 @@ import java.util.Set;
  */
 public interface ClusterController {
 
+
   // ----- Monitoring ------------------------------------------------------
 
+  /**
+   * Get the resources of the given type filtered by the given request and
+   * predicate objects.
+   *
+   * @param type      the type of the requested resources
+   * @param request   the request object which defines the desired set of properties
+   * @param predicate the predicate object which filters which resources are returned
+   *
+   * @return an iterable object of the requested resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  Set<Resource> getResources(Resource.Type type, Request request, Predicate predicate)
+      throws UnsupportedPropertyException,
+      NoSuchResourceException,
+      NoSuchParentResourceException,
+      SystemException;
+
+  /**
+   * Get an iterable set of resources from the given set of resources filtered by the
+   * given request and predicate objects.
+   *
+   * @param type               type of resources
+   * @param providerResources  set of populated Resources
+   * @param request            the request
+   * @param predicate          the predicate object which filters which resources are returned
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  Iterable<Resource> getIterable(Resource.Type type, Set<Resource> providerResources,
+                                 Request request, Predicate predicate)
+      throws NoSuchParentResourceException,
+      UnsupportedPropertyException,
+      NoSuchResourceException,
+      SystemException;
+
+  /**
+   * Get a page of resources from the given set filtered by the given request,
+   * predicate objects and page request.
+   *
+   * @param type               type of resources
+   * @param providerResources  set of populated Resources
+   * @param request            the request
+   * @param predicate          the predicate object which filters which resources are returned
+   * @param pageRequest        the page request for a paginated response
+   *
+   * @return a page response representing the requested page of resources
+   *
+   * @throws UnsupportedPropertyException thrown if the request or predicate contain
+   *                                      unsupported property ids
+   * @throws SystemException an internal exception occurred
+   * @throws NoSuchResourceException no matching resource(s) found
+   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
+   */
+  PageResponse getPage(Resource.Type type, Set<Resource> providerResources,
+                       Request request, Predicate predicate, PageRequest pageRequest)
+      throws UnsupportedPropertyException,
+      SystemException,
+      NoSuchResourceException,
+      NoSuchParentResourceException;
 
   /**
    * Get the {@link Schema schema} for the given resource type.  The schema
@@ -106,103 +175,4 @@ public interface ClusterController {
              SystemException,
              NoSuchResourceException,
              NoSuchParentResourceException ;
-
-  /**
-   * Get the not populated resources of the given type filtered by the given request and
-   * predicate objects.
-   *
-   * @param type      the type of the requested resources
-   * @param request   the request object which defines the desired set of properties
-   * @param predicate the predicate object which filters which resources are returned
-   *
-   * @return an iterable object of the requested resources
-   *
-   * @throws UnsupportedPropertyException thrown if the request or predicate contain
-   *                                      unsupported property ids
-   * @throws SystemException an internal exception occurred
-   * @throws NoSuchResourceException no matching resource(s) found
-   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
-   */
-  Set<Resource> getRawResources(Resource.Type type, Request request, Predicate predicate)
-      throws UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException, SystemException;
-
-  /**
-   * Get a page of resources of the given type filtered by the given request,
-   * predicate objects and page request.
-   *
-   *
-   * @param type type of resources
-   * @param providerResources set of populated Resources
-   * @param predicate   the predicate object which filters which resources are returned
-   * @param pageRequest the page request for a paginated response
-   *
-   * @return a page response representing the requested page of resources
-   *
-   * @throws UnsupportedPropertyException thrown if the request or predicate contain
-   *                                      unsupported property ids
-   * @throws SystemException an internal exception occurred
-   * @throws NoSuchResourceException no matching resource(s) found
-   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
-   */
-  PageResponse getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate, PageRequest pageRequest)
-      throws UnsupportedPropertyException,
-      SystemException,
-      NoSuchResourceException,
-      NoSuchParentResourceException;
-
-  /**
-   * Get a page of resources of the given type filtered by the given request,
-   * predicate objects and page request.
-   *
-   *
-   * @param type type of resources
-   * @param providerResources set of populated Resources
-   * @param predicate   the predicate object which filters which resources are returned
-   *
-   * @return a page response representing the requested page of resources
-   *
-   * @throws UnsupportedPropertyException thrown if the request or predicate contain
-   *                                      unsupported property ids
-   * @throws SystemException an internal exception occurred
-   * @throws NoSuchResourceException no matching resource(s) found
-   * @throws NoSuchParentResourceException a specified parent resource doesn't exist
-   */
-  Iterable<Resource> getResources(Resource.Type type, Set<Resource> providerResources, Predicate predicate)
-  throws NoSuchParentResourceException, UnsupportedPropertyException, NoSuchResourceException, SystemException;
-
-  /**
-   * Populate the given resources from the associated property providers.  This
-   * method may filter the resources based on the predicate and return a subset
-   * of the given resources.
-   *
-   * @param type       the resource type
-   * @param resources  the resources to be populated
-   * @param request    the request
-   * @param predicate  the predicate
-   *
-   * @return the set of resources that were successfully populated
-   *
-   * @throws SystemException if unable to populate the resources
-   */
-  Set<Resource> populateResources(Resource.Type type,
-                                  Set<Resource> resources,
-                                  Request request,
-                                  Predicate predicate) throws SystemException;
-
-  /**
-   * Performs bulk population of the given resources from the associated property providers.  This
-   * method may filter the resources based on the predicate and return a subset
-   * of the given resources.
-   *
-   * @param resourceMap resources grouped by type
-   * @param requestMap type-request map
-   * @param predicateMap type-predicate map
-   *
-   * @return the set of resources that were successfully populated grouped by resource type
-   *
-   * @throws SystemException
-   */
-  Map<Resource.Type, Set<Resource>> populateResources(Map<Resource.Type, Set<Resource>> resourceMap,
-                         Map<Resource.Type, Request> requestMap,
-                         Map<Resource.Type, Predicate> predicateMap) throws SystemException;
 }

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/fc427d9f/ambari-server/src/test/java/org/apache/ambari/server/api/TestSuite.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/TestSuite.java b/ambari-server/src/test/java/org/apache/ambari/server/api/TestSuite.java
index 31fe7f8..716347f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/TestSuite.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/TestSuite.java
@@ -22,13 +22,34 @@ package org.apache.ambari.server.api;
  * All api unit tests.
  */
 
-import org.apache.ambari.server.api.handlers.*;
-import org.apache.ambari.server.api.predicate.QueryLexerTest;
-import org.apache.ambari.server.api.predicate.QueryParserTest;
-import org.apache.ambari.server.api.predicate.operators.*;
+import org.apache.ambari.server.api.handlers.CreateHandlerTest;
+import org.apache.ambari.server.api.handlers.DeleteHandlerTest;
+import org.apache.ambari.server.api.handlers.QueryCreateHandlerTest;
+import org.apache.ambari.server.api.handlers.ReadHandlerTest;
+import org.apache.ambari.server.api.handlers.UpdateHandlerTest;
+import org.apache.ambari.server.api.predicate.operators.AndOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.EqualsOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.GreaterEqualsOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.GreaterOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.InOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.LessEqualsOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.NotEqualsOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.NotOperatorTest;
+import org.apache.ambari.server.api.predicate.operators.OrOperatorTest;
 import org.apache.ambari.server.api.query.QueryImplTest;
-import org.apache.ambari.server.api.resources.ResourceInstanceImplTest;
-import org.apache.ambari.server.api.services.*;
+import org.apache.ambari.server.api.services.ClusterServiceTest;
+import org.apache.ambari.server.api.services.ComponentServiceTest;
+import org.apache.ambari.server.api.services.DeleteRequestTest;
+import org.apache.ambari.server.api.services.GetRequestTest;
+import org.apache.ambari.server.api.services.HostComponentServiceTest;
+import org.apache.ambari.server.api.services.HostServiceTest;
+import org.apache.ambari.server.api.services.NamedPropertySetTest;
+import org.apache.ambari.server.api.services.PersistenceManagerImplTest;
+import org.apache.ambari.server.api.services.PostRequestTest;
+import org.apache.ambari.server.api.services.PutRequestTest;
+import org.apache.ambari.server.api.services.QueryPostRequestTest;
+import org.apache.ambari.server.api.services.RequestBodyTest;
+import org.apache.ambari.server.api.services.ServiceServiceTest;
 import org.apache.ambari.server.api.services.parsers.BodyParseExceptionTest;
 import org.apache.ambari.server.api.services.parsers.JsonRequestBodyParserTest;
 import org.apache.ambari.server.api.services.serializers.JsonSerializerTest;
@@ -41,7 +62,6 @@ import org.junit.runners.Suite;
     JsonRequestBodyParserTest.class, CreateHandlerTest.class, UpdateHandlerTest.class, DeleteHandlerTest.class,
     PersistenceManagerImplTest.class, GetRequestTest.class, PutRequestTest.class, PostRequestTest.class,
     DeleteRequestTest.class, QueryPostRequestTest.class, JsonSerializerTest.class, QueryCreateHandlerTest.class,
-    ResourceInstanceImplTest.class, QueryLexerTest.class, QueryParserTest.class, IsEmptyOperatorTest.class,
     InOperatorTest.class,AndOperatorTest.class, OrOperatorTest.class, EqualsOperatorTest.class,
     GreaterEqualsOperatorTest.class, GreaterOperatorTest.class, LessEqualsOperatorTest.class,
     LessEqualsOperatorTest.class, NotEqualsOperatorTest.class, NotOperatorTest.class, RequestBodyTest.class,


Mime
View raw message