ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject [2/3] ambari git commit: AMBARI-19468. Run ZK operations in logsearch internally and enable ACL security (oleewere)
Date Thu, 19 Jan 2017 13:13:28 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
deleted file mode 100644
index da76924..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.dao;
-
-import org.apache.ambari.logsearch.conf.SolrPropsConfig;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
-
-import javax.inject.Named;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-
-@Named
-class SolrCollectionDao {
-
-  private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
-
-  private static final int SETUP_RETRY_SECOND = 30;
-  private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
-  private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
-
-  /**
-   * This will try to get the collections from the Solr. Ping doesn't work if
-   * collection is not given
-   */
-  boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
-    int waitDurationMS = 3 * 60 * 1000;
-    boolean status = false;
-    try {
-      long beginTimeMS = System.currentTimeMillis();
-      long waitIntervalMS = 2000;
-      int pingCount = 0;
-      while (true) {
-        pingCount++;
-        try {
-          List<String> collectionList = getCollections(cloudSolrClient);
-          if (collectionList != null) {
-            LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList);
-            status = true;
-            break;
-          }
-        } catch (Exception ex) {
-          LOG.error("Error while doing Solr check", ex);
-        }
-        if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
-          LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
-            "If you are using alias, then you might have to restart LogSearch after Solr is up and running.");
-          break;
-        } else {
-          LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
-            "Will sleep for " + waitIntervalMS + " ms and try again.");
-        }
-        Thread.sleep(waitIntervalMS);
-
-      }
-    } catch (Throwable t) {
-      LOG.error("Seems Solr is not up.");
-    }
-    return status;
-  }
-
-  void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
-    boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
-    LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
-    if (!setupStatus) {
-      // Start a background thread to do setup
-      Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) {
-        @Override
-        public void run() {
-          LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection());
-          int retryCount = 0;
-          while (true) {
-            try {
-              Thread.sleep(SETUP_RETRY_SECOND * 1000);
-              retryCount++;
-              boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
-              if (setupStatus) {
-                LOG.info("Setup for collection " + solrPropsConfig.getCollection() + " is successful. Exiting setup retry thread");
-                break;
-              }
-            } catch (InterruptedException sleepInterrupted) {
-              LOG.info("Sleep interrupted while setting up collection " + solrPropsConfig.getCollection());
-              break;
-            } catch (Exception e) {
-              LOG.error("Error setting up collection=" + solrPropsConfig.getCollection(), e);
-            }
-            LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
-          }
-        }
-      };
-      setupThread.setDaemon(true);
-      setupThread.start();
-    }
-  }
-
-  private boolean createCollectionsIfNeeded(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
-    boolean result = false;
-    try {
-      List<String> allCollectionList = getCollections(solrClient);
-      if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
-        result = createCollection(solrClient, solrPropsConfig, allCollectionList);
-      } else {
-        result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, allCollectionList);
-      }
-    } catch (Exception ex) {
-      LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection(), ex);
-    }
-    return result;
-  }
-
-  @SuppressWarnings("unchecked")
-  List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
-    IOException {
-    try {
-      CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
-      CollectionAdminResponse response = colListReq.process(solrClient);
-      if (response.getStatus() != 0) {
-        LOG.error("Error getting collection list from solr.  response=" + response);
-        return null;
-      }
-      return (List<String>) response.getResponse().get("collections");
-    } catch (SolrException e) {
-      LOG.error("getCollections() operation failed", e);
-      return new ArrayList<>();
-    }
-  }
-
-  private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
-    throws Exception {
-    LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
-      + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
-
-    // Default is true, because if the collection and shard is already there, then it will return true
-    boolean returnValue = true;
-
-    List<String> shardsList = new ArrayList<String>();
-    for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
-      shardsList.add("shard" + i);
-    }
-    String shardsListStr = StringUtils.join(shardsList, ',');
-
-    // Check if collection is already in zookeeper
-    if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
-      LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
-      CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
-      collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
-      collectionCreateRequest.setRouterName("implicit");
-      collectionCreateRequest.setShards(shardsListStr);
-      collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
-      collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
-      collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
-      collectionCreateRequest.setRouterField(ROUTER_FIELD);
-      collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
-
-      CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
-      if (createResponse.getStatus() != 0) {
-        returnValue = false;
-        LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
-          + ", shardsList=" + shardsList +", response=" + createResponse);
-      } else {
-        LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
-      }
-    } else {
-      LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
-      Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
-      Collection<String> existingShards = getShards(slices, solrPropsConfig);
-      if (existingShards.size() < shardsList.size()) {
-        try {
-          updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
-        } catch (Throwable t) {
-          returnValue = false;
-          LOG.error(String.format("Exception during updating collection (%s)", t));
-        }
-      }
-      for (String shard : shardsList) {
-        if (!existingShards.contains(shard)) {
-          try {
-            LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
-            CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
-            createShardRequest.setCollectionName(solrPropsConfig.getCollection());
-            createShardRequest.setShardName(shard);
-            CollectionAdminResponse response = createShardRequest.process(solrClient);
-            if (response.getStatus() != 0) {
-              LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
-              returnValue = false;
-              break;
-            } else {
-              LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
-            }
-          } catch (Throwable t) {
-            LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
-            returnValue = false;
-            break;
-          }
-        }
-      }
-    }
-    return returnValue;
-  }
-
-  private String getRandomBaseUrl(Collection<Slice> slices) {
-    String coreUrl = null;
-    if (slices != null) {
-      for (Slice slice : slices) {
-        if (!slice.getReplicas().isEmpty()) {
-          Replica replica = slice.getReplicas().iterator().next();
-          coreUrl = replica.getStr("base_url");
-          if (coreUrl != null) {
-            break;
-          }
-        }
-      }
-    }
-    return coreUrl;
-  }
-
-  private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
-    String baseUrl = getRandomBaseUrl(slices);
-    if (baseUrl != null) {
-      CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
-      HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
-        solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
-      HttpResponse response = httpClient.execute(request);
-      if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
-        throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
-      }
-    } else {
-      throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
-    }
-  }
-
-  private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
-    ZkStateReader reader = solrClient.getZkStateReader();
-    return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
-  }
-
-  private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
-    Collection<String> list = new HashSet<>();
-    for (Slice slice : slices) {
-      for (Replica replica : slice.getReplicas()) {
-        LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
-          ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
-        list.add(slice.getName());
-      }
-    }
-    return list;
-  }
-
-  private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
-
-    if (allCollectionList.contains(solrPropsConfig.getCollection())) {
-      LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
-      return true;
-    }
-
-    LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
-      ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
-
-    CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
-    collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
-    collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
-    collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
-    collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
-    collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
-    CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
-    if (createResponse.getStatus() != 0) {
-      LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
-      return false;
-    } else {
-      LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
-        ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
-      return true;
-    }
-  }
-
-  private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
-    return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
-  }
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 39b65ae..0568fd7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,6 +22,9 @@ package org.apache.ambari.logsearch.dao;
 import org.apache.ambari.logsearch.common.LogSearchContext;
 import org.apache.ambari.logsearch.common.LogType;
 import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
 import org.apache.ambari.logsearch.util.RESTErrorUtil;
 import org.apache.ambari.logsearch.util.SolrUtil;
 import org.apache.log4j.Logger;
@@ -36,6 +39,7 @@ import org.springframework.data.solr.core.SolrCallback;
 import org.springframework.data.solr.core.SolrTemplate;
 import org.springframework.data.solr.core.query.SolrDataQuery;
 
+import javax.inject.Inject;
 import java.io.IOException;
 
 public abstract class SolrDaoBase {
@@ -44,6 +48,9 @@ public abstract class SolrDaoBase {
   private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
 
   private LogType logType;
+
+  @Inject
+  private SolrKerberosConfig solrKerberosConfig;
   
   protected SolrDaoBase(LogType logType) {
     this.logType = logType;
@@ -108,5 +115,14 @@ public abstract class SolrDaoBase {
 
   public abstract SolrTemplate getSolrTemplate();
 
-  public abstract SolrSchemaFieldDao getSolrSchemaFieldDao();
+  public abstract void setSolrTemplate(SolrTemplate solrTemplate);
+
+  public abstract SolrCollectionState getSolrCollectionState();
+
+  public abstract SolrPropsConfig getSolrPropsConfig();
+
+  public SolrKerberosConfig getSolrKerberosConfig() {
+    return this.solrKerberosConfig;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index e6a562e..01378c1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -43,7 +43,6 @@ import org.apache.solr.common.util.NamedList;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.data.solr.core.SolrTemplate;
 import org.springframework.scheduling.annotation.Scheduled;
 
 import java.io.IOException;
@@ -54,54 +53,30 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import javax.annotation.PostConstruct;
 import javax.inject.Inject;
-import javax.inject.Named;
 
 public class SolrSchemaFieldDao {
 
   private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
 
   private static final int RETRY_SECOND = 30;
-  
+
   @Inject
-  @Named("serviceSolrTemplate")
-  private SolrTemplate serviceSolrTemplate;
+  private ServiceLogsSolrDao serviceLogsSolrDao;
 
   @Inject
-  @Named("auditSolrTemplate")
-  private SolrTemplate auditSolrTemplate;
+  private AuditSolrDao auditSolrDao;
   
   @Inject
   private SolrUserPropsConfig solrUserConfigPropsConfig;
   
-  private CloudSolrClient serviceSolrClient;
-  private CloudSolrClient auditSolrClient;
-  
   private int retryCount;
   private int skipCount;
-
-  private boolean serviceCollectionSetUp = false;
-  private boolean auditCollectionSetUp = false;
   
   private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
   private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
   private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
   private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
-
-  @PostConstruct
-  public void init() {
-    this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient();
-    this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient();
-  }
-  
-  void serviceCollectionSetUp() {
-    this.serviceCollectionSetUp = true;
-  }
-  
-  void auditCollectionSetUp() {
-    this.auditCollectionSetUp = true;
-  }
   
   @Scheduled(fixedDelay = RETRY_SECOND * 1000)
   public void populateAllSchemaFields() {
@@ -109,11 +84,12 @@ public class SolrSchemaFieldDao {
       skipCount--;
       return;
     }
-    
-    if (serviceCollectionSetUp) {
+    if (serviceLogsSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+      CloudSolrClient serviceSolrClient = (CloudSolrClient) serviceLogsSolrDao.getSolrTemplate().getSolrClient();
       populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap);
     }
-    if (auditCollectionSetUp) {
+    if (auditSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+      CloudSolrClient auditSolrClient = (CloudSolrClient) auditSolrDao.getSolrTemplate().getSolrClient();
       populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap);
     }
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index 106bedc..13d7e03 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -32,7 +32,11 @@ import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
 import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.common.LogSearchContext;
 import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
 import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.configurer.LogfeederFilterConfigurer;
+import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer;
 import org.apache.ambari.logsearch.model.common.LogFeederDataMap;
 import org.apache.ambari.logsearch.model.common.LogfeederFilterData;
 import org.apache.solr.client.solrj.SolrQuery;
@@ -66,12 +70,13 @@ public class UserConfigSolrDao extends SolrDaoBase {
   private SolrUserPropsConfig solrUserConfig;
 
   @Inject
-  private SolrCollectionDao solrCollectionDao;
-
-  @Inject
   @Named("userConfigSolrTemplate")
   private SolrTemplate userConfigSolrTemplate;
 
+  @Inject
+  @Named("solrUserConfigState")
+  private SolrCollectionState solrUserConfigState;
+
   public UserConfigSolrDao() {
     super(LogType.SERVICE);
   }
@@ -81,6 +86,11 @@ public class UserConfigSolrDao extends SolrDaoBase {
     return userConfigSolrTemplate;
   }
 
+  @Override
+  public void setSolrTemplate(SolrTemplate solrTemplate) {
+    this.userConfigSolrTemplate = solrTemplate;
+  }
+
   @PostConstruct
   public void postConstructor() {
     String solrUrl = solrUserConfig.getSolrUrl();
@@ -88,24 +98,14 @@ public class UserConfigSolrDao extends SolrDaoBase {
     String collection = solrUserConfig.getCollection();
 
     try {
-      solrCollectionDao.checkSolrStatus(getSolrClient());
-      solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
-      intializeLogFeederFilter();
-
+      new SolrCollectionConfigurer(this).start();
+      new LogfeederFilterConfigurer(this).start();
     } catch (Exception e) {
       LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
           ", collection=" + collection, e);
     }
   }
 
-  private void intializeLogFeederFilter() {
-    try {
-      getUserFilter();
-    } catch (SolrServerException | IOException e) {
-      LOG.error("not able to save logfeeder filter while initialization", e);
-    }
-  }
-
   public void saveUserFilter(LogFeederDataMap logfeederFilterWrapper) throws SolrException, SolrServerException, IOException {
     String filterName = LogSearchConstants.LOGFEEDER_FILTER_NAME;
     String json = JSONUtil.objToJson(logfeederFilterWrapper);
@@ -179,7 +179,12 @@ public class UserConfigSolrDao extends SolrDaoBase {
   }
 
   @Override
-  public SolrSchemaFieldDao getSolrSchemaFieldDao() {
-    throw new UnsupportedOperationException();
+  public SolrCollectionState getSolrCollectionState() {
+    return solrUserConfigState;
+  }
+
+  @Override
+  public SolrPropsConfig getSolrPropsConfig() {
+    return solrUserConfig;
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 0fc9852..caf0636 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -115,4 +115,11 @@ public class DocConstants {
     public static final String GET_USER_FILTER_BY_ID_OD = "Get user filter by id";
     public static final String GET_ALL_USER_NAMES_OD = "Get all user names";
   }
+
+  public class StatusOperationDescriptions {
+    public static final String STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+    public static final String SERVICE_LOGS_STATUS_OD = "Get statuses for service log collection (not health state - show true if something already done)";
+    public static final String AUDIT_LOGS_STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+    public static final String USER_CONFIG_STATUS_OD = "Get statuses for userconfig collection (not health state - show true if something already done)";
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
new file mode 100644
index 0000000..fde176f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class ACLHandler implements SolrZkRequestHandler<Boolean> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ACLHandler.class);
+
+  @Override
+  public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+    List<ACL> aclsToSetList = solrPropsConfig.getZkAcls();
+    if (CollectionUtils.isNotEmpty(aclsToSetList)) {
+      LOG.info("Setting acls for '{}' collection...", solrPropsConfig.getCollection());
+      SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+      SolrZooKeeper solrZooKeeper = zkClient.getSolrZooKeeper();
+      String collectionPath = String.format("/collections/%s", solrPropsConfig.getCollection());
+      String configsPath = String.format("/configs/%s", solrPropsConfig.getConfigName());
+      List<ACL> collectionAcls = solrZooKeeper.getACL(collectionPath, new Stat());
+      if (isRefreshAclsNeeded(aclsToSetList, collectionAcls)) {
+        LOG.info("Acls differs for {}, update acls.", collectionPath);
+        setRecursivelyOn(solrZooKeeper, collectionPath, aclsToSetList);
+      }
+      List<ACL> configsAcls = solrZooKeeper.getACL(configsPath, new Stat());
+      if (isRefreshAclsNeeded(aclsToSetList, configsAcls)) {
+        LOG.info("Acls differs for {}, update acls.", configsPath);
+        setRecursivelyOn(solrZooKeeper, configsPath, aclsToSetList);
+      }
+    }
+    return true;
+  }
+
+  private boolean isRefreshAclsNeeded(List<ACL> acls, List<ACL> newAcls) {
+    boolean result = false;
+    if (acls != null) {
+      if (acls.size() != newAcls.size()) {
+        return true;
+      }
+      result = aclDiffers(acls, newAcls);
+      if (!result) {
+        result = aclDiffers(newAcls, acls);
+      }
+    }
+    return result;
+  }
+
+  private boolean aclDiffers(List<ACL> aclList1, List<ACL> aclList2) {
+    for (ACL acl : aclList1) {
+      for (ACL newAcl : aclList2) {
+        if (acl.getId() != null && acl.getId().getId().equals(newAcl.getId().getId())
+          && acl.getPerms() != newAcl.getPerms()) {
+          LOG.info("ACL for '{}' differs: '{}' on znode, should be '{}'",
+            acl.getId().getId(), acl.getPerms(), newAcl.getPerms());
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  private void setRecursivelyOn(SolrZooKeeper solrZooKeeper, String node, List<ACL> acls)
+    throws KeeperException, InterruptedException {
+    solrZooKeeper.setACL(node, acls, -1);
+    for (String child : solrZooKeeper.getChildren(node, null)) {
+      String path = node.endsWith("/") ? node + child : node + "/" + child;
+      setRecursivelyOn(solrZooKeeper, path, acls);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
new file mode 100644
index 0000000..752a1e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
+
+public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(CreateCollectionHandler.class);
+
+  private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
+  private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+
+  private List<String> allCollectionList;
+
+  public CreateCollectionHandler(List<String> allCollectionList) {
+    this.allCollectionList = allCollectionList;
+  }
+
+  @Override
+  public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+    boolean result;
+    if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
+      result = createCollection(solrClient, solrPropsConfig, this.allCollectionList);
+    } else {
+      result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList);
+    }
+    return result;
+  }
+
+  private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
+    throws Exception {
+    LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
+      + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
+
+    // Default is true, because if the collection and shard is already there, then it will return true
+    boolean returnValue = true;
+
+    List<String> shardsList = new ArrayList<String>();
+    for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
+      shardsList.add("shard" + i);
+    }
+    String shardsListStr = StringUtils.join(shardsList, ',');
+
+    // Check if collection is already in zookeeper
+    if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
+      LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+      CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+      collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+      collectionCreateRequest.setRouterName("implicit");
+      collectionCreateRequest.setShards(shardsListStr);
+      collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+      collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+      collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+      collectionCreateRequest.setRouterField(ROUTER_FIELD);
+      collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+
+      CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+      if (createResponse.getStatus() != 0) {
+        returnValue = false;
+        LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
+          + ", shardsList=" + shardsList +", response=" + createResponse);
+      } else {
+        LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+      }
+    } else {
+      LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
+      Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
+      Collection<String> existingShards = getShards(slices, solrPropsConfig);
+      if (existingShards.size() < shardsList.size()) {
+        try {
+          updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
+        } catch (Throwable t) {
+          returnValue = false;
+          LOG.error(String.format("Exception during updating collection (%s)", t));
+        }
+      }
+      for (String shard : shardsList) {
+        if (!existingShards.contains(shard)) {
+          try {
+            LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
+            CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
+            createShardRequest.setCollectionName(solrPropsConfig.getCollection());
+            createShardRequest.setShardName(shard);
+            CollectionAdminResponse response = createShardRequest.process(solrClient);
+            if (response.getStatus() != 0) {
+              LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
+              returnValue = false;
+              break;
+            } else {
+              LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
+            }
+          } catch (Throwable t) {
+            LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
+            returnValue = false;
+            break;
+          }
+        }
+      }
+    }
+    return returnValue;
+  }
+
+  private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
+
+    if (allCollectionList.contains(solrPropsConfig.getCollection())) {
+      LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
+      return true;
+    }
+
+    LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+      ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+
+    CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+    collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+    collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+    collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+    collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+    collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
+    CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+    if (createResponse.getStatus() != 0) {
+      LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
+      return false;
+    } else {
+      LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+        ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+      return true;
+    }
+  }
+
+  private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
+    String baseUrl = getRandomBaseUrl(slices);
+    if (baseUrl != null) {
+      CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
+      HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
+        solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
+      HttpResponse response = httpClient.execute(request);
+      if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
+        throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
+      }
+    } else {
+      throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
+    }
+  }
+
+  private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
+    ZkStateReader reader = solrClient.getZkStateReader();
+    return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+  }
+
+  private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
+    Collection<String> list = new HashSet<>();
+    for (Slice slice : slices) {
+      for (Replica replica : slice.getReplicas()) {
+        LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
+          ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
+        list.add(slice.getName());
+      }
+    }
+    return list;
+  }
+
+  private String getRandomBaseUrl(Collection<Slice> slices) {
+    String coreUrl = null;
+    if (slices != null) {
+      for (Slice slice : slices) {
+        if (!slice.getReplicas().isEmpty()) {
+          Replica replica = slice.getReplicas().iterator().next();
+          coreUrl = replica.getStr("base_url");
+          if (coreUrl != null) {
+            break;
+          }
+        }
+      }
+    }
+    return coreUrl;
+  }
+
+  private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
+    return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
new file mode 100644
index 0000000..124ce40
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ListCollectionHandler implements SolrZkRequestHandler<List<String>> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ListCollectionHandler.class);
+
+  @Override
+  public List<String> handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+    try {
+      CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
+      CollectionAdminResponse response = colListReq.process(solrClient);
+      if (response.getStatus() != 0) {
+        LOG.error("Error getting collection list from solr.  response=" + response);
+        return null;
+      }
+      return (List<String>) response.getResponse().get("collections");
+    } catch (SolrException e) {
+      LOG.error("getCollections() operation failed", e);
+      return new ArrayList<>();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
new file mode 100644
index 0000000..52f3366
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReloadCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ReloadCollectionHandler.class);
+
+  @Override
+  public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+    boolean result = false;
+    try {
+      LOG.info("Reload collection - '{}'", solrPropsConfig.getCollection());
+      CollectionAdminRequest.Reload reloadCollectionRequest = new CollectionAdminRequest.Reload();
+      reloadCollectionRequest.setCollectionName(solrPropsConfig.getCollection());
+      reloadCollectionRequest.process(solrClient);
+      result = true;
+    } catch (Exception e) {
+      LOG.error(String.format("Reload collection ('%s') failed.", solrPropsConfig.getCollection()), e);
+    }
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
new file mode 100644
index 0000000..85ae6cb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+
+interface SolrZkRequestHandler<T> {
+  T handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
new file mode 100644
index 0000000..23defea
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class UploadConfigurationHandler implements SolrZkRequestHandler<Boolean> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class);
+
+  private static final String SCHEMA_FILE = "managed-schema";
+  private static final String SOLR_CONFIG_FILE = "solrconfig.xml";
+
+  private File configSetFolder;
+
+  public UploadConfigurationHandler(File configSetFolder) {
+    this.configSetFolder = configSetFolder;
+  }
+
+  @Override
+  public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+    boolean reloadCollectionNeeded = false;
+    String separator = FileSystems.getDefault().getSeparator();
+    String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator,
+      UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName());
+    solrClient.connect();
+    SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+    File tmpDir = new File(downloadFolderLocation);
+    try {
+      ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
+      boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
+      if (configExists) {
+        LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection());
+        if (!tmpDir.mkdirs()) {
+          LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath());
+        }
+        zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation));
+        File[] listOfFiles = configSetFolder.listFiles();
+        if (listOfFiles != null) {
+          for (File file : listOfFiles) {
+            if (file.getName().equals(SOLR_CONFIG_FILE) || file.getName().equals(SCHEMA_FILE)) { // TODO: try to find an another solution to reload schema
+              if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))){
+                LOG.info("One of the local solr config file differs ('{}'), upload config set to zookeeper", file.getName());
+                zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+                reloadCollectionNeeded = true;
+                break;
+              }
+            }
+          }
+        }
+      } else {
+        LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection());
+        File[] listOfFiles = configSetFolder.listFiles();
+        if (listOfFiles != null) {
+          zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+        }
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
+        solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
+    } finally {
+      if (tmpDir.exists()) {
+        try {
+          FileUtils.deleteDirectory(tmpDir);
+        } catch (IOException e){
+          LOG.error("Cannot delete temp directory.", e);
+        }
+      }
+    }
+    return reloadCollectionNeeded;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
new file mode 100644
index 0000000..8cc6b94
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.rest;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.springframework.context.annotation.Scope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.AUDIT_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.SERVICE_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.USER_CONFIG_STATUS_OD;
+
+@Api(value = "status", description = "Status Operations")
+@Path("status")
+@Named
+@Scope("request")
+public class StatusResource {
+
+  @Inject
+  @Named("solrServiceLogsState")
+  private SolrCollectionState solrServiceLogsState;
+
+  @Inject
+  @Named("solrAuditLogsState")
+  private SolrCollectionState solrAuditLogsState;
+
+  @Inject
+  @Named("solrUserConfigState")
+  private SolrCollectionState solrUserConfigState;
+
+  @GET
+  @Produces({"application/json"})
+  @ApiOperation(STATUS_OD)
+  public Map<String, SolrCollectionState> getStatus() {
+    Map<String, SolrCollectionState> response = new HashMap<>();
+    response.put("serviceLogs", solrServiceLogsState);
+    response.put("auditLogs", solrAuditLogsState);
+    response.put("userConfig", solrUserConfigState);
+    return response;
+  }
+
+  @GET
+  @Path("/servicelogs")
+  @Produces({"application/json"})
+  @ApiOperation(SERVICE_LOGS_STATUS_OD)
+  public SolrCollectionState getServiceLogStatus() {
+    return solrServiceLogsState;
+  }
+
+  @GET
+  @Path("/auditlogs")
+  @Produces({"application/json"})
+  @ApiOperation(AUDIT_LOGS_STATUS_OD)
+  public SolrCollectionState getSolrAuditLogsStatus() {
+    return solrAuditLogsState;
+  }
+
+  @GET
+  @Path("/userconfig")
+  @Produces({"application/json"})
+  @ApiOperation(USER_CONFIG_STATUS_OD)
+  public SolrCollectionState getSolrUserConfigStatus() {
+    return solrUserConfigState;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
index 532428b..b1a53f2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
@@ -41,14 +41,18 @@ public class RESTErrorUtil {
     return createRESTException(response, HttpServletResponse.SC_BAD_REQUEST);
   }
 
-  public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
-    List<MessageData> messageList = new ArrayList<MessageData>();
+  public static VResponse createMessageResponse(String errorMessage, MessageEnums messageEnum) {
+    List<MessageData> messageList = new ArrayList<>();
     messageList.add(messageEnum.getMessage());
-
     VResponse response = new VResponse();
     response.setStatusCode(VResponse.STATUS_ERROR);
     response.setMsgDesc(errorMessage);
     response.setMessageList(messageList);
+    return response;
+  }
+
+  public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
+    VResponse response = createMessageResponse(errorMessage, messageEnum);
     WebApplicationException webAppEx = createRESTException(response);
     logger.error("Operation error. response=" + response, webAppEx);
     return webAppEx;

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
new file mode 100644
index 0000000..4d529fb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+abstract public class AbstractLogsearchGlobalStateFilter implements Filter {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractLogsearchGlobalStateFilter.class);
+
+  protected static final String ZNODE_NOT_READY_MSG = "ZNode is not available for %s. (connection string: %s, endpoint: %s)";
+  protected static final String ZK_CONFIG_NOT_READY_MSG = "Collection configuration has not uploaded yet for %s. (configuration name: %s, collection name: %s, endpoint: %s)";
+  protected static final String SOLR_COLLECTION_NOT_READY_MSG = "Solr has not accessible yet for %s collection. (endpoint: %s)";
+
+  private SolrCollectionState solrCollectionState;
+  private RequestMatcher requestMatcher;
+  private SolrPropsConfig solrPropsConfig;
+
+  public AbstractLogsearchGlobalStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+    this.requestMatcher = requestMatcher;
+    this.solrCollectionState = state;
+    this.solrPropsConfig = solrPropsConfig;
+  }
+
+  @Override
+  public void init(FilterConfig filterConfig) throws ServletException {
+  }
+
+  @Override
+  public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+    HttpServletRequest request = (HttpServletRequest) servletRequest;
+    if (requestMatcher.matches(request)) {
+      VResponse errorResponse = getErrorResponse(solrCollectionState, solrPropsConfig, request);
+      if (errorResponse != null) {
+        LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
+        HttpServletResponse resp = (HttpServletResponse) servletResponse;
+        resp.setStatus(500);
+        resp.setContentType("application/json");
+        resp.getWriter().print(createStringFromErrorMessageObject(errorResponse));
+        return;
+      }
+    }
+    filterChain.doFilter(servletRequest, servletResponse);
+  }
+
+  @Override
+  public void destroy() {
+  }
+
+  /**
+   * Fill the error message with data in case of any condition based on collection state.
+   * If return value is null, that means there were no any errors
+   * @param solrCollectionState object to store solr state (e.g. : zookeeper/solr collection availability)
+   * @param solrPropsConfig object to store details for solr collection (e.g. : collection name, connection string)
+   */
+  public abstract VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request);
+
+  private String createStringFromErrorMessageObject(VResponse responseObject) {
+    try {
+      ObjectMapper mapper = new ObjectMapper();
+      return mapper.writeValueAsString(responseObject);
+    } catch (Exception e) {
+      throw RESTErrorUtil.createRESTException("Cannot parse response object on backend", MessageEnums.ERROR_CREATING_OBJECT);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
new file mode 100644
index 0000000..117fdd4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchAuditLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+  public LogsearchAuditLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+    super(requestMatcher, state, solrPropsConfig);
+  }
+
+  @Override
+  public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+    String requestUri = request.getRequestURI();
+    if (!solrCollectionState.isZnodeReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+        "audit logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+    } else if (!solrCollectionState.isConfigurationUploaded()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "audit logs",
+        solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+    } else if (!solrCollectionState.isSolrCollectionReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+        solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
new file mode 100644
index 0000000..74b30e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchServiceLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+  public LogsearchServiceLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+    super(requestMatcher, state, solrPropsConfig);
+  }
+
+  @Override
+  public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+    String requestUri = request.getRequestURI();
+    if (!solrCollectionState.isZnodeReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+        "service logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+    } else if (!solrCollectionState.isConfigurationUploaded()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "service logs",
+        solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+    } else if (!solrCollectionState.isSolrCollectionReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+        solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
new file mode 100644
index 0000000..037bed0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchUserConfigStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+
+  public LogsearchUserConfigStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+    super(requestMatcher, state, solrPropsConfig);
+  }
+
+  @Override
+  public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+    String requestUri = request.getRequestURI();
+    if (!solrCollectionState.isZnodeReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+        "userconfig", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+    } else if (!solrCollectionState.isConfigurationUploaded()) {
+      return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "userconfig",
+        solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+    } else if (!solrCollectionState.isSolrCollectionReady()) {
+      return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+        solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
index 85c8245..15322ef 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
@@ -411,8 +411,15 @@ define(['require',
             try {
                 if (!errorShown) {
                     errorShown = true;
+                    var errorMessage = "Some issues on server, Please try again later."
+                    if (error != null && error.responseText != null) {
+                      var errorObj = JSON.parse(error.responseText);
+                      if (errorObj.hasOwnProperty('msgDesc')) {
+                        errorMessage = errorObj.msgDesc;
+                      }
+                    }
                     Utils.notifyError({
-                        content: "Some issue on server, Please try again later."
+                      content: errorMessage
                     });
                     setTimeout(function() {
                         errorShown = false;

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
index fa6162a..6bb5a77 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
@@ -20,7 +20,6 @@ package org.apache.ambari.logsearch.solr;
 
 import org.apache.ambari.logsearch.solr.commands.CheckConfigZkCommand;
 import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand;
-import org.apache.ambari.logsearch.solr.commands.CreateSaslUsersZkCommand;
 import org.apache.ambari.logsearch.solr.commands.CreateShardCommand;
 import org.apache.ambari.logsearch.solr.commands.CreateSolrZnodeZkCommand;
 import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand;
@@ -34,7 +33,6 @@ import org.apache.ambari.logsearch.solr.commands.SetClusterPropertyZkCommand;
 import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand;
 import org.apache.ambari.logsearch.solr.commands.CheckZnodeZkCommand;
 import org.apache.ambari.logsearch.solr.util.ShardUtils;
-import org.apache.commons.lang.StringUtils;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
@@ -129,18 +127,6 @@ public class AmbariSolrCloudClient {
   }
 
   /**
-   * Add sasl user (only if not exist on the znode).
-   */
-  public String addSaslUsers() throws Exception {
-    LOG.info("Add sasl user to znode: {}", this.saslUsers);
-    String newUsers = new CreateSaslUsersZkCommand(getRetryTimes(), getInterval()).run(this);
-    if (StringUtils.isNotEmpty(newUsers)) {
-      LOG.info("New sasl users added to znode: {}", newUsers);
-    }
-    return newUsers;
-  }
-
-  /**
    * Create a znode only if it does not exist. Return 0 code if it exists.
    */
   public void createZnode() throws Exception {

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
deleted file mode 100644
index b144b41..0000000
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.solr.commands;
-
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.util.AclUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.SolrZooKeeper;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-public class CreateSaslUsersZkCommand extends AbstractZookeeperRetryCommand<String> {
-
-  public CreateSaslUsersZkCommand(int maxRetries, int interval) {
-    super(maxRetries, interval);
-  }
-
-  @Override
-  protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
-    List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat());
-    List<String> newUsers = new ArrayList<>();
-    Set<String> existingUsers = AclUtils.getUsersFromAclData(acls);
-    String saslUsers = client.getSaslUsers();
-    if (StringUtils.isNotEmpty(saslUsers)) {
-      String[] saslUserNames = saslUsers.split(",");
-      for (String saslUser : saslUserNames) {
-        if (!existingUsers.contains(saslUser)) {
-          acls.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", saslUser)));
-          newUsers.add(saslUser);
-        }
-      }
-    }
-    acls = AclUtils.updatePermissionForScheme(acls, "world", ZooDefs.Perms.READ);
-    solrZooKeeper.setACL(client.getZnode(), acls, -1);
-    return StringUtils.join(newUsers, ",");
-  }
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
index faa21d4..31ad5d3 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
@@ -23,6 +23,8 @@ import org.apache.ambari.logsearch.solr.util.AclUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
@@ -52,9 +54,13 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
 
     String configsPath = String.format("%s/%s", zNode, "configs");
     String collectionsPath = String.format("%s/%s", zNode, "collections");
-    List<String> exlustePaths = Arrays.asList(configsPath, collectionsPath);
+    String aliasesPath = String.format("%s/%s", zNode, "aliases.json"); // TODO: protect this later somehow
+    List<String> excludePaths = Arrays.asList(configsPath, collectionsPath, aliasesPath);
 
-    AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, exlustePaths);
+    createZnodeIfNeeded(configsPath, client.getSolrZkClient());
+    createZnodeIfNeeded(collectionsPath, client.getSolrZkClient());
+
+    AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, excludePaths);
 
     List<ACL> commonConfigAcls = new ArrayList<>();
     commonConfigAcls.addAll(saslUserList);
@@ -71,4 +77,11 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
 
     return true;
   }
+
+  private void createZnodeIfNeeded(String configsPath, SolrZkClient zkClient) throws KeeperException, InterruptedException {
+    if (!zkClient.exists(configsPath, true)) {
+      LOG.info("'{}' does not exist. Creating it ...", configsPath);
+      zkClient.makePath(configsPath, true);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
index 8e0132d..fc25c49 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
@@ -34,36 +34,6 @@ import java.util.Set;
 
 public class AclUtils {
 
-  public static Set<String> getUsersFromAclData(List<ACL> acls) {
-    Set<String> result = new HashSet<>();
-    if (!acls.isEmpty()) {
-      for (ACL acl : acls) {
-        String username = "";
-        String id = acl.getId().getId();
-        String[] splitted = id.split(":");
-        if (splitted.length > 1) {
-          username = splitted[0];
-        } else {
-          username = id;
-        }
-        result.add(username);
-      }
-    }
-    return result;
-  }
-
-  public static List<ACL> updatePermissionForScheme(List<ACL> acls, String scheme, int permission) {
-    List<ACL> aclResult = new ArrayList<>();
-    if (!acls.isEmpty()) {
-      for (ACL acl : acls) {
-        int permissionToAdd = scheme.equals(acl.getId().getScheme()) ? permission : acl.getPerms();
-        acl.setPerms(permissionToAdd);
-        aclResult.add(acl);
-      }
-    }
-    return aclResult;
-  }
-
   public static List<ACL> mergeAcls(List<ACL> originalAcls, List<ACL> updateAcls) {
     Map<String, ACL> aclMap = new HashMap<>();
     List<ACL> acls = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/bin/start.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh
index 81c9e66..4c60981 100644
--- a/ambari-logsearch/docker/bin/start.sh
+++ b/ambari-logsearch/docker/bin/start.sh
@@ -81,12 +81,6 @@ function start_solr() {
 }
 
 function start_logsearch() {
-  echo "Upload configuration sets ..."
-
-  $ZKCLI  -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/audit_logs/conf -confname audit_logs
-  $ZKCLI  -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/hadoop_logs/conf -confname hadoop_logs
-  $ZKCLI  -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/history/conf -confname history
-  
   $LOGSEARCH_SERVER_PATH/run.sh
   touch /var/log/ambari-logsearch-portal/logsearch-app.log
 }


Mime
View raw message