ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject ambari git commit: AMBARI-17541. LogSearch should also show audit logs from Apache Ranger Solr (Bosco Durai via oleewere)
Date Mon, 04 Jul 2016 10:09:29 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 196657bf3 -> ea07c3414


AMBARI-17541. LogSearch should also show audit logs from Apache Ranger Solr (Bosco Durai via
oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ea07c341
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ea07c341
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ea07c341

Branch: refs/heads/trunk
Commit: ea07c34140d7394ec694bc71273a89c237e815de
Parents: 196657b
Author: oleewere <oleewere@gmail.com>
Authored: Mon Jul 4 12:06:56 2016 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Mon Jul 4 12:06:56 2016 +0200

----------------------------------------------------------------------
 .../ambari/logsearch/dao/AuditSolrDao.java      |  14 +++
 .../ambari/logsearch/dao/SolrDaoBase.java       | 103 ++++++++++++++++++-
 2 files changed, 116 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ea07c341/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
index f1789c1..98a3cfe 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java
@@ -19,6 +19,9 @@
 
 package org.apache.ambari.logsearch.dao;
 
+import java.util.ArrayList;
+import java.util.Collection;
+
 import javax.annotation.PostConstruct;
 
 import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
@@ -41,6 +44,11 @@ public class AuditSolrDao extends SolrDaoBase {
     String zkConnectString = PropertiesUtil.getProperty("logsearch.solr.audit.logs.zk_connect_string");
     String collection = PropertiesUtil.getProperty(
       "logsearch.solr.collection.audit.logs", "audit_logs");
+    String aliasNameIn = PropertiesUtil.getProperty(
+        "logsearch.solr.audit.logs.alias.name", "audit_logs_alias");
+    String rangerAuditCollection = PropertiesUtil.getProperty(
+        "logsearch.ranger.audit.logs.collection.name", "ranger_audits");
+    
     String splitInterval = PropertiesUtil.getProperty(
       "logsearch.audit.logs.split.interval.mins", "none");
     String configName = PropertiesUtil.getProperty(
@@ -54,6 +62,12 @@ public class AuditSolrDao extends SolrDaoBase {
       connectToSolr(solrUrl, zkConnectString, collection);
       setupCollections(splitInterval, configName, numberOfShards,
         replicationFactor);
+      if(aliasNameIn != null && rangerAuditCollection != null && rangerAuditCollection.trim().length()
>0) {
+        Collection<String> collectionsIn = new ArrayList<String>();
+        collectionsIn.add(collection);
+        collectionsIn.add(rangerAuditCollection.trim());
+        setupAlias(aliasNameIn, collectionsIn);
+      }
     } catch (Exception e) {
       logger.error(
         "Error while connecting to Solr for audit logs : solrUrl="

http://git-wip-us.apache.org/repos/asf/ambari/blob/ea07c341/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 3a1a1ca..d57daa6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -78,7 +78,9 @@ public abstract class SolrDaoBase {
   RESTErrorUtil restErrorUtil;
 
   String collectionName = null;
-  // List<String> collectionList = new ArrayList<String>();
+  
+  String aliasName = null;
+  Collection<String> aliasCollectionList = new ArrayList<String>();
 
   private SolrClient solrClient = null;
   CloudSolrClient solrClouldClient = null;
@@ -89,9 +91,11 @@ public abstract class SolrDaoBase {
   boolean isSolrInitialized = false;
 
   private boolean setup_status = false;
+
   private boolean populateFieldsThreadActive = false;
 
   int SETUP_RETRY_SECOND = 30;
+  int ALIAS_SETUP_RETRY_SECOND = 30*60; //30 minutes
   
   private boolean isZkConnectString=false;//by default its false
   
@@ -190,6 +194,103 @@ public abstract class SolrDaoBase {
     return status;
   }
 
+  public void setupAlias(final String aliasNameIn, final Collection<String> collectionListIn
) throws Exception {
+    if( aliasNameIn == null || collectionListIn== null || collectionListIn.size() == 0 ||
solrClouldClient == null) {
+      logger.info("Will not create alias " + aliasNameIn + " for "
+        + (collectionListIn==null?null: collectionListIn.toString()) + ", solrCloudClient="
+ solrClouldClient);
+      return;
+    }
+    logger.info("setupAlias " + aliasNameIn + " for " + (collectionListIn==null?null: collectionListIn.toString()));
+    aliasName = aliasNameIn;
+    aliasCollectionList = collectionListIn;
+
+    // Start a background thread to do setup
+    Thread setupThread = new Thread("setup_alias_" + aliasNameIn) {
+      @Override
+      public void run() {
+        logger.info("Started monitoring thread to check availability of Solr server. alias="
+            + aliasNameIn + ", collections=" + collectionListIn.toString());
+        int retryCount = 0;
+        while (true) {
+          try {
+            int count = createAlias(aliasNameIn,collectionListIn);
+            if (count > 0) {
+              solrClouldClient.setDefaultCollection(aliasNameIn);
+              if( count == collectionListIn.size()) {
+                logger.info("Setup for alias " + aliasNameIn
+                    + " is successful. Exiting setup retry thread. Collections=" + collectionListIn);
+                break;
+              }
+            } else {
+              logger.warn("Not able to create alias="
+                  + aliasNameIn + ", retryCount=" + retryCount);
+            }
+          } catch (Exception e) {
+            logger.error("Error setting up alias=" + aliasNameIn, e);
+          }
+          try {
+            Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000);
+          } catch (InterruptedException sleepInterrupted) {
+            logger.info("Sleep interrupted while setting up alias "
+                + aliasNameIn);
+            break;
+          }
+          retryCount++;
+        }
+      }
+    };
+    setupThread.setDaemon(true);
+    setupThread.start();     
+  }
+  
+  /**
+   * @param aliasNameIn
+   * @param collectionListIn
+   * @return
+   * @throws IOException 
+   * @throws SolrServerException 
+   */
+  protected int createAlias(String aliasNameIn,
+      Collection<String> collectionListIn) throws SolrServerException, IOException
{
+    List<String> collections = getCollections();
+    List<String> collectionToAdd = new ArrayList<String>();
+    for (String col : collections) {
+      if( collectionListIn.contains(col)) {
+        collectionToAdd.add(col);
+      }
+    }
+    String collectionsCSV = null;
+    if( collectionToAdd.size() > 0 ) {
+      for (String col : collectionToAdd) {
+        if(collectionsCSV == null) {
+          collectionsCSV = col;
+        } else {
+          collectionsCSV = collectionsCSV + ","  + col;
+        }
+      }
+      CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias();

+      aliasCreateRequest.setAliasName(aliasNameIn);
+      aliasCreateRequest.setAliasedCollections(collectionsCSV);
+      CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClouldClient);
+      if (createResponse.getStatus() != 0) {
+        logger.error("Error creating alias. alias="
+        + aliasNameIn + ", collectionList=" + collectionsCSV
+        + ", solrDetail=" + solrDetail + ", response="
+        + createResponse);
+      }
+    } else {
+      if( collectionToAdd.size() == collectionListIn.size()) {
+        logger.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV="
+            + collectionsCSV + ", solrDetail=" + solrDetail);        
+      } else {
+        logger.info("Created alias for " + collectionToAdd.size() + " out of " + 
+            + collectionListIn.size() + " collections. alias=" + aliasNameIn 
+            + ", collectionsCSV=" + collectionsCSV + ", solrDetail=" + solrDetail);
+      }
+    }
+    return collectionToAdd.size();
+  }
+
   public void setupCollections(final String splitMode, final String configName,
       final int numberOfShards, final int replicationFactor) throws Exception {
     if (isZkConnectString) {


Mime
View raw message