ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject ambari git commit: AMBARI-21277. ADDENDUM part 3: Fail to create solr clients in Log Search / Log Feeder (oleewere)
Date Mon, 26 Jun 2017 13:40:29 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 379b95b63 -> 4c3ecb92e


AMBARI-21277. ADDENDUM part 3: Fail to create solr clients in Log Search / Log Feeder (oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4c3ecb92
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4c3ecb92
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4c3ecb92

Branch: refs/heads/branch-2.5
Commit: 4c3ecb92ec317e04cf7405cc38d3fb5134092d2f
Parents: 379b95b
Author: oleewere <oleewere@gmail.com>
Authored: Mon Jun 26 15:33:55 2017 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Mon Jun 26 15:33:55 2017 +0200

----------------------------------------------------------------------
 .../ambari-logsearch-logfeeder/pom.xml          |  6 ---
 .../ambari/logfeeder/output/OutputSolr.java     | 10 ++---
 .../ambari-logsearch-portal/pom.xml             | 32 -------------
 .../logsearch/common/ExternalServerClient.java  |  4 +-
 .../configurer/SolrCollectionConfigurer.java    |  9 +---
 .../logsearch/dao/SolrSchemaFieldDao.java       | 47 +++++++-------------
 .../handler/CreateCollectionHandler.java        |  6 +--
 ambari-metrics/ambari-metrics-common/pom.xml    |  4 ++
 8 files changed, 27 insertions(+), 91 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index 1ad59b3..49122e8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -143,12 +143,6 @@
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-common</artifactId>
     <version>${hadoop.version}</version>
-     <exclusions>
-       <exclusion>
-         <groupId>commons-httpclient</groupId>
-         <artifactId>commons-httpclient</artifactId>
-       </exclusion>
-     </exclusions>
   </dependency>
   <dependency>
     <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index d37c388..d37a3bb 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -37,7 +37,6 @@ import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.ambari.logfeeder.util.DateUtil;
 import org.apache.ambari.logfeeder.util.LogFeederUtil;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.solr.client.solrj.SolrClient;
@@ -51,8 +50,6 @@ import org.apache.solr.client.solrj.response.SolrPingResponse;
 import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 
 public class OutputSolr extends Output {
   private static final Logger LOG = Logger.getLogger(OutputSolr.class);
@@ -94,6 +91,7 @@ public class OutputSolr extends Output {
   public void init() throws Exception {
     super.init();
     initParams();
+    setupSecurity();
     createOutgoingBuffer();
     createSolrWorkers();
   }
@@ -181,10 +179,8 @@ public class OutputSolr extends Output {
       throw new Exception("For solr cloud property collection is mandatory");
     }
     LOG.info("Using collection=" + collection);
-    setupSecurity();
-    SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
-    HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null));
-    CloudSolrClient solrClient = new CloudSolrClient(zkConnectString, httpClient);
+
+    CloudSolrClient solrClient = new CloudSolrClient(zkConnectString);
     solrClient.setDefaultCollection(collection);
     return solrClient;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index 46b4350..a439266 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -38,7 +38,6 @@
     <swagger.version>1.5.8</swagger.version>
     <spring-data-solr.version>2.0.2.RELEASE</spring-data-solr.version>
     <jjwt.version>0.6.0</jjwt.version>
-    <http-components.version>4.4.1</http-components.version>
   </properties>
   <profiles>
     <!-- Dev Profile Start -->
@@ -524,27 +523,6 @@
       <version>${spring.security.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-      <version>${http-components.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpcore</artifactId>
-      <version>${http-components.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpmime</artifactId>
-      <version>${http-components.version}</version>
-    </dependency>
-    <dependency>
       <groupId>org.glassfish.jersey.ext</groupId>
       <artifactId>jersey-spring3</artifactId>
       <version>2.23.2</version>
@@ -559,12 +537,6 @@
       <groupId>org.glassfish.jersey.connectors</groupId>
       <artifactId>jersey-apache-connector</artifactId>
       <version>${jersey.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.httpcomponents</groupId>
-          <artifactId>httpclient</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.glassfish.jersey.core</groupId>
@@ -656,10 +628,6 @@
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-server</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>commons-httpclient</groupId>
-          <artifactId>commons-httpclient</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
index 99844f5..df00c15 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
@@ -20,13 +20,13 @@ package org.apache.ambari.logsearch.common;
 
 import javax.inject.Inject;
 import javax.inject.Named;
-import javax.security.auth.login.CredentialException;
 import javax.ws.rs.client.Invocation;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response;
 
 import org.apache.ambari.logsearch.conf.AuthPropsConfig;
 import org.apache.ambari.logsearch.util.SSLUtil;
+import org.apache.commons.httpclient.auth.InvalidCredentialsException;
 import org.apache.log4j.Logger;
 import org.glassfish.jersey.client.JerseyClient;
 import org.glassfish.jersey.client.JerseyClientBuilder;
@@ -70,7 +70,7 @@ public class ExternalServerClient {
       Response response = invocationBuilder.get();
       if (response.getStatus() != Response.Status.OK.getStatusCode()
         && response.getStatus() != Response.Status.FOUND.getStatusCode()) {
-        throw new CredentialException(String.format("External auth failed with status code:
%d, response: %s",
+        throw new InvalidCredentialsException(String.format("External auth failed with status
code: %d, response: %s",
           response.getStatus(), response.readEntity(String.class)));
       }
       return response.readEntity(klass);

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
index aba059d..7259bf8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -28,14 +28,9 @@ import org.apache.ambari.logsearch.handler.ReloadCollectionHandler;
 import org.apache.ambari.logsearch.handler.UpgradeSchemaHandler;
 import org.apache.ambari.logsearch.handler.UploadConfigurationHandler;
 import org.apache.commons.lang.StringUtils;
-import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooKeeper;
@@ -128,9 +123,7 @@ public class SolrCollectionConfigurer implements SolrConfigurer {
 
   private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String
defaultCollection) {
     if (StringUtils.isNotEmpty(zookeeperConnectString)) {
-      DefaultHttpClient httpClient = new SystemDefaultHttpClient();
-      HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null));
-      CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString, httpClient);
+      CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString);
       cloudSolrClient.setDefaultCollection(defaultCollection);
       return cloudSolrClient;
     } else if (StringUtils.isNotEmpty(solrUrl)) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index 3124f35..01378c1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -22,11 +22,9 @@ import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.common.LogType;
 import org.apache.ambari.logsearch.common.MessageEnums;
 import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
-import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
-import org.apache.http.util.EntityUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -40,8 +38,6 @@ import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.common.util.NamedList;
 import org.codehaus.jettison.json.JSONObject;
@@ -70,18 +66,18 @@ public class SolrSchemaFieldDao {
 
   @Inject
   private AuditSolrDao auditSolrDao;
-
+  
   @Inject
   private SolrUserPropsConfig solrUserConfigPropsConfig;
-
+  
   private int retryCount;
   private int skipCount;
-
+  
   private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
   private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
   private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
   private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
-
+  
   @Scheduled(fixedDelay = RETRY_SECOND * 1000)
   public void populateAllSchemaFields() {
     if (skipCount > 0) {
@@ -111,7 +107,7 @@ public class SolrSchemaFieldDao {
         schemaRequest.setMethod(SolrRequest.METHOD.GET);
         schemaRequest.setPath("/schema");
         schemaResponse = schemaRequest.process(solrClient);
-
+        
         LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection()
+ ", luke=" + lukeResponses +
             ", schema= " + schemaResponse);
       } catch (SolrException | SolrServerException | IOException e) {
@@ -131,37 +127,26 @@ public class SolrSchemaFieldDao {
       }
     }
   }
-
+  
   private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2";
-
+  
   @SuppressWarnings("unchecked")
   private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
     ZkStateReader zkStateReader = solrClient.getZkStateReader();
     Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection());
-
+    
     List<LukeResponse> lukeResponses = new ArrayList<>();
     for (Slice slice : activeSlices) {
       for (Replica replica : slice.getReplicas()) {
-        HttpEntity httpEntity = null;
-        try(SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient()) {
-          HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams)
null));
+        try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
           HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
           HttpResponse response = httpClient.execute(request);
-          httpEntity = response.getEntity();
-          NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null,
null).unmarshal(httpEntity.getContent());
+          NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null,
null).unmarshal(response.getEntity().getContent());
           LukeResponse lukeResponse = new LukeResponse();
           lukeResponse.setResponse(lukeData);
           lukeResponses.add(lukeResponse);
         } catch (IOException e) {
           LOG.error("Exception during getting luke responses", e);
-        } finally {
-          if (httpEntity != null) {
-            try {
-              EntityUtils.consume(httpEntity);
-            } catch (IOException e) {
-              LOG.error("Error during consuming http entity...", e);
-            }
-          }
         }
       }
     }
@@ -173,7 +158,7 @@ public class SolrSchemaFieldDao {
     try {
       HashMap<String, String> _schemaFieldNameMap = new HashMap<>();
       HashMap<String, String> _schemaFieldTypeMap = new HashMap<>();
-
+      
       for (LukeResponse lukeResponse : lukeResponses) {
         for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
           String name = e.getKey();
@@ -184,7 +169,7 @@ public class SolrSchemaFieldDao {
           }
         }
       }
-
+      
       List<FieldTypeDefinition> fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes();
       for (FieldTypeDefinition fieldType : fieldTypes) {
         Map<String, Object> fieldAttributes = fieldType.getAttributes();
@@ -192,7 +177,7 @@ public class SolrSchemaFieldDao {
         String fieldTypeJson = new JSONObject(fieldAttributes).toString();
         _schemaFieldTypeMap.put(name, fieldTypeJson);
       }
-
+      
       List<Map<String, Object>> fields = schemaResponse.getSchemaRepresentation().getFields();
       for (Map<String, Object> field : fields) {
         String name = (String) field.get("name");
@@ -202,11 +187,11 @@ public class SolrSchemaFieldDao {
           _schemaFieldNameMap.put(name, type);
         }
       }
-
+      
       if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) {
         return;
       }
-
+      
       synchronized (this) {
         schemaFieldNameMap.clear();
         schemaFieldNameMap.putAll(_schemaFieldNameMap);

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index 9ffe16d..752a1e1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -23,7 +23,6 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -32,8 +31,6 @@ import org.apache.solr.client.solrj.response.CollectionAdminResponse;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -173,8 +170,7 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean>
{
   private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig
solrPropsConfig) throws IOException {
     String baseUrl = getRandomBaseUrl(slices);
     if (baseUrl != null) {
-      SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
-      HttpClientUtil.configureClient(httpClient, new ModifiableSolrParams((SolrParams) null));
+      CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
       HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
         solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
       HttpResponse response = httpClient.execute(request);

http://git-wip-us.apache.org/repos/asf/ambari/blob/4c3ecb92/ambari-metrics/ambari-metrics-common/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml
index 687b704..b52b1d9 100644
--- a/ambari-metrics/ambari-metrics-common/pom.xml
+++ b/ambari-metrics/ambari-metrics-common/pom.xml
@@ -108,6 +108,10 @@
                   <pattern>org.jboss</pattern>
                   <shadedPattern>org.apache.hadoop.metrics2.sink.relocated.jboss</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>org.apache.http</pattern>
+                  <shadedPattern>org.apache.hadoop.metrics2.sink.relocated.apache.http</shadedPattern>
+                </relocation>
               </relocations>
             </configuration>
           </execution>


Mime
View raw message