Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 85120200BC6 for ; Sat, 5 Nov 2016 17:29:09 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 83EF5160B17; Sat, 5 Nov 2016 16:29:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6B600160B1A for ; Sat, 5 Nov 2016 17:29:06 +0100 (CET) Received: (qmail 22856 invoked by uid 500); 5 Nov 2016 16:29:05 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 22385 invoked by uid 99); 5 Nov 2016 16:29:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 05 Nov 2016 16:29:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E19C5F170B; Sat, 5 Nov 2016 16:29:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Date: Sat, 05 Nov 2016 16:29:22 -0000 Message-Id: <1e6c463d1f5e4b42ba0ffb94a34be3ef@git.apache.org> In-Reply-To: <136c347d02b542e294ec4c78d773c5c6@git.apache.org> References: <136c347d02b542e294ec4c78d773c5c6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/58] [abbrv] ambari git commit: AMBARI-18310. Refactor logsearch portal code - part 3 (oleewere) archived-at: Sat, 05 Nov 2016 16:29:09 -0000 AMBARI-18310. Refactor logsearch portal code - part 3 (oleewere) Change-Id: Ibb5fc43360da84ae8d21f064d39f0f1ef7b82e1e Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6fa4abfe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6fa4abfe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6fa4abfe Branch: refs/heads/branch-2.5 Commit: 6fa4abfeadfcb3b942ab4f677c16a2031f14327b Parents: 199c6e3 Author: oleewere Authored: Sat Sep 10 21:15:02 2016 +0200 Committer: oleewere Committed: Sat Nov 5 14:46:44 2016 +0100 ---------------------------------------------------------------------- .../test-output/service-log-schema.json | 38 +- .../ambari/logsearch/common/ConfigHelper.java | 106 --- .../logsearch/common/ExternalServerClient.java | 7 +- .../logsearch/common/LogSearchConstants.java | 14 +- .../logsearch/common/PropertiesHelper.java | 44 - .../ambari/logsearch/common/PropertyMapper.java | 59 -- .../ambari/logsearch/conf/AuthConfig.java | 97 --- .../ambari/logsearch/conf/AuthPropsConfig.java | 97 +++ .../logsearch/conf/SolrAuditLogConfig.java | 181 ---- .../logsearch/conf/SolrAuditLogPropsConfig.java | 142 +++ .../ambari/logsearch/conf/SolrColumnConfig.java | 36 - .../ambari/logsearch/conf/SolrConfig.java | 88 +- .../logsearch/conf/SolrConnectionConfig.java | 49 -- .../conf/SolrConnectionPropsConfig.java | 49 ++ .../ambari/logsearch/conf/SolrPropsConfig.java | 50 ++ .../logsearch/conf/SolrServiceLogConfig.java | 144 --- .../conf/SolrServiceLogPropsConfig.java | 94 ++ .../ambari/logsearch/conf/SolrUserConfig.java | 104 --- .../logsearch/conf/SolrUserPropsConfig.java | 104 +++ .../ambari/logsearch/dao/AuditSolrDao.java | 59 +- .../logsearch/dao/ServiceLogsSolrDao.java | 47 +- .../ambari/logsearch/dao/SolrAliasDao.java | 123 +++ .../ambari/logsearch/dao/SolrCollectionDao.java | 259 ++++++ .../ambari/logsearch/dao/SolrDaoBase.java | 508 +---------- .../logsearch/dao/SolrSchemaFieldDao.java | 159 ++++ .../ambari/logsearch/dao/UserConfigSolrDao.java | 63 +- .../apache/ambari/logsearch/dao/UserDao.java | 10 +- .../ambari/logsearch/doc/DocConstants.java | 21 +- .../logsearch/graph/GraphDataGenerator.java | 2 +- .../logsearch/manager/AuditLogsManager.java | 162 +--- .../logsearch/manager/LogFileManager.java | 155 ---- .../ambari/logsearch/manager/PublicManager.java | 6 +- .../logsearch/manager/ServiceLogsManager.java | 122 +-- .../logsearch/manager/UserConfigManager.java | 1 - .../model/request/LogFileParamDefinition.java | 45 - .../request/LogFileTailParamDefinition.java | 32 - .../model/request/LogParamDefinition.java | 13 - .../request/ServiceLogFileParamDefinition.java | 38 - .../request/ServiceLogParamDefinition.java | 17 - .../model/request/impl/BaseLogRequest.java | 25 - .../request/impl/BaseServiceLogRequest.java | 39 - .../model/request/impl/LogFileRequest.java | 67 -- .../model/request/impl/LogFileTailRequest.java | 40 - .../model/request/impl/ServiceGraphRequest.java | 2 +- .../request/impl/ServiceLogExportRequest.java | 2 +- .../request/impl/ServiceLogFileRequest.java | 54 -- .../model/request/impl/ServiceLogRequest.java | 2 +- .../impl/ServiceLogTruncatedRequest.java | 2 +- .../ambari/logsearch/query/QueryGeneration.java | 96 +- .../logsearch/query/QueryGenerationBase.java | 28 - .../query/SearchCriteriaConstants.java | 8 - .../AbstractCommonAuditLogRequestConverter.java | 4 - ...bstractCommonServiceLogRequestConverter.java | 6 - .../converter/LogFileRequestConverter.java | 37 - .../converter/LogFileTailRequestConverter.java | 38 - .../converter/ServiceGraphRequestConverter.java | 2 - .../ServiceLogExportRequestConverter.java | 2 - .../ServiceLogFileRequestConverter.java | 36 - .../converter/ServiceLogRequestConverter.java | 2 - .../ServiceLogTruncatedRequestConverter.java | 2 - .../model/CommonServiceLogSearchCriteria.java | 8 - .../query/model/LogFileSearchCriteria.java | 52 -- .../query/model/LogFileTailSearchCriteria.java | 32 - .../logsearch/query/model/SearchCriteria.java | 9 + .../model/ServiceAnyGraphSearchCriteria.java | 2 +- .../query/model/ServiceGraphSearchCriteria.java | 2 +- .../model/ServiceLogExportSearchCriteria.java | 2 +- .../model/ServiceLogFileSearchCriteria.java | 40 - .../query/model/ServiceLogSearchCriteria.java | 2 +- .../ServiceLogTruncatedSearchCriteria.java | 2 +- .../logsearch/rest/AuditLogsResource.java | 1 - .../ambari/logsearch/rest/LogFileResource.java | 69 -- .../logsearch/rest/ServiceLogsResource.java | 28 +- ...rchExternalServerAuthenticationProvider.java | 8 +- .../LogsearchFileAuthenticationProvider.java | 6 +- .../LogsearchLdapAuthenticationProvider.java | 8 +- .../LogsearchSimpleAuthenticationProvider.java | 6 +- .../src/main/resources/default.properties | 21 - .../src/main/webapp/scripts/App.js | 4 +- .../collection_bases/VAuditLogListBase.js | 2 +- .../scripts/collection_bases/VLogListBase.js | 19 +- .../scripts/collections/BaseCollection.js | 1 - .../scripts/collections/SchemaFieldList.js | 29 + .../src/main/webapp/scripts/mgrs/SessionMgr.js | 137 --- .../src/main/webapp/scripts/utils/Globals.js | 158 ++-- .../src/main/webapp/scripts/utils/ViewUtils.js | 423 ++++----- .../scripts/views/audit/AuditTabLayoutView.js | 858 +++++++++--------- .../scripts/views/common/CustomBackgrid.js | 1 - .../main/webapp/scripts/views/common/Header.js | 6 +- .../webapp/scripts/views/common/ProfileBar.js | 3 +- .../dashboard/BubbleGraphTableLayoutView.js | 164 +--- .../views/dashboard/ComponentListView.js | 4 +- .../scripts/views/dashboard/DashboardView.js | 4 - .../scripts/views/dashboard/HostListView.js | 4 +- .../scripts/views/dashboard/MainLayoutView.js | 15 +- .../scripts/views/dialog/DetailLogFileView.js | 4 +- .../views/filter/CreateLogfeederFilterView.js | 6 +- .../webapp/scripts/views/tabs/ComparisonView.js | 15 +- .../views/tabs/HierarchyTabLayoutView.js | 871 +++++++++---------- .../webapp/scripts/views/tabs/LogFileView.js | 73 +- .../main/webapp/scripts/views/tabs/TreeView.js | 6 +- .../scripts/views/tabs/VisualSearchView.js | 437 ++++------ .../src/main/webapp/static/schema_fields.json | 71 ++ .../dashboard/MainLayoutView_tmpl.html | 121 --- .../main/webapp/templates/helpers/Helpers.js | 39 - .../ambari/logsearch/dao/AuditSolrDaoTest.java | 99 --- .../logsearch/dao/ServiceLogsSolrDaoTest.java | 95 -- .../ambari/logsearch/dao/SolrDaoBaseTest.java | 306 ------- .../logsearch/dao/UserConfigSolrDaoTest.java | 164 ---- .../test-config/logsearch/logsearch.properties | 2 - 110 files changed, 2946 insertions(+), 5332 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json index 9efa58b..2933123 100644 --- a/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json +++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json @@ -1,18 +1,24 @@ { - "log_message": "Log Message", - "bundle_id": "Bundle Id", - "case_id": "Case Id", - "cluster": "Cluster", - "event_count": "Event Count", - "file": "File", - "host": "Host", - "id": "Id", - "ip": "IP", - "level": "Level", - "logfile_line_number": "Logfile Line Number", - "logger_name": "Logger Name", - "method": "method", - "path": "Path", - "text": "Text", - "type": "Type" + "logfile_line_number": "tint", + "logtime": "tdate", + "text": "text_std_token_lower_case", + "host": "key_lower_case", + "seq_num": "tlong", + "logtype": "key_lower_case", + "rowtype": "key_lower_case", + "log_message": "text_std_token_lower_case", + "line_number": "tint", + "type": "key_lower_case", + "ip": "string", + "id": "string", + "thread_name": "key_lower_case", + "level": "key_lower_case", + "file": "key_lower_case", + "bundle_id": "key_lower_case", + "path": "key_lower_case", + "cluster": "key_lower_case", + "case_id": "key_lower_case", + "logger_name": "key_lower_case", + "method": "key_lower_case", + "event_count": "tlong" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ConfigHelper.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ConfigHelper.java deleted file mode 100644 index a3a71ea..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ConfigHelper.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ambari.logsearch.common; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.ambari.logsearch.dao.SolrDaoBase; -import org.apache.ambari.logsearch.manager.ManagerBase; -import org.apache.commons.lang.ArrayUtils; -import org.apache.log4j.Logger; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONObject; - -public class ConfigHelper { - private static final Logger logger = Logger.getLogger(ManagerBase.class); - - private ConfigHelper() { - throw new UnsupportedOperationException(); - } - - public static void extractSchemaFieldsName(String responseString, HashMap schemaFieldsNameMap, - HashMap schemaFieldTypeMap) { - try { - JSONObject jsonObject = new JSONObject(responseString); - JSONObject schemajsonObject = jsonObject.getJSONObject("schema"); - JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields"); - JSONArray fieldTypeJsonArray = schemajsonObject - .getJSONArray("fieldTypes"); - if (jsonArrayList == null) { - return; - } - if (fieldTypeJsonArray == null) { - return; - } - HashMap _schemaFieldTypeMap = new HashMap(); - HashMap _schemaFieldsNameMap = new HashMap(); - for (int i = 0; i < fieldTypeJsonArray.length(); i++) { - JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i); - String name = typeObject.getString("name"); - String fieldTypeJson = typeObject.toString(); - _schemaFieldTypeMap.put(name, fieldTypeJson); - } - - for (int i = 0; i < jsonArrayList.length(); i++) { - JSONObject explrObject = jsonArrayList.getJSONObject(i); - String name = explrObject.getString("name"); - String type = explrObject.getString("type"); - if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && - !name.contains(LogSearchConstants.NGRAM_SUFFIX) && !name.contains("tags") && !name.contains("_str")) { - _schemaFieldsNameMap.put(name, type); - } - } - schemaFieldsNameMap.clear(); - schemaFieldTypeMap.clear(); - schemaFieldsNameMap.putAll(_schemaFieldsNameMap); - schemaFieldTypeMap.putAll(_schemaFieldTypeMap); - } catch (Exception e) { - logger.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM); - } - } - - @SuppressWarnings("rawtypes") - public static void getSchemaFieldsName(String excludeArray[], List fieldNames, SolrDaoBase solrDaoBase) { - if (!solrDaoBase.schemaFieldsNameMap.isEmpty()) { - Iterator iteratorSechmaFieldsName = solrDaoBase.schemaFieldsNameMap.entrySet().iterator(); - while (iteratorSechmaFieldsName.hasNext()) { - Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName.next(); - String field = "" + fieldName.getKey(); - if (!isExclude(field, excludeArray)) { - fieldNames.add(field); - } - } - } - } - - private static boolean isExclude(String name, String excludeArray[]) { - if (!ArrayUtils.isEmpty(excludeArray)) { - for (String exclude : excludeArray) { - if (name.equals(exclude)){ - return true; - } - } - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java index 2590dfe..9682a3d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java @@ -27,13 +27,12 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; -import org.apache.ambari.logsearch.conf.AuthConfig; +import org.apache.ambari.logsearch.conf.AuthPropsConfig; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.glassfish.jersey.client.JerseyClient; import org.glassfish.jersey.client.JerseyClientBuilder; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; -import org.glassfish.jersey.filter.LoggingFilter; import org.springframework.stereotype.Component; /** @@ -51,7 +50,7 @@ public class ExternalServerClient { }; @Inject - private AuthConfig authConfig; + private AuthPropsConfig authPropsConfig; /** * Send GET request to an external server @@ -59,7 +58,7 @@ public class ExternalServerClient { public Object sendGETRequest(String url, Class klass, MultivaluedMap queryParam, String username, String password) throws Exception { - url = authConfig.getExternalAuthHostUrl() + url; + url = authPropsConfig.getExternalAuthHostUrl() + url; JerseyClient client = localJerseyClient.get(); HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder() .credentials(username, password) http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index 36ecb81..d9d3b86 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -73,7 +73,6 @@ public class LogSearchConstants { //operation public static final String EXCLUDE_QUERY = "excludeQuery"; public static final String INCLUDE_QUERY = "includeQuery"; - public static final String COLUMN_QUERY = "columnQuery"; // Seprator's public static final String I_E_SEPRATOR = "\\|i\\:\\:e\\|"; @@ -139,13 +138,8 @@ public class LogSearchConstants { public static final String REQUEST_PARAM_FIELD = "field"; public static final String REQUEST_PARAM_FORMAT = "format"; public static final String REQUEST_PARAM_LAST_PAGE = "lastPage"; - public static final String REQUEST_PARAM_LOG_TYPE = "logType"; - public static final String REQUEST_PARAM_COMPONENT = "component"; - public static final String REQUEST_PARAM_HOST = "host"; - public static final String REQUEST_PARAM_TAIL_SIZE = "tailSize"; - public static final String REQUEST_PARAM_COLUMN_QUERY = "columnQuery"; public static final String REQUEST_PARAM_I_MESSAGE = "iMessage"; - public static final String REQUEST_PARAM_G_E_MESSAGE = "gEMessage"; + public static final String REQUEST_PARAM_E_MESSAGE = "eMessage"; public static final String REQUEST_PARAM_MUST_BE = "mustBe"; public static final String REQUEST_PARAM_MUST_NOT = "mustNot"; public static final String REQUEST_PARAM_INCLUDE_QUERY = "includeQuery"; @@ -153,13 +147,7 @@ public class LogSearchConstants { public static final String REQUEST_PARAM_ID = "id"; public static final String REQUEST_PARAM_SCROLL_TYPE = "scrollType"; public static final String REQUEST_PARAM_NUMBER_ROWS = "numberRows"; - public static final String REQUEST_PARAM_HOST_LOG_FILE = "host"; - public static final String REQUEST_PARAM_COMPONENT_LOG_FILE = "component"; public static final String REQUEST_PARAM_LEVEL = "level"; - public static final String REQUEST_PARAM_ADVANCED_SEARCH = "advancedSearch"; - public static final String REQUEST_PARAM_TREE_PARAMS = "treeParams"; - public static final String REQUEST_PARAM_E_MESSAGE = "eMessage"; - public static final String REQUEST_PARAM_G_MUST_NOT = "gMustNot"; public static final String REQUEST_PARAM_HOST_NAME = "host_name"; public static final String REQUEST_PARAM_COMPONENT_NAME = "component_name"; public static final String REQUEST_PARAM_FILE_NAME = "file_name"; http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java index adb0699..1844d8f 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertiesHelper.java @@ -124,50 +124,6 @@ public class PropertiesHelper extends PropertyPlaceholderConfigurer { } } - public static Integer getIntProperty(String key, int defaultValue) { - if (key == null) { - return null; - } - String rtrnVal = propertiesMap.get(key); - if (rtrnVal == null) { - return defaultValue; - } - return Integer.valueOf(rtrnVal); - } - - public static Integer getIntProperty(String key) { - if (key == null) { - return null; - } - String rtrnVal = propertiesMap.get(key); - if (rtrnVal == null) { - return null; - } - return Integer.valueOf(rtrnVal); - } - - public static Long getLongProperty(String key, long defaultValue) { - if (key == null) { - return null; - } - String rtrnVal = propertiesMap.get(key); - if (rtrnVal == null) { - return defaultValue; - } - return Long.valueOf(rtrnVal); - } - - public static Long getLongProperty(String key) { - if (key == null) { - return null; - } - String rtrnVal = propertiesMap.get(key); - if (rtrnVal == null) { - return null; - } - return Long.valueOf(rtrnVal); - } - public static boolean getBooleanProperty(String key, boolean defaultValue) { if (key == null) { return defaultValue; http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertyMapper.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertyMapper.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertyMapper.java deleted file mode 100644 index 90e2114..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/PropertyMapper.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.common; - -import com.google.common.base.Splitter; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Component("propertyMapper") -public class PropertyMapper { - - public Map map(String property) { - return this.map(property, ","); - } - - public List list(String property) { - return this.list(property, ","); - } - - public Map solrUiMap(String property) { return this.solrUiMap(property, ","); } - - private List list(String property, String splitter) { - return Splitter.on(splitter).omitEmptyStrings().trimResults().splitToList(property); - } - - private Map map(String property, String splitter) { - return Splitter.on(splitter).omitEmptyStrings().trimResults().withKeyValueSeparator(":").split(property); - } - - private Map solrUiMap(String property, String splitter) { - Map result = new HashMap<>(); - Map map = this.map(property, splitter); - for (Map.Entry propEntry : map.entrySet()) { - result.put(propEntry.getKey() + LogSearchConstants.SOLR_SUFFIX, propEntry.getValue()); - result.put(propEntry.getValue() + LogSearchConstants.UI_SUFFIX, propEntry.getKey()); - } - return result; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java deleted file mode 100644 index 46d07bd..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class AuthConfig { - - @Value("${logsearch.auth.file.enable:true}") - boolean authFileEnabled; - @Value("${logsearch.auth.ldap.enable:false}") - boolean authLdapEnabled; - @Value("${logsearch.auth.simple.enable:false}") - boolean authSimpleEnabled; - @Value("${logsearch.auth.external_auth.enable:false}") - boolean authExternalEnabled; - @Value("${logsearch.auth.external_auth.host_url:'http://ip:port'}") - private String externalAuthHostUrl; - @Value("${logsearch.auth.external_auth.login_url:/api/v1/users/$USERNAME/privileges?fields=*}") - private String externalAuthLoginUrl; - @Value("${logsearch.login.credentials.file:user_pass.json}") - private String credentialsFile; - - public boolean isAuthFileEnabled() { - return authFileEnabled; - } - - public void setAuthFileEnabled(boolean authFileEnabled) { - this.authFileEnabled = authFileEnabled; - } - - public boolean isAuthLdapEnabled() { - return authLdapEnabled; - } - - public void setAuthLdapEnabled(boolean authLdapEnabled) { - this.authLdapEnabled = authLdapEnabled; - } - - public boolean isAuthSimpleEnabled() { - return authSimpleEnabled; - } - - public void setAuthSimpleEnabled(boolean authSimpleEnabled) { - this.authSimpleEnabled = authSimpleEnabled; - } - - public String getCredentialsFile() { - return credentialsFile; - } - - public void setCredentialsFile(String credentialsFile) { - this.credentialsFile = credentialsFile; - } - - public String getExternalAuthHostUrl() { - return externalAuthHostUrl; - } - - public void setExternalAuthHostUrl(String externalAuthHostUrl) { - this.externalAuthHostUrl = externalAuthHostUrl; - } - - public String getExternalAuthLoginUrl() { - return externalAuthLoginUrl; - } - - public void setExternalAuthLoginUrl(String externalAuthLoginUrl) { - this.externalAuthLoginUrl = externalAuthLoginUrl; - } - - public boolean isAuthExternalEnabled() { - return authExternalEnabled; - } - - public void setAuthExternalEnabled(boolean authExternalEnabled) { - this.authExternalEnabled = authExternalEnabled; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java new file mode 100644 index 0000000..11b1cda --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class AuthPropsConfig { + + @Value("${logsearch.auth.file.enable:true}") + boolean authFileEnabled; + @Value("${logsearch.auth.ldap.enable:false}") + boolean authLdapEnabled; + @Value("${logsearch.auth.simple.enable:false}") + boolean authSimpleEnabled; + @Value("${logsearch.auth.external_auth.enable:false}") + boolean authExternalEnabled; + @Value("${logsearch.auth.external_auth.host_url:'http://ip:port'}") + private String externalAuthHostUrl; + @Value("${logsearch.auth.external_auth.login_url:/api/v1/users/$USERNAME/privileges?fields=*}") + private String externalAuthLoginUrl; + @Value("${logsearch.login.credentials.file:user_pass.json}") + private String credentialsFile; + + public boolean isAuthFileEnabled() { + return authFileEnabled; + } + + public void setAuthFileEnabled(boolean authFileEnabled) { + this.authFileEnabled = authFileEnabled; + } + + public boolean isAuthLdapEnabled() { + return authLdapEnabled; + } + + public void setAuthLdapEnabled(boolean authLdapEnabled) { + this.authLdapEnabled = authLdapEnabled; + } + + public boolean isAuthSimpleEnabled() { + return authSimpleEnabled; + } + + public void setAuthSimpleEnabled(boolean authSimpleEnabled) { + this.authSimpleEnabled = authSimpleEnabled; + } + + public String getCredentialsFile() { + return credentialsFile; + } + + public void setCredentialsFile(String credentialsFile) { + this.credentialsFile = credentialsFile; + } + + public String getExternalAuthHostUrl() { + return externalAuthHostUrl; + } + + public void setExternalAuthHostUrl(String externalAuthHostUrl) { + this.externalAuthHostUrl = externalAuthHostUrl; + } + + public String getExternalAuthLoginUrl() { + return externalAuthLoginUrl; + } + + public void setExternalAuthLoginUrl(String externalAuthLoginUrl) { + this.externalAuthLoginUrl = externalAuthLoginUrl; + } + + public boolean isAuthExternalEnabled() { + return authExternalEnabled; + } + + public void setAuthExternalEnabled(boolean authExternalEnabled) { + this.authExternalEnabled = authExternalEnabled; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogConfig.java deleted file mode 100644 index ae4dca9..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogConfig.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import java.util.List; -import java.util.Map; - -@Configuration -public class SolrAuditLogConfig implements SolrConfig, SolrColumnConfig { - - @Value("${logsearch.solr.audit.logs.url:}") - private String solrUrl; - - @Value("${logsearch.solr.audit.logs.zk_connect_string:}") - private String zkConnectString; - - @Value("${logsearch.solr.collection.audit.logs:audit_logs}") - private String collection; - - @Value("${logsearch.ranger.audit.logs.collection.name:}") - private String rangerCollection; - - @Value("${logsearch.solr.audit.logs.config.name:audit_logs}") - private String configName; - - @Value("${logsearch.solr.audit.logs.alias.name:audit_logs_alias}") - private String aliasNameIn; - - @Value("${logsearch.audit.logs.split.interval.mins:none}") - private String splitInterval; - - @Value("${logsearch.collection.audit.logs.numshards:1}") - private Integer numberOfShards; - - @Value("${logsearch.collection.audit.logs.replication.factor:1}") - private Integer replicationFactor; - - @Value("#{propertyMapper.map('${logsearch.solr.audit.logs.column.mapping}')}") - private Map columnMapping; - - @Value("#{propertyMapper.list('${logsearch.solr.audit.logs.exclude.columnlist}')}") - private List excludeColumnList; - - @Value("#{propertyMapper.solrUiMap('${logsearch.solr.audit.logs.column.mapping}')}") - private Map solrAndUiColumns; - - @Override - public String getSolrUrl() { - return solrUrl; - } - - @Override - public void setSolrUrl(String solrUrl) { - this.solrUrl = solrUrl; - } - - @Override - public String getCollection() { - return collection; - } - - @Override - public void setCollection(String collection) { - this.collection = collection; - } - - @Override - public String getZkConnectString() { - return zkConnectString; - } - - @Override - public void setZkConnectString(String zkConnectString) { - this.zkConnectString = zkConnectString; - } - - @Override - public String getConfigName() { - return configName; - } - - @Override - public void setConfigName(String configName) { - this.configName = configName; - } - - @Override - public Integer getNumberOfShards() { - return numberOfShards; - } - - @Override - public void setNumberOfShards(Integer numberOfShards) { - this.numberOfShards = numberOfShards; - } - - @Override - public Integer getReplicationFactor() { - return replicationFactor; - } - - @Override - public void setReplicationFactor(Integer replicationFactor) { - this.replicationFactor = replicationFactor; - } - - @Override - public String getSplitInterval() { - return splitInterval; - } - - @Override - public void setSplitInterval(String splitInterval) { - this.splitInterval = splitInterval; - } - - @Override - public List getExcludeColumnList() { - return excludeColumnList; - } - - @Override - public void setExcludeColumnList(List excludeColumnList) { - this.excludeColumnList = excludeColumnList; - } - - @Override - public Map getColumnMapping() { - return columnMapping; - } - - @Override - public void setColumnMapping(Map columnMappings) { - this.columnMapping = columnMappings; - } - - @Override - public Map getSolrAndUiColumns() { - return solrAndUiColumns; - } - - @Override - public void setSolrAndUiColumns(Map solrAndUiColumns) { - this.solrAndUiColumns = solrAndUiColumns; - } - - public String getRangerCollection() { - return rangerCollection; - } - - public void setRangerCollection(String rangerCollection) { - this.rangerCollection = rangerCollection; - } - - public String getAliasNameIn() { - return aliasNameIn; - } - - public void setAliasNameIn(String aliasNameIn) { - this.aliasNameIn = aliasNameIn; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java new file mode 100644 index 0000000..d270b13 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrAuditLogPropsConfig.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import java.util.List; +import java.util.Map; + +@Configuration +public class SolrAuditLogPropsConfig implements SolrPropsConfig { + + @Value("${logsearch.solr.audit.logs.url:}") + private String solrUrl; + + @Value("${logsearch.solr.audit.logs.zk_connect_string:}") + private String zkConnectString; + + @Value("${logsearch.solr.collection.audit.logs:audit_logs}") + private String collection; + + @Value("${logsearch.ranger.audit.logs.collection.name:}") + private String rangerCollection; + + @Value("${logsearch.solr.audit.logs.config.name:audit_logs}") + private String configName; + + @Value("${logsearch.solr.audit.logs.alias.name:audit_logs_alias}") + private String aliasNameIn; + + @Value("${logsearch.audit.logs.split.interval.mins:none}") + private String splitInterval; + + @Value("${logsearch.collection.audit.logs.numshards:1}") + private Integer numberOfShards; + + @Value("${logsearch.collection.audit.logs.replication.factor:1}") + private Integer replicationFactor; + + @Override + public String getSolrUrl() { + return solrUrl; + } + + @Override + public void setSolrUrl(String solrUrl) { + this.solrUrl = solrUrl; + } + + @Override + public String getCollection() { + return collection; + } + + @Override + public void setCollection(String collection) { + this.collection = collection; + } + + @Override + public String getZkConnectString() { + return zkConnectString; + } + + @Override + public void setZkConnectString(String zkConnectString) { + this.zkConnectString = zkConnectString; + } + + @Override + public String getConfigName() { + return configName; + } + + @Override + public void setConfigName(String configName) { + this.configName = configName; + } + + @Override + public Integer getNumberOfShards() { + return numberOfShards; + } + + @Override + public void setNumberOfShards(Integer numberOfShards) { + this.numberOfShards = numberOfShards; + } + + @Override + public Integer getReplicationFactor() { + return replicationFactor; + } + + @Override + public void setReplicationFactor(Integer replicationFactor) { + this.replicationFactor = replicationFactor; + } + + @Override + public String getSplitInterval() { + return splitInterval; + } + + @Override + public void setSplitInterval(String splitInterval) { + this.splitInterval = splitInterval; + } + + public String getRangerCollection() { + return rangerCollection; + } + + public void setRangerCollection(String rangerCollection) { + this.rangerCollection = rangerCollection; + } + + public String getAliasNameIn() { + return aliasNameIn; + } + + public void setAliasNameIn(String aliasNameIn) { + this.aliasNameIn = aliasNameIn; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrColumnConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrColumnConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrColumnConfig.java deleted file mode 100644 index a12b2ce..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrColumnConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import java.util.List; -import java.util.Map; - -public interface SolrColumnConfig { - Map getColumnMapping(); - - void setColumnMapping(Map columnMapping); - - List getExcludeColumnList(); - - void setExcludeColumnList(List excludeColumnList); - - Map getSolrAndUiColumns(); - - void setSolrAndUiColumns(Map solrAndUiColumns); -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java index cfbe097..7508fb1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java @@ -18,33 +18,87 @@ */ package org.apache.ambari.logsearch.conf; -public interface SolrConfig { - String getSolrUrl(); +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.repository.config.EnableSolrRepositories; - void setSolrUrl(String solrUrl); +import javax.inject.Inject; - String getZkConnectString(); +@Configuration +@EnableSolrRepositories +public class SolrConfig { - void setZkConnectString(String zkConnectString); + private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class); - String getCollection(); + @Inject + private SolrServiceLogPropsConfig solrServiceLogPropsConfig; - void setCollection(String collection); + @Inject + private SolrAuditLogPropsConfig solrAuditLogPropsConfig; - String getConfigName(); + @Inject + private SolrUserPropsConfig solrUserConfigPropsConfig; - void setConfigName(String configName); + @Inject + private SolrKerberosConfig solrKerberosConfig; - Integer getNumberOfShards(); + @Bean(name = "serviceSolrTemplate") + public SolrTemplate serviceSolrTemplate() { + setupSecurity(); + return new SolrTemplate(createClient( + solrServiceLogPropsConfig.getSolrUrl(), + solrServiceLogPropsConfig.getZkConnectString(), + solrServiceLogPropsConfig.getCollection())); + } - void setNumberOfShards(Integer numberOfShards); + @Bean(name = "auditSolrTemplate") + @DependsOn("serviceSolrTemplate") + public SolrTemplate auditSolrTemplate() { + return new SolrTemplate(createClient( + solrAuditLogPropsConfig.getSolrUrl(), + solrAuditLogPropsConfig.getZkConnectString(), + solrAuditLogPropsConfig.getCollection())); + } - Integer getReplicationFactor(); + @Bean(name = "userConfigSolrTemplate") + @DependsOn("serviceSolrTemplate") + public SolrTemplate userConfigSolrTemplate() { + return new SolrTemplate(createClient( + solrUserConfigPropsConfig.getSolrUrl(), + solrUserConfigPropsConfig.getZkConnectString(), + solrUserConfigPropsConfig.getCollection())); + } - void setReplicationFactor(Integer replicationFactor); - - String getSplitInterval(); - - void setSplitInterval(String splitInterval); + private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { + if (StringUtils.isNotEmpty(zookeeperConnectString)) { + CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); + cloudSolrClient.setDefaultCollection(defaultCollection); + return cloudSolrClient; + } else if (StringUtils.isNotEmpty(solrUrl)) { + throw new UnsupportedOperationException("Currently only cloud mode is supported. Set zookeeper connect string."); + } + throw new IllegalStateException( + "Solr url or zookeeper connection string is missing. collection: " + defaultCollection); + } + private void setupSecurity() { + String jaasFile = solrKerberosConfig.getJaasFile(); + boolean securityEnabled = solrKerberosConfig.isEnabled(); + if (securityEnabled) { + System.setProperty("java.security.auth.login.config", jaasFile); + HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); + LOG.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile); + } + } } + http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionConfig.java deleted file mode 100644 index b3dceea..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import org.springframework.beans.factory.annotation.Value; - -public abstract class SolrConnectionConfig implements SolrConfig { - @Value("${logsearch.solr.url:}") - private String solrUrl; - - @Value("${logsearch.solr.zk_connect_string:}") - private String zkConnectString; - - @Override - public String getSolrUrl() { - return solrUrl; - } - - @Override - public void setSolrUrl(String solrUrl) { - this.solrUrl = solrUrl; - } - - @Override - public String getZkConnectString() { - return zkConnectString; - } - - @Override - public void setZkConnectString(String zkConnectString) { - this.zkConnectString = zkConnectString; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java new file mode 100644 index 0000000..82dd7df --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConnectionPropsConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.springframework.beans.factory.annotation.Value; + +public abstract class SolrConnectionPropsConfig implements SolrPropsConfig { + @Value("${logsearch.solr.url:}") + private String solrUrl; + + @Value("${logsearch.solr.zk_connect_string:}") + private String zkConnectString; + + @Override + public String getSolrUrl() { + return solrUrl; + } + + @Override + public void setSolrUrl(String solrUrl) { + this.solrUrl = solrUrl; + } + + @Override + public String getZkConnectString() { + return zkConnectString; + } + + @Override + public void setZkConnectString(String zkConnectString) { + this.zkConnectString = zkConnectString; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java new file mode 100644 index 0000000..78012b0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +public interface SolrPropsConfig { + String getSolrUrl(); + + void setSolrUrl(String solrUrl); + + String getZkConnectString(); + + void setZkConnectString(String zkConnectString); + + String getCollection(); + + void setCollection(String collection); + + String getConfigName(); + + void setConfigName(String configName); + + Integer getNumberOfShards(); + + void setNumberOfShards(Integer numberOfShards); + + Integer getReplicationFactor(); + + void setReplicationFactor(Integer replicationFactor); + + String getSplitInterval(); + + void setSplitInterval(String splitInterval); + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogConfig.java deleted file mode 100644 index e768402..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogConfig.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import java.util.List; -import java.util.Map; - -@Configuration -public class SolrServiceLogConfig extends SolrConnectionConfig implements SolrColumnConfig { - - @Value("${logsearch.solr.collection.service.logs:hadoop_logs}") - private String collection; - - @Value("${logsearch.service.logs.split.interval.mins:none}") - private String splitInterval; - - @Value("${logsearch.solr.service.logs.config.name:hadoop_logs}") - private String configName; - - @Value("${logsearch.collection.service.logs.numshards:1}") - private Integer numberOfShards; - - @Value("${logsearch.collection.service.logs.replication.factor:1}") - private Integer replicationFactor; - - @Value("#{propertyMapper.list('${logsearch.service.logs.fields}')}") - private List fields; - - @Value("#{propertyMapper.map('${logsearch.solr.audit.logs.column.mapping}')}") - private Map columnMapping; - - @Value("#{propertyMapper.list('${logsearch.solr.audit.logs.exclude.columnlist}')}") - private List excludeColumnList; - - @Value("#{propertyMapper.solrUiMap('${logsearch.solr.audit.logs.column.mapping}}')}") - private Map solrAndUiColumns; - - @Override - public String getCollection() { - return collection; - } - - @Override - public void setCollection(String collection) { - this.collection = collection; - } - - @Override - public String getSplitInterval() { - return splitInterval; - } - - @Override - public void setSplitInterval(String splitInterval) { - this.splitInterval = splitInterval; - } - - @Override - public String getConfigName() { - return configName; - } - - @Override - public void setConfigName(String configName) { - this.configName = configName; - } - - @Override - public Integer getNumberOfShards() { - return numberOfShards; - } - - @Override - public void setNumberOfShards(Integer numberOfShards) { - this.numberOfShards = numberOfShards; - } - - @Override - public Integer getReplicationFactor() { - return replicationFactor; - } - - @Override - public void setReplicationFactor(Integer replicationFactor) { - this.replicationFactor = replicationFactor; - } - - @Override - public Map getColumnMapping() { - return columnMapping; - } - - @Override - public void setColumnMapping(Map columnMapping) { - this.columnMapping = columnMapping; - } - - @Override - public List getExcludeColumnList() { - return excludeColumnList; - } - - @Override - public void setExcludeColumnList(List excludeColumnList) { - this.excludeColumnList = excludeColumnList; - } - - @Override - public Map getSolrAndUiColumns() { - return solrAndUiColumns; - } - - @Override - public void setSolrAndUiColumns(Map solrAndUiColumns) { - this.solrAndUiColumns = solrAndUiColumns; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogPropsConfig.java new file mode 100644 index 0000000..81935d4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrServiceLogPropsConfig.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import java.util.List; +import java.util.Map; + +@Configuration +public class SolrServiceLogPropsConfig extends SolrConnectionPropsConfig { + + @Value("${logsearch.solr.collection.service.logs:hadoop_logs}") + private String collection; + + @Value("${logsearch.service.logs.split.interval.mins:none}") + private String splitInterval; + + @Value("${logsearch.solr.service.logs.config.name:hadoop_logs}") + private String configName; + + @Value("${logsearch.collection.service.logs.numshards:1}") + private Integer numberOfShards; + + @Value("${logsearch.collection.service.logs.replication.factor:1}") + private Integer replicationFactor; + + @Override + public String getCollection() { + return collection; + } + + @Override + public void setCollection(String collection) { + this.collection = collection; + } + + @Override + public String getSplitInterval() { + return splitInterval; + } + + @Override + public void setSplitInterval(String splitInterval) { + this.splitInterval = splitInterval; + } + + @Override + public String getConfigName() { + return configName; + } + + @Override + public void setConfigName(String configName) { + this.configName = configName; + } + + @Override + public Integer getNumberOfShards() { + return numberOfShards; + } + + @Override + public void setNumberOfShards(Integer numberOfShards) { + this.numberOfShards = numberOfShards; + } + + @Override + public Integer getReplicationFactor() { + return replicationFactor; + } + + @Override + public void setReplicationFactor(Integer replicationFactor) { + this.replicationFactor = replicationFactor; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserConfig.java deleted file mode 100644 index 8ae630a..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserConfig.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.conf; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -@Configuration -public class SolrUserConfig extends SolrConnectionConfig { - - @Value("${logsearch.solr.collection.history:history}") - private String collection; - - @Value("${logsearch.history.split.interval.mins:none}") - private String splitInterval; - - @Value("${logsearch.solr.history.config.name:history}") - private String configName; - - @Value("${logsearch.collection.history.numshards:1}") - private Integer numberOfShards; - - @Value("${logsearch.collection.history.replication.factor:2}") - private Integer replicationFactor; - - @Value("#{'${logsearch.logfeeder.include.default.level:FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}'.split(',')}") - private List logLevels; - - @Override - public String getCollection() { - return collection; - } - - @Override - public void setCollection(String collection) { - this.collection = collection; - } - - @Override - public String getSplitInterval() { - return splitInterval; - } - - @Override - public void setSplitInterval(String splitInterval) { - this.splitInterval = splitInterval; - } - - @Override - public String getConfigName() { - return configName; - } - - @Override - public void setConfigName(String configName) { - this.configName = configName; - } - - @Override - public Integer getNumberOfShards() { - return numberOfShards; - } - - @Override - public void setNumberOfShards(Integer numberOfShards) { - this.numberOfShards = numberOfShards; - } - - @Override - public Integer getReplicationFactor() { - return replicationFactor; - } - - @Override - public void setReplicationFactor(Integer replicationFactor) { - this.replicationFactor = replicationFactor; - } - - public List getLogLevels() { - return logLevels; - } - - public void setLogLevels(List logLevels) { - this.logLevels = logLevels; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java new file mode 100644 index 0000000..38366d5 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +@Configuration +public class SolrUserPropsConfig extends SolrConnectionPropsConfig { + + @Value("${logsearch.solr.collection.history:history}") + private String collection; + + @Value("${logsearch.history.split.interval.mins:none}") + private String splitInterval; + + @Value("${logsearch.solr.history.config.name:history}") + private String configName; + + @Value("${logsearch.collection.history.numshards:1}") + private Integer numberOfShards; + + @Value("${logsearch.collection.history.replication.factor:2}") + private Integer replicationFactor; + + @Value("#{'${logsearch.logfeeder.include.default.level:FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}'.split(',')}") + private List logLevels; + + @Override + public String getCollection() { + return collection; + } + + @Override + public void setCollection(String collection) { + this.collection = collection; + } + + @Override + public String getSplitInterval() { + return splitInterval; + } + + @Override + public void setSplitInterval(String splitInterval) { + this.splitInterval = splitInterval; + } + + @Override + public String getConfigName() { + return configName; + } + + @Override + public void setConfigName(String configName) { + this.configName = configName; + } + + @Override + public Integer getNumberOfShards() { + return numberOfShards; + } + + @Override + public void setNumberOfShards(Integer numberOfShards) { + this.numberOfShards = numberOfShards; + } + + @Override + public Integer getReplicationFactor() { + return replicationFactor; + } + + @Override + public void setReplicationFactor(Integer replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public List getLogLevels() { + return logLevels; + } + + public void setLogLevels(List logLevels) { + this.logLevels = logLevels; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index 4f5e734..8d6a4da 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -19,58 +19,65 @@ package org.apache.ambari.logsearch.dao; -import java.util.Arrays; -import java.util.Collection; - import javax.annotation.PostConstruct; import javax.inject.Inject; -import org.apache.ambari.logsearch.common.PropertiesHelper; -import org.apache.ambari.logsearch.conf.SolrAuditLogConfig; +import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; import org.apache.ambari.logsearch.manager.ManagerBase.LogType; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.solr.core.SolrTemplate; import org.springframework.stereotype.Component; @Component public class AuditSolrDao extends SolrDaoBase { - private static final Logger logger = Logger.getLogger(AuditSolrDao.class); + private static final Logger LOG = Logger.getLogger(AuditSolrDao.class); + + @Inject + private SolrAuditLogPropsConfig solrAuditLogPropsConfig; + + @Inject + @Qualifier("auditSolrTemplate") + private SolrTemplate auditSolrTemplate; @Inject - private SolrAuditLogConfig solrAuditLogConfig; + private SolrAliasDao solrAliasDao; + + @Inject + private SolrCollectionDao solrCollectionDao; + + @Inject + private SolrSchemaFieldDao solrSchemaFieldDao; public AuditSolrDao() { super(LogType.AUDIT); } + @Override + public CloudSolrClient getSolrClient() { + return (CloudSolrClient) auditSolrTemplate.getSolrClient(); + } + @PostConstruct public void postConstructor() { - String solrUrl = solrAuditLogConfig.getSolrUrl(); - String zkConnectString = solrAuditLogConfig.getZkConnectString(); - String collection = solrAuditLogConfig.getCollection(); - String aliasNameIn = solrAuditLogConfig.getAliasNameIn(); - String rangerAuditCollection = solrAuditLogConfig.getRangerCollection(); - String splitInterval = solrAuditLogConfig.getSplitInterval(); - String configName = solrAuditLogConfig.getConfigName(); - int numberOfShards = solrAuditLogConfig.getNumberOfShards(); - int replicationFactor = solrAuditLogConfig.getReplicationFactor(); + String aliasNameIn = solrAuditLogPropsConfig.getAliasNameIn(); + String rangerAuditCollection = solrAuditLogPropsConfig.getRangerCollection(); try { - connectToSolr(solrUrl, zkConnectString, collection); - + solrCollectionDao.checkSolrStatus(getSolrClient()); boolean createAlias = (aliasNameIn != null && !StringUtils.isBlank(rangerAuditCollection)); - boolean needToPopulateSchemaField = !createAlias; - - setupCollections(splitInterval, configName, numberOfShards, replicationFactor, needToPopulateSchemaField); - + solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig); if (createAlias) { - Collection collectionsIn = Arrays.asList(collection, rangerAuditCollection.trim()); - setupAlias(aliasNameIn, collectionsIn); + solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig, this); + } else { + solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig, this); } } catch (Exception e) { - logger.error("Error while connecting to Solr for audit logs : solrUrl=" + solrUrl + ", zkConnectString=" + - zkConnectString + ", collection=" + collection, e); + LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" + + solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java index e338b7c..a6f5acf 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -22,41 +22,52 @@ package org.apache.ambari.logsearch.dao; import javax.annotation.PostConstruct; import javax.inject.Inject; -import org.apache.ambari.logsearch.common.PropertiesHelper; -import org.apache.ambari.logsearch.conf.SolrServiceLogConfig; +import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig; import org.apache.ambari.logsearch.manager.ManagerBase.LogType; import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.solr.core.SolrTemplate; import org.springframework.stereotype.Component; @Component public class ServiceLogsSolrDao extends SolrDaoBase { - private static final Logger logger = Logger.getLogger(ServiceLogsSolrDao.class); + private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class); @Inject - private SolrServiceLogConfig solrServiceLogConfig; - + private SolrCollectionDao solrCollectionDao; + + @Inject + private SolrServiceLogPropsConfig solrServiceLogPropsConfig; + + @Inject + @Qualifier("serviceSolrTemplate") + private SolrTemplate serviceSolrTemplate; + + @Inject + private SolrSchemaFieldDao solrSchemaFieldDao; + public ServiceLogsSolrDao() { super(LogType.SERVICE); } + @Override + public CloudSolrClient getSolrClient() { + return (CloudSolrClient) serviceSolrTemplate.getSolrClient(); + } + @PostConstruct public void postConstructor() { - logger.info("postConstructor() called."); - String solrUrl = solrServiceLogConfig.getSolrUrl(); - String zkConnectString = solrServiceLogConfig.getZkConnectString(); - String collection = solrServiceLogConfig.getCollection(); - String splitInterval = solrServiceLogConfig.getSplitInterval(); - String configName = solrServiceLogConfig.getConfigName(); - int numberOfShards = solrServiceLogConfig.getNumberOfShards(); - int replicationFactor = solrServiceLogConfig.getReplicationFactor(); - + LOG.info("postConstructor() called."); try { - connectToSolr(solrUrl, zkConnectString, collection); - setupCollections(splitInterval, configName, numberOfShards, replicationFactor, true); + solrCollectionDao.checkSolrStatus(getSolrClient()); + solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig); + solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig, this); } catch (Exception e) { - logger.error("error while connecting to Solr for service logs : solrUrl=" + solrUrl + ", zkConnectString=" + - zkConnectString + ", collection=" + collection, e); + LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() + + ", collection=" + solrServiceLogPropsConfig.getCollection(), e); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6fa4abfe/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java new file mode 100644 index 0000000..81471d3 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.dao; + +import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; +import org.apache.commons.lang.StringUtils; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +@Component +public class SolrAliasDao { + + private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class); + + private static final int ALIAS_SETUP_RETRY_SECOND = 30*60; + + @Inject + private SolrCollectionDao solrCollectionDao; + + protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient, + final SolrAuditLogPropsConfig solrPropsConfig, final SolrDaoBase solrDaoBase) throws Exception { + final Collection collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); + + if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) { + LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " + + collectionListIn.toString() + ", solrCloudClient=" + solrClient); + return; + } + + LOG.info("setupAlias " + solrPropsConfig.getAliasNameIn() + " for " + collectionListIn.toString()); + // Start a background thread to do setup + Thread setupThread = new Thread("setup_alias_" + solrPropsConfig.getAliasNameIn()) { + @Override + public void run() { + LOG.info("Started monitoring thread to check availability of Solr server. alias=" + solrPropsConfig.getAliasNameIn() + + ", collections=" + collectionListIn.toString()); + int retryCount = 0; + while (true) { + try { + int count = createAlias(solrClient, solrPropsConfig.getAliasNameIn(), collectionListIn); + if (count > 0) { + solrClient.setDefaultCollection(solrPropsConfig.getAliasNameIn()); + if (count == collectionListIn.size()) { + LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " + + "Collections=" + collectionListIn); + solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig, solrDaoBase); + break; + } + } else { + LOG.warn("Not able to create alias=" + solrPropsConfig.getAliasNameIn() + ", retryCount=" + retryCount); + } + } catch (Exception e) { + LOG.error("Error setting up alias=" + solrPropsConfig.getAliasNameIn(), e); + } + try { + Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000); + } catch (InterruptedException sleepInterrupted) { + LOG.info("Sleep interrupted while setting up alias " + solrPropsConfig.getAliasNameIn()); + break; + } + retryCount++; + } + } + }; + setupThread.setDaemon(true); + setupThread.start(); + } + + private int createAlias(final CloudSolrClient solrClient, String aliasNameIn, Collection collectionListIn) + throws SolrServerException, IOException { + List collectionToAdd = solrCollectionDao.getCollections(solrClient); + collectionToAdd.retainAll(collectionListIn); + + String collectionsCSV = null; + if (!collectionToAdd.isEmpty()) { + collectionsCSV = StringUtils.join(collectionToAdd, ','); + CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias(); + aliasCreateRequest.setAliasName(aliasNameIn); + aliasCreateRequest.setAliasedCollections(collectionsCSV); + CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClient); + if (createResponse.getStatus() != 0) { + LOG.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV + + ", response=" + createResponse); + return 0; + } + } + if ( collectionToAdd.size() == collectionListIn.size()) { + LOG.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); + } else { + LOG.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " + + "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV); + } + return collectionToAdd.size(); + } +}