Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 68A11200BDF for ; Mon, 7 Nov 2016 21:51:33 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 6739E160AEC; Mon, 7 Nov 2016 20:51:33 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 974E5160B1E for ; Mon, 7 Nov 2016 21:51:30 +0100 (CET) Received: (qmail 59738 invoked by uid 500); 7 Nov 2016 20:51:28 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 56629 invoked by uid 99); 7 Nov 2016 20:51:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Nov 2016 20:51:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A9A99E02E4; Mon, 7 Nov 2016 20:51:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Mon, 07 Nov 2016 20:51:57 -0000 Message-Id: In-Reply-To: <4ebb78072e18430f93a3fac65827b60c@git.apache.org> References: <4ebb78072e18430f93a3fac65827b60c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/55] [abbrv] ambari git commit: AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via oleewere) archived-at: Mon, 07 Nov 2016 20:51:33 -0000 AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via oleewere) Change-Id: I9e357536115a691801013932cf13051908170d93 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/98da69f4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/98da69f4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/98da69f4 Branch: refs/heads/branch-dev-patch-upgrade Commit: 98da69f4498cd7f9c0f80f637bf33f8d5bcfe087 Parents: aaf802b Author: Miklos Gergely Authored: Wed Oct 19 17:24:58 2016 +0200 Committer: oleewere Committed: Sat Nov 5 16:08:56 2016 +0100 ---------------------------------------------------------------------- .../ambari/logfeeder/filter/FilterKeyValue.java | 65 ++- .../logfeeder/mapper/MapperFieldCopy.java | 58 ++ .../src/main/resources/alias_config.json | 3 + .../logfeeder/mapper/MapperFieldCopyTest.java | 71 +++ .../configsets/audit_logs/conf/managed-schema | 5 + .../logsearch/common/LogSearchConstants.java | 2 +- .../ambari/logsearch/dao/AuditSolrDao.java | 2 +- .../ambari/logsearch/dao/SolrCollectionDao.java | 2 +- .../logsearch/dao/SolrSchemaFieldDao.java | 105 ++-- .../apache/ambari/logsearch/dao/UserDao.java | 2 +- .../logsearch/graph/GraphDataGenerator.java | 10 +- .../ambari/logsearch/manager/ManagerBase.java | 3 +- .../logsearch/manager/ServiceLogsManager.java | 6 +- .../logsearch/manager/UserConfigManager.java | 8 +- .../logsearch/model/response/CommonLogData.java | 13 +- .../logsearch/solr/model/SolrAuditLogData.java | 1 - .../logsearch/solr/model/SolrCommonLogData.java | 42 ++ .../solr/model/SolrServiceLogData.java | 34 -- .../apache/ambari/logsearch/util/SolrUtil.java | 62 +- .../LogsearchKRBAuthenticationFilter.java | 12 +- .../logsearch/web/security/LdapProperties.java | 2 +- .../scripts/views/audit/AuditTabLayoutView.js | 2 +- .../src/main/webapp/static/schema_fields.json | 19 +- .../test-config/logfeeder/logfeeder.properties | 3 +- .../shipper-conf/input.config-ambari.json | 585 +++++++++++++++++++ .../test-logs/ambari-server/ambari-audit.log | 390 +++++++++++++ .../templates/input.config-ambari.json.j2 | 119 ++++ .../templates/input.config-falcon.json.j2 | 2 +- 28 files changed, 1458 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java index 5bb15ff..b04a439 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java @@ -19,6 +19,7 @@ package org.apache.ambari.logfeeder.filter; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -68,18 +69,26 @@ public class FilterKeyValue extends Filter { if (sourceField == null) { return; } - Object valueObj = jsonObj.get(sourceField); - if (valueObj != null) { + if (jsonObj.containsKey(sourceField)) { + String keyValueString = (String) jsonObj.get(sourceField); + Map valueMap = new HashMap<>(); + if (valueBorders != null) { + keyValueString = preProcessBorders(keyValueString, valueMap); + } + String splitPattern = Pattern.quote(fieldSplit); - String[] tokens = valueObj.toString().split(splitPattern); + String[] tokens = keyValueString.split(splitPattern); for (String nv : tokens) { String[] nameValue = getNameValue(nv); String name = nameValue != null && nameValue.length == 2 ? nameValue[0] : null; String value = nameValue != null && nameValue.length == 2 ? nameValue[1] : null; if (name != null && value != null) { - jsonObj.put(name, value); - } else { - logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj); + if (valueMap.containsKey(value)) { + value = valueMap.get(value); + } + jsonObj.put(name, value); + } else { + logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + keyValueString); } } } @@ -87,19 +96,41 @@ public class FilterKeyValue extends Filter { statMetric.value++; } - private String[] getNameValue(String nv) { - if (valueBorders != null) { - if (nv.charAt(nv.length() - 1) == valueBorders.charAt(1)) { - String splitPattern = Pattern.quote("" + valueBorders.charAt(0)); - return nv.substring(0, nv.length() - 1).split(splitPattern); - } else { - return null; + private String preProcessBorders(String keyValueString, Map valueMap) { + char openBorder = valueBorders.charAt(0); + char closeBorder = valueBorders.charAt(1); + + StringBuilder processed = new StringBuilder(); + int lastPos = 0; + int openBorderNum = 0; + int valueNum = 0; + for (int pos = 0; pos < keyValueString.length(); pos++) { + char c = keyValueString.charAt(pos); + if (c == openBorder) { + if (openBorderNum == 0 ) { + processed.append(keyValueString.substring(lastPos, pos)); + lastPos = pos + 1; + } + openBorderNum++; + } + if (c == closeBorder) { + openBorderNum--; + if (openBorderNum == 0) { + String value = keyValueString.substring(lastPos, pos).trim(); + String valueId = "$VALUE" + (++valueNum); + valueMap.put(valueId, value); + processed.append(valueSplit + valueId); + lastPos = pos + 1; + } } } - else { - String splitPattern = Pattern.quote(valueSplit); - return nv.split(splitPattern); - } + + return processed.toString(); + } + + private String[] getNameValue(String nv) { + String splitPattern = Pattern.quote(valueSplit); + return nv.split(splitPattern); } private void logParseError(String inputStr) { http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopy.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopy.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopy.java new file mode 100644 index 0000000..39e1ff4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopy.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logfeeder.mapper; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +/** + * Overrides the value for the field + */ +public class MapperFieldCopy extends Mapper { + private static final Logger LOG = Logger.getLogger(MapperFieldCopy.class); + + private String copyName = null; + + @Override + public boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs) { + init(inputDesc, fieldName, mapClassCode); + if (!(mapConfigs instanceof Map)) { + LOG.fatal("Can't initialize object. mapConfigs class is not of type Map. " + mapConfigs.getClass().getName()); + return false; + } + + @SuppressWarnings("unchecked") + Map mapObjects = (Map) mapConfigs; + copyName = (String) mapObjects.get("copy_name"); + if (StringUtils.isEmpty(copyName)) { + LOG.fatal("Map copy name is empty."); + return false; + } + return true; + } + + @Override + public Object apply(Map jsonObj, Object value) { + jsonObj.put(copyName, value); + return value; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json index b15db63..e2ed625 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json @@ -25,6 +25,9 @@ "map_date": { "klass": "org.apache.ambari.logfeeder.mapper.MapperDate" }, + "map_fieldcopy": { + "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldCopy" + }, "map_fieldname": { "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java new file mode 100644 index 0000000..108c96e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.logfeeder.mapper; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MapperFieldCopyTest { + private static final Logger LOG = Logger.getLogger(MapperFieldCopyTest.class); + + @Test + public void testMapperFieldCopy_copyField() { + LOG.info("testMapperFieldCopy_copyField()"); + + Map mapConfigs = new HashMap<>(); + mapConfigs.put("copy_name", "someOtherField"); + + MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); + assertTrue("Could not initialize!", mapperFieldCopy.init(null, "someField", null, mapConfigs)); + + Map jsonObj = new HashMap<>(); + jsonObj.put("someField", "someValue"); + + mapperFieldCopy.apply(jsonObj, "someValue"); + + assertEquals("Old field name wasn't removed", "someValue", jsonObj.remove("someField")); + assertEquals("New field wasn't put", "someValue", jsonObj.remove("someOtherField")); + assertTrue("jsonObj is not empty", jsonObj.isEmpty()); + } + + @Test + public void testMapperFielCopy_configNotMap() { + LOG.info("testMapperFieldCopy_configNotMap()"); + + MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); + assertFalse("Was able to initialize!", mapperFieldCopy.init(null, "someField", null, "")); + } + + @Test + public void testMapperFieldCopy_noNewFieldName() { + LOG.info("testMapperFieldCopy_noNewFieldName()"); + + Map mapConfigs = new HashMap<>(); + + MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); + assertFalse("Was able to initialize!", mapperFieldCopy.init(null, "someField", null, mapConfigs)); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema index f77bec0..4cd412b 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema @@ -109,4 +109,9 @@ + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index b13768f..fe31e6d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -41,7 +41,7 @@ public class LogSearchConstants { public static final String I_E_SEPRATOR = "\\|i\\:\\:e\\|"; //SUFFIX - public static final String NGRAM_SUFFIX = "ngram_"; + public static final String NGRAM_PREFIX = "ngram_"; //Date Format for SOLR public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS"; http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index 959d6f4..2fbdb46 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -67,7 +67,7 @@ public class AuditSolrDao extends SolrDaoBase { try { solrCollectionDao.checkSolrStatus(getSolrClient()); - boolean createAlias = (aliasNameIn != null && !StringUtils.isBlank(rangerAuditCollection)); + boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection)); solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig); if (createAlias) { solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig); http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java index 1cbac31..c83cd99 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java @@ -30,6 +30,7 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD; import javax.inject.Named; import java.io.IOException; @@ -43,7 +44,6 @@ public class SolrCollectionDao { private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class); - private static final String ROUTER_FIELD = "_router_field_"; private static final int SETUP_RETRY_SECOND = 30; /** http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index f16dc41..35bc2dc 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -25,11 +25,13 @@ import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.LukeRequest; +import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.response.LukeResponse; +import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; import org.apache.solr.client.solrj.response.schema.SchemaResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,15 +39,17 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; public class SolrSchemaFieldDao { private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class); private static final int SETUP_RETRY_SECOND = 30; - private static final int SETUP_UPDATE_SECOND = 10 * 60; // 10 min - + private static final int SETUP_UPDATE_SECOND = 1 * 60; // 1 min + private boolean populateFieldsThreadActive = false; private Map schemaFieldNameMap = new HashMap<>(); @@ -93,73 +97,84 @@ public class SolrSchemaFieldDao { * Called from the thread. Don't call this directly */ private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { - SolrRequest request = new SchemaRequest(); - request.setMethod(SolrRequest.METHOD.GET); - request.setPath("/schema"); String historyCollection = solrUserPropsConfig.getCollection(); if (solrClient != null && !solrPropsConfig.getCollection().equals(historyCollection)) { - NamedList namedList = null; + LukeResponse lukeResponse = null; + SchemaResponse schemaResponse = null; try { - namedList = solrClient.request(request); - LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", fields=" + namedList); + LukeRequest lukeRequest = new LukeRequest(); + lukeRequest.setNumTerms(0); + lukeResponse = lukeRequest.process(solrClient); + + SolrRequest schemaRequest = new SchemaRequest(); + schemaRequest.setMethod(SolrRequest.METHOD.GET); + schemaRequest.setPath("/schema"); + schemaResponse = schemaRequest.process(solrClient); + + LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", luke=" + lukeResponse + + ", schema= " + schemaResponse); } catch (SolrException | SolrServerException | IOException e) { LOG.error("Error occured while popuplating field. collection=" + solrPropsConfig.getCollection(), e); } - if (namedList != null) { - extractSchemaFieldsName(namedList.toString(), schemaFieldNameMap, schemaFieldTypeMap); + if (lukeResponse != null && schemaResponse != null) { + extractSchemaFieldsName(lukeResponse, schemaResponse); return true; } } return false; } - public void extractSchemaFieldsName(String responseString, - final Map schemaFieldsNameMap, - final Map schemaFieldTypeMap) { + private void extractSchemaFieldsName(LukeResponse lukeResponse, SchemaResponse schemaResponse) { try { - JSONObject jsonObject = new JSONObject(responseString); - JSONObject schemajsonObject = jsonObject.getJSONObject("schema"); - JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields"); - JSONArray fieldTypeJsonArray = schemajsonObject - .getJSONArray("fieldTypes"); - if (jsonArrayList == null) { - return; - } - if (fieldTypeJsonArray == null) { - return; - } + HashMap _schemaFieldNameMap = new HashMap<>(); HashMap _schemaFieldTypeMap = new HashMap<>(); - HashMap _schemaFieldsNameMap = new HashMap(); - for (int i = 0; i < fieldTypeJsonArray.length(); i++) { - JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i); - String name = typeObject.getString("name"); - String fieldTypeJson = typeObject.toString(); + + for (Entry e : lukeResponse.getFieldInfo().entrySet()) { + String name = e.getKey(); + String type = e.getValue().getType(); + if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && + !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldNameMap.put(name, type); + } + } + + List fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes(); + for (FieldTypeDefinition fieldType : fieldTypes) { + Map fieldAttributes = fieldType.getAttributes(); + String name = (String) fieldAttributes.get("name"); + String fieldTypeJson = new JSONObject(fieldAttributes).toString(); _schemaFieldTypeMap.put(name, fieldTypeJson); } - for (int i = 0; i < jsonArrayList.length(); i++) { - JSONObject explrObject = jsonArrayList.getJSONObject(i); - String name = explrObject.getString("name"); - String type = explrObject.getString("type"); + + List> fields = schemaResponse.getSchemaRepresentation().getFields(); + for (Map field : fields) { + String name = (String) field.get("name"); + String type = (String) field.get("type"); if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && - !name.contains(LogSearchConstants.NGRAM_SUFFIX) && !name.contains("tags") && !name.contains("_str")) { - _schemaFieldsNameMap.put(name, type); + !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldNameMap.put(name, type); } } - schemaFieldsNameMap.clear(); - schemaFieldTypeMap.clear(); - schemaFieldsNameMap.putAll(_schemaFieldsNameMap); - schemaFieldTypeMap.putAll(_schemaFieldTypeMap); + + if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) { + return; + } + + synchronized (this) { + schemaFieldNameMap = _schemaFieldNameMap; + schemaFieldTypeMap = _schemaFieldTypeMap; + } } catch (Exception e) { LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM); } } - public Map getSchemaFieldTypeMap() { - return schemaFieldTypeMap; + public synchronized Map getSchemaFieldNameMap() { + return schemaFieldNameMap; } - public Map getSchemaFieldNameMap() { - return schemaFieldNameMap; + public synchronized Map getSchemaFieldTypeMap() { + return schemaFieldTypeMap; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java index 912442d..63cc89e 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java @@ -135,7 +135,7 @@ public class UserDao { String encPassword = user.get(ENC_PASSWORD); String username = user.get(USER_NAME); String password = user.get(PASSWORD); - if (!StringUtils.isBlank(password)) { + if (StringUtils.isNotBlank(password)) { encPassword = CommonUtil.encryptPassword(username, password); user.put(PASSWORD, ""); user.put(ENC_PASSWORD, encPassword); http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java index 85b5fda..741c523 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java @@ -249,13 +249,13 @@ public class GraphDataGenerator { NodeData hostNode = new NodeData(); String name = (pivotHost.getValue() == null ? "" : "" + pivotHost.getValue()); String value = "" + pivotHost.getCount(); - if (!StringUtils.isBlank(name)) { + if (StringUtils.isNotBlank(name)) { hostNode.setName(name); } - if (!StringUtils.isBlank(value)) { + if (StringUtils.isNotBlank(value)) { hostNode.setValue(value); } - if (!StringUtils.isBlank(firstPriority)) { + if (StringUtils.isNotBlank(firstPriority)) { hostNode.setType(firstPriority); } @@ -263,7 +263,7 @@ public class GraphDataGenerator { hostNode.setRoot(true); PivotField hostPivot = null; for (PivotField searchHost : secondHirarchicalPivotFields) { - if (!StringUtils.isBlank(hostNode.getName()) + if (StringUtils.isNotBlank(hostNode.getName()) && hostNode.getName().equals(searchHost.getValue())) { hostPivot = searchHost; break; @@ -291,7 +291,7 @@ public class GraphDataGenerator { NodeData compNode = new NodeData(); String compName = (pivotComp.getValue() == null ? "" : "" + pivotComp.getValue()); compNode.setName(compName); - if (!StringUtils.isBlank(secondPriority)) { + if (StringUtils.isNotBlank(secondPriority)) { compNode.setType(secondPriority); } compNode.setValue("" + pivotComp.getCount()); http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java index 45d66cf..6c280ac 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java @@ -32,6 +32,7 @@ import org.apache.ambari.logsearch.model.response.LogSearchResponse; import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; @@ -110,7 +111,7 @@ public abstract class ManagerBase serviceLogDataList = convertToSolrBeans(response); - if (docList != null && !docList.isEmpty()) { + if (CollectionUtils.isNotEmpty(docList)) { logResponse.setLogList(serviceLogDataList); logResponse.setStartIndex((int) docList.getStart()); logResponse.setTotalCount(docList.getNumFound()); http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java index d276769..74c549a 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java @@ -118,7 +118,7 @@ public class ServiceLogsManager extends ManagerBase getAllDynamicFields(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java index 0fd65a3..9de05db 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java @@ -23,7 +23,6 @@ import org.apache.solr.client.solrj.beans.Field; import java.util.Date; import java.util.List; -import java.util.Map; import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.*; http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java index bac5080..b5ce7ad 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrCommonLogData.java @@ -22,8 +22,13 @@ import org.apache.ambari.logsearch.model.response.CommonLogData; import org.apache.solr.client.solrj.beans.Field; import java.util.Date; +import java.util.HashMap; +import java.util.Map; import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.*; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.KEY_DYNAMIC_FIELDS; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.STORED_TOKEN_DYNAMIC_FIELDS; +import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.WS_DYNAMIC_FIELDS; public class SolrCommonLogData implements CommonLogData { @@ -78,6 +83,15 @@ public class SolrCommonLogData implements CommonLogData { @Field(ROUTER_FIELD) private Integer routerField; + @Field(STORED_TOKEN_DYNAMIC_FIELDS) + private Map stdDynamicFields; + + @Field(KEY_DYNAMIC_FIELDS) + private Map keyDynamicFields; + + @Field(WS_DYNAMIC_FIELDS) + private Map wsDynamicFields; + @Override public String getId() { return this.id; @@ -247,4 +261,32 @@ public class SolrCommonLogData implements CommonLogData { public void setType(String type) { this.type = type; } + + @Override + public Map getAllDynamicFields() { + Map allDynamicFields = new HashMap<>(); + if (stdDynamicFields != null) { + allDynamicFields.putAll(stdDynamicFields); + } + if (keyDynamicFields != null) { + allDynamicFields.putAll(keyDynamicFields); + } + if (wsDynamicFields != null) { + allDynamicFields.putAll(wsDynamicFields); + } + + return allDynamicFields; + } + + public void setStdDynamicFields(Map stdDynamicFields) { + this.stdDynamicFields = stdDynamicFields; + } + + public void setKeyDynamicFields(Map keyDynamicFields) { + this.keyDynamicFields = keyDynamicFields; + } + + public void setWsDynamicFields(Map wsDynamicFields) { + this.wsDynamicFields = wsDynamicFields; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java index 1d4cecf..c6fdba3 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java @@ -22,7 +22,6 @@ import org.apache.ambari.logsearch.model.response.ServiceLogData; import org.apache.solr.client.solrj.beans.Field; import java.util.Date; -import java.util.Map; import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.*; @@ -49,15 +48,6 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD @Field(HOST) private String host; - @Field(STORED_TOKEN_DYNAMIC_FIELDS) - private Map stdDynamicFields; - - @Field(KEY_DYNAMIC_FIELDS) - private Map keyDynamicFields; - - @Field(WS_DYNAMIC_FIELDS) - private Map wsDynamicFields; - @Override public String getPath() { return path; @@ -127,28 +117,4 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD public void setLevel(String level) { this.level = level; } - - public Map getStdDynamicFields() { - return stdDynamicFields; - } - - public void setStdDynamicFields(Map stdDynamicFields) { - this.stdDynamicFields = stdDynamicFields; - } - - public Map getKeyDynamicFields() { - return keyDynamicFields; - } - - public void setKeyDynamicFields(Map keyDynamicFields) { - this.keyDynamicFields = keyDynamicFields; - } - - public Map getWsDynamicFields() { - return wsDynamicFields; - } - - public void setWsDynamicFields(Map wsDynamicFields) { - this.wsDynamicFields = wsDynamicFields; - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java index 6369107..7a856d2 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java @@ -28,11 +28,13 @@ import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.lucene.analysis.core.KeywordTokenizerFactory; import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory; import org.apache.lucene.analysis.standard.StandardTokenizerFactory; +import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.schema.TrieDoubleField; import org.apache.solr.schema.TrieFloatField; import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; public class SolrUtil { @@ -116,18 +118,7 @@ public class SolrUtil { return search; } - public static String makeSolrSearchString(String search) { - String newString = search.trim(); - String newSearch = newString.replaceAll("(?=[]\\[+&|!(){},:\"^~/=$@%?:.\\\\])", "\\\\"); - newSearch = newSearch.replace("\n", "*"); - newSearch = newSearch.replace("\t", "*"); - newSearch = newSearch.replace("\r", "*"); - newSearch = newSearch.replace("**", "*"); - newSearch = newSearch.replace("***", "*"); - return "*" + newSearch + "*"; - } - - public static String makeSolrSearchStringWithoutAsterisk(String search) { + private static String makeSolrSearchStringWithoutAsterisk(String search) { String newString = search.trim(); String newSearch = newString.replaceAll("(?=[]\\[+&|!(){}^\"~=/$@%?:.\\\\])", "\\\\"); newSearch = newSearch.replace("\n", "*"); @@ -161,28 +152,19 @@ public class SolrUtil { } - public static boolean isSolrFieldNumber(String fieldType, Map schemaFieldsMap) { + private static boolean isSolrFieldNumber(String fieldType, Map schemaFieldsMap) { if (StringUtils.isBlank(fieldType)) { return false; } else { HashMap typeInfoMap = getFieldTypeInfoMap(fieldType, schemaFieldsMap); - if (typeInfoMap == null || typeInfoMap.isEmpty()) { + if (MapUtils.isEmpty(typeInfoMap)) { return false; } String fieldTypeClassName = (String) typeInfoMap.get("class"); - if (fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName())) { - return true; - } - if (fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())) { - return true; - } - if (fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName())) { - return true; - } - if (fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName())) { - return true; - } - return false; + return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName()); } } @@ -237,7 +219,7 @@ public class SolrUtil { } } - public static HashMap getFieldTypeInfoMap(String fieldType, Map schemaFieldsTypeMap) { + private static HashMap getFieldTypeInfoMap(String fieldType, Map schemaFieldsTypeMap) { String fieldTypeMetaData = schemaFieldsTypeMap.get(fieldType); HashMap fieldTypeMap = JSONUtil.jsonToMapObject(fieldTypeMetaData); if (fieldTypeMap == null) { @@ -273,7 +255,7 @@ public class SolrUtil { setFacetLimit(solrQuery, -1); } - public static void setFacetLimit(SolrQuery solrQuery, int limit) { + private static void setFacetLimit(SolrQuery solrQuery, int limit) { solrQuery.set("facet.limit", limit); } @@ -286,22 +268,20 @@ public class SolrUtil { } } - private static boolean checkTokenizer(String fieldType, Class tokenizerFactoryClass, Map schemaFieldsMap) { + @SuppressWarnings("unchecked") + private static boolean checkTokenizer(String fieldType, Class tokenizerFactoryClass, + Map schemaFieldsMap) { HashMap fieldTypeMap = SolrUtil.getFieldTypeInfoMap(fieldType ,schemaFieldsMap); HashMap analyzer = (HashMap) fieldTypeMap.get("analyzer"); - if (analyzer != null) { - HashMap tokenizerMap = (HashMap) analyzer.get("tokenizer"); - if (tokenizerMap != null) { - String tokenizerClass = (String) tokenizerMap.get("class"); - if (!StringUtils.isEmpty(tokenizerClass)) { - tokenizerClass =tokenizerClass.replace("solr.", ""); - if (tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass - .getSimpleName())) { - return true; - } - } + HashMap tokenizerMap = (HashMap)MapUtils.getObject(analyzer, "tokenizer"); + if (tokenizerMap != null) { + String tokenizerClass = (String) tokenizerMap.get("class"); + if (StringUtils.isNotEmpty(tokenizerClass)) { + tokenizerClass = tokenizerClass.replace("solr.", ""); + return tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass.getSimpleName()); } } + return false; } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java index 29fd5b2..808320d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java @@ -148,7 +148,7 @@ public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter { HttpServletResponse response) throws IOException, ServletException { logger.debug("LogsearchKRBAuthenticationFilter private filter"); String userName = getUsernameFromResponse(response); - if (!StringUtils.isEmpty(userName)) { + if (StringUtils.isNotEmpty(userName)) { Authentication existingAuth = SecurityContextHolder.getContext() .getAuthentication(); if (existingAuth == null || !existingAuth.isAuthenticated()) { @@ -199,7 +199,7 @@ public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter { KerberosName.setRules(PropertiesHelper.getProperty(NAME_RULES, "DEFAULT")); String userName = getUsernameFromRequest(httpRequest); if ((existingAuth == null || !existingAuth.isAuthenticated()) - && (!StringUtils.isEmpty(userName))) { + && (StringUtils.isNotEmpty(userName))) { // --------------------------- To Create Logsearch Session-------------------------------------- // if we get the userName from the token then log into logsearch using the same user final List grantedAuths = new ArrayList<>(); @@ -236,8 +236,8 @@ public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter { String keytab = PropertiesHelper.getProperty(KEYTAB); String principal = PropertiesHelper.getProperty(PRINCIPAL); String hostname = PropertiesHelper.getProperty(HOST_NAME); - if (!StringUtils.isEmpty(keytab) && !StringUtils.isEmpty(principal) - && !StringUtils.isEmpty(hostname)) { + if (StringUtils.isNotEmpty(keytab) && StringUtils.isNotEmpty(principal) + && StringUtils.isNotEmpty(hostname)) { spnegoEnable = true; } } @@ -306,7 +306,7 @@ public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter { Iterator iterator = cookiesCollection.iterator(); while (iterator.hasNext()) { String cookie = iterator.next(); - if (!StringUtils.isEmpty(cookie)) { + if (StringUtils.isNotEmpty(cookie)) { if (cookie.toLowerCase().startsWith(AUTH_COOKIE_NAME.toLowerCase())) { Matcher m = usernamePattern.matcher(cookie); if (m.find()) { @@ -314,7 +314,7 @@ public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter { } } } - if (!StringUtils.isEmpty(userName)) { + if (StringUtils.isNotEmpty(userName)) { break; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LdapProperties.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LdapProperties.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LdapProperties.java index 2a1b4ee..82e71fe 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LdapProperties.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LdapProperties.java @@ -63,7 +63,7 @@ public class LdapProperties { } else { List list = new ArrayList(); list.add(protocol + primaryUrl); - if (!StringUtils.isEmpty(secondaryUrl)) { + if (StringUtils.isNotEmpty(secondaryUrl)) { list.add(protocol + secondaryUrl); } return list; http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js index 4e09e88..7710156 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js @@ -312,7 +312,7 @@ define(['require', if (columns[name] === undefined && value != "_version_") { var columnObj = { name: Globals.invertedAuditLogMappings[value], - label:value, + label: value, cell: "String", sortType: 'toggle', editable: false http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json index 7b1a9a9..55f1d3c 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json @@ -65,7 +65,24 @@ "text": "Text", "type": "Type", "ugi": "UGI", - "reqUser": "User" + "reqUser": "User", + + "ws_base_url": "Base URL", + "ws_command": "Command", + "ws_component": "Component", + "ws_details": "Details", + "ws_display_name": "Display Name", + "ws_os": "OS", + "ws_repo_id": "Repo Id", + "ws_repo_version": "Repo Version", + "ws_repositories": "Repositories", + "ws_result_status": "Result Status", + "ws_roles": "Roles", + "ws_stack_version": "Stack Version", + "ws_stack": "Stack", + "ws_status": "Status", + "ws_version_note": "Version Note", + "ws_version_number": "Version Number" }, "auditLogExcludes": ["tags","tags_str", "seq_num"] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties index f930ec9..879b786 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties +++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties @@ -22,7 +22,8 @@ logfeeder.config.files=shipper-conf/global.config.json,\ shipper-conf/input.config-hst.json,\ shipper-conf/input.config-system_message.json,\ shipper-conf/input.config-secure_log.json,\ - shipper-conf/input.config-hdfs.json + shipper-conf/input.config-hdfs.json,\ + shipper-conf/input.config-ambari.json logfeeder.log.filter.enable=true logfeeder.solr.config.interval=5 logfeeder.solr.core.config.name=history http://git-wip-us.apache.org/repos/asf/ambari/blob/98da69f4/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json new file mode 100644 index 0000000..576bfa8 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json @@ -0,0 +1,585 @@ +{ + "input":[ + { + "type":"ambari_agent", + "rowtype":"service", + "path":"/root/test-logs/ambari-server/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"/root/test-logs/ambari-server/ambari-server.log" + }, + { + "type":"ambari_alerts", + "rowtype":"service", + "add_fields":{ + "level":"INFO" + }, + "path":"/root/test-logs/ambari-server/ambari-alerts.log" + }, + { + "type":"ambari_config_changes", + "rowtype":"service", + "path":"/root/test-logs/ambari-server/ambari-config-changes.log" + }, + { + "type":"ambari_eclipselink", + "rowtype":"service", + "path":"/root/test-logs/ambari-server/ambari-eclipselink.log" + }, + { + "type":"ambari_server_check_database", + "rowtype":"service", + "path":"/root/test-logs/ambari-server/ambari-server-check-database.log" + }, + { + "type":"ambari_audit", + "rowtype":"audit", + "add_fields":{ + "logType":"AmbariAudit", + "enforcer":"ambari-acl", + "repoType":"1", + "repo":"ambari", + "level":"INFO" + }, + "path":"/root/test-logs/ambari-server/ambari-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_alerts" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_config_changes" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_eclipselink" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(\\[EL%{SPACE}%{LOGLEVEL:level}\\])", + "message_pattern":"(?m)^\\[EL%{SPACE}%{LOGLEVEL:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"Warning", + "post_value":"Warn" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server_check_database" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "source_field":"log_message", + "field_split":", ", + "value_borders":"()", + "post_map_values":{ + "User":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "Hostname":{ + "map_fieldname":{ + "new_fieldname":"host" + } + }, + "Host name":{ + "map_fieldname":{ + "new_fieldname":"host" + } + }, + "RemoteIp":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + }, + "RequestType":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + }, + "RequestId":{ + "map_fieldname":{ + "new_fieldname":"request_id" + } + }, + "TaskId":{ + "map_fieldname":{ + "new_fieldname":"task_id" + } + }, + "Operation":{ + "map_fieldname":{ + "new_fieldname":"action" + } + }, + "url":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "ResourcePath":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "Cluster name":{ + "map_fieldname":{ + "new_fieldname":"cluster" + } + }, + "Reason":{ + "map_fieldname":{ + "new_fieldname":"reason" + } + }, + "Base URL":{ + "map_fieldname":{ + "new_fieldname":"ws_base_url" + } + }, + "Command":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_command" + } + }, + "Component":{ + "map_fieldname":{ + "new_fieldname":"ws_component" + } + }, + "Details":{ + "map_fieldname":{ + "new_fieldname":"ws_details" + } + }, + "Display name":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_display_name" + } + }, + "OS":{ + "map_fieldname":{ + "new_fieldname":"ws_os" + } + }, + "Repo id":{ + "map_fieldname":{ + "new_fieldname":"ws_repo_id" + } + }, + "Repo version":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_repo_version" + } + }, + "Repositories":{ + "map_fieldname":{ + "new_fieldname":"ws_repositories" + } + }, + "Roles":{ + "map_fieldname":{ + "new_fieldname":"ws_roles" + } + }, + "Stack":{ + "map_fieldname":{ + "new_fieldname":"ws_stack" + } + }, + "Stack version":{ + "map_fieldname":{ + "new_fieldname":"ws_stack_version" + } + }, + "VersionNote":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_version_note" + } + }, + "VersionNumber":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_version_number" + } + }, + "Status":[ + { + "map_fieldcopy":{ + "copy_name": "ws_status" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Success", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Successfully queued", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"QUEUED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"PENDING", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"COMPLETED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"IN_PROGRESS", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed to queue", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"ABORTED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"SKIPPED_FAILED", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ], + "ResultStatus":[ + { + "map_fieldcopy":{ + "copy_name": "ws_result_status" + } + }, + { + "map_fieldvalue":{ + "pre_value":"200 OK", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"201 Created", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"202 Accepted", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"400 Bad Request", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"401 Unauthorized", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"403 Forbidden", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"404 Not Found", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"409 Resource Conflict", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"500 Internal Server Error", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ] + + } + + } + + ] + +}