Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 26BBB200B92 for ; Wed, 28 Sep 2016 11:41:34 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 25574160AEC; Wed, 28 Sep 2016 09:41:34 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id DC93B160AF3 for ; Wed, 28 Sep 2016 11:41:30 +0200 (CEST) Received: (qmail 23191 invoked by uid 500); 28 Sep 2016 09:41:30 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 21953 invoked by uid 99); 28 Sep 2016 09:41:29 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Sep 2016 09:41:29 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 526E0EEE25; Wed, 28 Sep 2016 09:41:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Date: Wed, 28 Sep 2016 09:41:45 -0000 Message-Id: <0f561ea87b724660bc512f0a5524acd8@git.apache.org> In-Reply-To: <036b5005c34744d18f1506dbea4ae745@git.apache.org> References: <036b5005c34744d18f1506dbea4ae745@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/52] [abbrv] ambari git commit: AMBARI-18310. Refactor logsearch portal side code (oleewere) archived-at: Wed, 28 Sep 2016 09:41:34 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index 0ceb76b..8e14452 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -26,18 +26,25 @@ public class DocConstants { public static final String STACK_BY_D = "The graph property for stacking the plot"; public static final String EXCLUDE_QUERY_D = "Exclude the values in query result e.g.: [{message:*timeout*}]"; public static final String INCLUDE_QUERY_D = "Include the values in query result e.g.: [{message:*exception*}]"; - public static final String MUST_BE_D = "Include the components, comman separated values"; - public static final String MUST_NOT_D = "Exclude the components, comman separated values"; + public static final String MUST_BE_D = "Include the components, comma separated values"; + public static final String MUST_NOT_D = "Exclude the components, comma separated values"; public static final String FROM_D = "Date range param, start date"; public static final String TO_D = "Date range param, end date"; + public static final String START_TIME_D = "Date range param which is suportted from browser url"; + public static final String END_TIME_D = "Date range param which is supported from browser url"; + public static final String START_INDEX_D = "Start index of the queried result"; + public static final String SORT_TYPE_D = "Type of sorting (osc, desc)"; + public static final String SORT_BY_D = "Sorting the results based on this field"; + public static final String PAGE_D = "Number of pages for the results"; + public static final String PAGE_SIZE_D = "Page size of the results"; public static final String UNIT_D = "Aggregate the data with time gap as unit i.e 1MINUTE"; public static final String QUERY_D = "not required"; public static final String COLUMN_QUERY_D = "not required"; - public static final String I_MESSAGE_D = "Include query which will query againt message column"; + public static final String I_MESSAGE_D = "Include query which will query against message column"; public static final String G_E_MESSAGE_D = "not required"; - public static final String E_MESSAGE_D = "Exclude query which will query againt message column"; - public static final String IS_LAST_PAGE_D = ""; - public static final String FIELD_D = "Get top ten values for particular field"; + public static final String E_MESSAGE_D = "Exclude query which will query against message column"; + public static final String IS_LAST_PAGE_D = "Show last page (true/false)"; + public static final String FIELD_D = "Get values for particular field"; public static final String FORMAT_D = "File Export format, can be 'txt' or 'json'"; } @@ -48,7 +55,6 @@ public class DocConstants { public static final String GET_AUDIT_LINE_GRAPH_DATA_OD = "Get the data required for line graph"; public static final String GET_TOP_AUDIT_USERS_OD = "Get the top audit users having maximum access"; public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resources having maximum access"; - public static final String GET_TOP_AUDIT_COMPONENTS_OD = "not required"; public static final String GET_LIVE_LOGS_COUNT_OD = "not required"; public static final String GET_REQUEST_USER_LINE_GRAPH_OD = "not required"; public static final String GET_ANY_GRAPH_DATA_OD = "Get the data generic enough to use for graph plots"; @@ -59,11 +65,11 @@ public class DocConstants { public class ServiceDescriptions { public static final String LEVEL_D = "filter for log level"; public static final String ADVANCED_SEARCH_D = "not required"; - public static final String TREE_PARAMS_D = "Host hierarchy shown on UI,filtering there is supported by this param"; - public static final String START_TIME_D = "Date range param which is suportted from browser url"; - public static final String END_TIME_D = "Date range param which is supported from browser url"; + public static final String BUNDLE_ID = "filter for host"; + public static final String TREE_PARAMS_D = "Host hierarchy shown on UI, filtering there is supported by this param"; public static final String FILE_NAME_D = "File name filter which is supported from browser url"; public static final String HOST_NAME_D = "Host name filter which is supported from browser url"; + public static final String DATE_RANGE_LABEL_D = "Date range label (e.g.: Today)"; public static final String COMPONENT_NAME_D = "Component name filter which is supported from browser url"; public static final String FIND_D = "Finding particular text on subsequent pages in case of table view with pagination"; public static final String ID_D = "Log id value for traversing to that particular record with that log id"; http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java index d84b7b9..bc377e5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java @@ -26,14 +26,14 @@ import java.util.List; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.model.response.BarGraphData; +import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; +import org.apache.ambari.logsearch.model.response.NameValueData; +import org.apache.ambari.logsearch.query.model.SearchCriteria; import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.ambari.logsearch.query.QueryGeneration; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.ambari.logsearch.util.SolrUtil; -import org.apache.ambari.logsearch.view.VBarDataList; -import org.apache.ambari.logsearch.view.VBarGraphData; -import org.apache.ambari.logsearch.view.VNameValue; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; @@ -44,18 +44,19 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.inject.Inject; + @Component public class GraphDataGenerator extends GraphDataGeneratorBase { private static final Logger logger = Logger.getLogger(GraphDataGenerator.class); - @Autowired + @Inject private QueryGeneration queryGenerator; - public VBarDataList getAnyGraphData(SearchCriteria searchCriteria, SolrDaoBase solrDaoBase, SolrQuery solrQuery) { + public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria, SolrDaoBase solrDaoBase, SolrQuery solrQuery) { // X axis credentials String xAxisField = (String) searchCriteria.getParamValue("xAxis"); String stackField = (String) searchCriteria.getParamValue("stackBy"); @@ -119,12 +120,12 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } @SuppressWarnings("unchecked") - private VBarDataList normalGraph(String xAxisField, String yAxisField, String from, String to, SolrDaoBase solrDaoBase, + private BarGraphDataListResponse normalGraph(String xAxisField, String yAxisField, String from, String to, SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { - VBarDataList dataList = new VBarDataList(); - Collection vBarGraphDatas = new ArrayList(); - VBarGraphData vBarGraphData = new VBarGraphData(); - Collection vNameValues = new ArrayList(); + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + Collection vBarGraphDatas = new ArrayList(); + BarGraphData vBarGraphData = new BarGraphData(); + Collection vNameValues = new ArrayList(); SolrUtil.setMainQuery(solrQuery, null); queryGenerator.setSingleIncludeFilter(solrQuery, fieldTime, "[" + from + " TO " + to + "]"); if (typeXAxis.contains("string") || typeXAxis.contains("key_lower_case") || typeXAxis.contains("text")) { @@ -140,7 +141,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { if (countValues != null) { for (Count countValue : countValues) { if (countValue != null) { - VNameValue vNameValue = new VNameValue(); + NameValueData vNameValue = new NameValueData(); vNameValue.setName(countValue.getName()); vNameValue.setValue("" + countValue.getCount()); vNameValues.add(vNameValue); @@ -154,12 +155,12 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } } if (xAxisField.equalsIgnoreCase(LogSearchConstants.SOLR_LEVEL)) { - Collection sortedVNameValues = new ArrayList(); + Collection sortedVNameValues = new ArrayList(); for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { - VNameValue value = new VNameValue(); + NameValueData value = new NameValueData(); value.setName(level); String val = "0"; - for (VNameValue valueLevel : vNameValues) { + for (NameValueData valueLevel : vNameValues) { if (valueLevel.getName().equalsIgnoreCase(level)) { val = valueLevel.getValue(); break; @@ -168,9 +169,9 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { value.setValue(val); sortedVNameValues.add(value); } - vBarGraphData.setDataCounts(sortedVNameValues); + vBarGraphData.setDataCount(sortedVNameValues); } else { - vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setDataCount(vNameValues); } return dataList; } catch (SolrException | SolrServerException | IOException e) { @@ -188,12 +189,12 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { if (jsonFacetResponse.toString().equals("{count=0}")) { return dataList; } - VNameValue value = new VNameValue(); + NameValueData value = new NameValueData(); String sum = (String) jsonFacetResponse.getVal(1); value.setName(xAxisField); value.setValue(sum != null ? sum.substring(0, sum.indexOf(".")) : ""); vNameValues.add(value); - vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setDataCount(vNameValues); vBarGraphData.setName(xAxisField); vBarGraphDatas.add(vBarGraphData); dataList.setGraphData(vBarGraphDatas); @@ -207,10 +208,10 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } @SuppressWarnings("unchecked") - private VBarDataList nonRangeStackGraph(String xAxisField, String yAxisField, String stackField, String from, String to, + private BarGraphDataListResponse nonRangeStackGraph(String xAxisField, String yAxisField, String stackField, String from, String to, SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { - VBarDataList dataList = new VBarDataList(); - Collection vGraphData = new ArrayList(); + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + Collection vGraphData = new ArrayList(); String mainQuery = queryGenerator.buildInclusiveRangeFilterQuery(fieldTime, from, to); SolrUtil.setMainQuery(solrQuery, mainQuery); SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); @@ -238,24 +239,24 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } extractNonRangeStackValuesFromBucket(jsonFacetResponse, stackField, vGraphData, typeXAxis); if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField) && LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(xAxisField)) { - Collection levelVGraphData = dataList.getGraphData(); - for (VBarGraphData garphData : levelVGraphData) { - Collection valueList = garphData.getDataCount(); - Collection valueListSorted = new ArrayList(); + Collection levelVGraphData = dataList.getGraphData(); + for (BarGraphData graphData : levelVGraphData) { + Collection valueList = graphData.getDataCount(); + Collection valueListSorted = new ArrayList(); for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { String val = "0"; - for (VNameValue value : valueList) { + for (NameValueData value : valueList) { if (value.getName().equalsIgnoreCase(level)) { val = value.getValue(); break; } } - VNameValue v1 = new VNameValue(); + NameValueData v1 = new NameValueData(); v1.setName(level.toUpperCase()); v1.setValue(val); valueListSorted.add(v1); } - garphData.setDataCounts(valueListSorted); + graphData.setDataCount(valueListSorted); } } return dataList; @@ -267,12 +268,12 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } @SuppressWarnings("unchecked") - private VBarDataList rangeNonStackGraph(String xAxisField, String yAxisField, String from, String to, String unit, + private BarGraphDataListResponse rangeNonStackGraph(String xAxisField, String yAxisField, String from, String to, String unit, SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { - VBarDataList dataList = new VBarDataList(); - Collection vBarGraphDatas = new ArrayList(); - VBarGraphData vBarGraphData = new VBarGraphData(); - Collection vNameValues = new ArrayList(); + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + Collection vBarGraphDatas = new ArrayList(); + BarGraphData vBarGraphData = new BarGraphData(); + Collection vNameValues = new ArrayList(); SolrUtil.setMainQuery(solrQuery, null); if (SolrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) { queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to); @@ -290,12 +291,12 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { List listCount = rangeFacet.get(0).getCounts(); if (listCount != null) { for (RangeFacet.Count cnt : listCount) { - VNameValue vNameValue = new VNameValue(); + NameValueData vNameValue = new NameValueData(); vNameValue.setName(cnt.getValue()); vNameValue.setValue("" + cnt.getCount()); vNameValues.add(vNameValue); } - vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setDataCount(vNameValues); vBarGraphDatas.add(vBarGraphData); vBarGraphData.setName(xAxisField); dataList.setGraphData(vBarGraphDatas); @@ -312,10 +313,10 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } @SuppressWarnings("unchecked") - private VBarDataList rangeStackGraph(String xAxisField, String stackField, String from, String to, String unit, + private BarGraphDataListResponse rangeStackGraph(String xAxisField, String stackField, String from, String to, String unit, SolrDaoBase solrDaoBase, SolrQuery solrQuery) { - VBarDataList dataList = new VBarDataList(); - List histogramData = new ArrayList(); + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + List histogramData = new ArrayList(); SolrUtil.setMainQuery(solrQuery, null); SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); String jsonHistogramQuery = http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java index e7fab9a..c57e0e9 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java @@ -23,15 +23,15 @@ import java.util.Collection; import java.util.Date; import java.util.List; -import org.apache.ambari.logsearch.manager.MgrBase; +import org.apache.ambari.logsearch.manager.ManagerBase; +import org.apache.ambari.logsearch.model.response.BarGraphData; +import org.apache.ambari.logsearch.model.response.NameValueData; import org.apache.ambari.logsearch.util.DateUtil; -import org.apache.ambari.logsearch.view.VBarGraphData; -import org.apache.ambari.logsearch.view.VNameValue; import org.apache.commons.lang.StringUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; -class GraphDataGeneratorBase extends MgrBase { +class GraphDataGeneratorBase { private static final String BUCKETS = "buckets"; @@ -62,19 +62,19 @@ class GraphDataGeneratorBase extends MgrBase { @SuppressWarnings("unchecked") protected void extractRangeStackValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String outerField, - String innerField, List histogramData) { + String innerField, List histogramData) { if (jsonFacetResponse != null) { NamedList stack = (NamedList) jsonFacetResponse.get(outerField); if (stack != null) { ArrayList stackBuckets = (ArrayList) stack.get(BUCKETS); if (stackBuckets != null) { for (Object stackBucket : stackBuckets) { - VBarGraphData vBarGraphData = new VBarGraphData(); + BarGraphData vBarGraphData = new BarGraphData(); SimpleOrderedMap level = (SimpleOrderedMap) stackBucket; if (level != null) { String name = level.getVal(0) != null ? level.getVal(0).toString().toUpperCase() : ""; vBarGraphData.setName(name); - Collection vNameValues = new ArrayList(); + Collection vNameValues = new ArrayList(); NamedList innerFiledValue = (NamedList) level.get(innerField); if (innerFiledValue != null) { ArrayList levelBuckets = (ArrayList) innerFiledValue.get(BUCKETS); @@ -84,13 +84,15 @@ class GraphDataGeneratorBase extends MgrBase { if (countValue != null) { String innerName = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); String innerValue = countValue.getVal(1) != null ? countValue.getVal(1).toString() : ""; - VNameValue vNameValue = new VNameValue(innerName, innerValue); + NameValueData vNameValue = new NameValueData(); + vNameValue.setName(innerName); + vNameValue.setValue(innerValue); vNameValues.add(vNameValue); } } } } - vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setDataCount(vNameValues); } histogramData.add(vBarGraphData); } @@ -101,7 +103,7 @@ class GraphDataGeneratorBase extends MgrBase { @SuppressWarnings("unchecked") protected boolean extractNonRangeStackValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String level, - Collection vGraphDatas, String typeXAxis) { + Collection vGraphDatas, String typeXAxis) { boolean zeroFlag = true; if (jsonFacetResponse == null || jsonFacetResponse.get(level) == null || jsonFacetResponse.get(level).toString().equals("{count=0}")) { @@ -114,11 +116,11 @@ class GraphDataGeneratorBase extends MgrBase { for (int index = 0; index < bucketList.size(); index++) { SimpleOrderedMap valueCount = (SimpleOrderedMap) bucketList.get(index); if (valueCount != null && valueCount.size() > 2) { - VBarGraphData vGraphData = new VBarGraphData(); - Collection levelCounts = new ArrayList(); + BarGraphData vGraphData = new BarGraphData(); + Collection levelCounts = new ArrayList(); String name = valueCount.getVal(0) != null ? valueCount.getVal(0).toString().trim() : ""; if (isTypeNumber(typeXAxis)) { - VNameValue nameValue = new VNameValue(); + NameValueData nameValue = new NameValueData(); Double sumValue = (Double) valueCount.getVal(2); String value = "0";// default is zero if (sumValue != null) { @@ -137,7 +139,9 @@ class GraphDataGeneratorBase extends MgrBase { if (innerValueCount != null) { String innerName = innerValueCount.getVal(0) != null ? innerValueCount.getVal(0).toString().trim() : ""; String innerValue = innerValueCount.getVal(1) != null ? innerValueCount.getVal(1).toString().trim() : ""; - VNameValue nameValue = new VNameValue(innerName, innerValue); + NameValueData nameValue = new NameValueData(); + nameValue.setValue(innerValue); + nameValue.setName(innerName); levelCounts.add(nameValue); } } @@ -145,7 +149,7 @@ class GraphDataGeneratorBase extends MgrBase { } } vGraphData.setName(name); - vGraphData.setDataCounts(levelCounts); + vGraphData.setDataCount(levelCounts); vGraphDatas.add(vGraphData); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java new file mode 100644 index 0000000..7affc5a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java @@ -0,0 +1,597 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.manager; + +import javax.inject.Inject; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import org.apache.ambari.logsearch.common.ConfigHelper; +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.ManageStartEndTime; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.conf.SolrAuditLogConfig; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.graph.GraphDataGenerator; +import org.apache.ambari.logsearch.model.response.AuditLogResponse; +import org.apache.ambari.logsearch.model.response.BarGraphData; +import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; +import org.apache.ambari.logsearch.model.response.GroupListResponse; +import org.apache.ambari.logsearch.model.response.LogData; +import org.apache.ambari.logsearch.model.response.LogSearchResponse; +import org.apache.ambari.logsearch.model.response.NameValueData; +import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; +import org.apache.ambari.logsearch.solr.model.SolrAuditLogData; +import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; +import org.apache.ambari.logsearch.util.BizUtil; +import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.ambari.logsearch.view.VResponse; +import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria; +import org.apache.ambari.logsearch.query.model.SearchCriteria; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.springframework.stereotype.Component; + +@Component +public class AuditLogsManager extends ManagerBase { + private static final Logger logger = Logger.getLogger(AuditLogsManager.class); + + @Inject + private AuditSolrDao auditSolrDao; + @Inject + private GraphDataGenerator graphDataGenerator; + @Inject + private SolrAuditLogConfig solrAuditLogConfig; + + public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) { + Boolean isLastPage = (Boolean) searchCriteria.getParamValue("isLastPage"); + if (isLastPage) { + SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + LogSearchResponse logResponse = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery); + if (logResponse == null) { + logResponse = new AuditLogResponse(); + } + return (AuditLogResponse) logResponse; + } + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + return getLogAsPaginationProvided(solrQuery, auditSolrDao); + } + + private List getComponents(SearchCriteria searchCriteria) { + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + List docList = new ArrayList<>(); + try { + SolrUtil.setFacetField(solrQuery, LogSearchConstants.AUDIT_COMPONENT); + SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + List facetFields = null; + List componentsCount = new ArrayList(); + FacetField facetField = null; + + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + return docList; + } + + facetFields = queryResponse.getFacetFields(); + if (facetFields == null) { + return docList; + } + if (!facetFields.isEmpty()) { + facetField = facetFields.get(0); + } + if (facetField != null) { + componentsCount = facetField.getValues(); + } + + for (Count component : componentsCount) { + SolrComponentTypeLogData logData = new SolrComponentTypeLogData(); + logData.setType(component.getName()); + docList.add(logData); + } + return docList; + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } + + public GroupListResponse getAuditComponents(SearchCriteria searchCriteria) { + GroupListResponse componentResponse = new GroupListResponse(); + List docList = getComponents(searchCriteria); + componentResponse.setGroupList(docList); + return componentResponse; + } + + @SuppressWarnings("unchecked") + public BarGraphDataListResponse getAuditBarGraphData(SearchCriteria searchCriteria) { + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + + String from = getFrom((String) searchCriteria.getParamValue("startTime")); + String to = getTo((String) searchCriteria.getParamValue("endTime")); + String unit = getUnit((String) searchCriteria.getParamValue("unit")); + + List histogramData = new ArrayList(); + String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT, + LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); + + try { + SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); + SolrUtil.setRowCount(solrQuery, 0); + QueryResponse response = auditSolrDao.process(solrQuery); + if (response == null) { + return dataList; + } + SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) response.getResponse().get("facets"); + + if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { + return dataList; + } + + extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); + + dataList.setGraphData(histogramData); + return dataList; + + } catch (SolrServerException | SolrException | IOException e) { + logger.error(e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public NameValueDataListResponse getLiveLogCounts() { + NameValueDataListResponse nameValueList = new NameValueDataListResponse(); + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setParam("event", "/audit/logs/live/count"); + try { + Date[] timeRange = ManageStartEndTime.getStartEndTime(); + String startDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[0]); + String endDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[1]); + + SolrUtil.setMainQuery(solrQuery, null); + SolrUtil.setFacetRange(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, "+2MINUTE"); + List listCount; + + QueryResponse response = auditSolrDao.process(solrQuery); + + List rangeFacet = response.getFacetRanges(); + if (rangeFacet == null) { + return nameValueList; + } + RangeFacet range = rangeFacet.get(0); + + if (range == null) { + return nameValueList; + } + + listCount = range.getCounts(); + + List nameValues = new ArrayList<>(); + int count = 0; + for (RangeFacet.Count cnt : listCount) { + NameValueData nameValue = new NameValueData(); + nameValue.setName("" + count); + nameValue.setValue("" + cnt.getCount()); + nameValues.add(nameValue); + count++; + } + nameValueList.setvNameValues(nameValues); + return nameValueList; + + } catch (SolrException | SolrServerException | ParseException + | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } + + public BarGraphDataListResponse topTenUsers(SearchCriteria searchCriteria) { + + String jsonUserQuery = + "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}"; + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); + SolrUtil.setRowCount(solrQuery, 0); + try { + BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + return barGraphDataListResponse; + } + + NamedList namedList = queryResponse.getResponse(); + + if (namedList == null) { + return barGraphDataListResponse; + } + + @SuppressWarnings("unchecked") + SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); + if (jsonFacetResponse == null) { + return barGraphDataListResponse; + } + if (jsonFacetResponse.toString().equals("{count=0}")) { + return barGraphDataListResponse; + } + barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); + return barGraphDataListResponse; + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } + + public BarGraphDataListResponse topTenResources(SearchCriteria searchCriteria) { + + String jsonUserQuery = + "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}"; + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); + SolrUtil.setRowCount(solrQuery, 0); + try { + BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + return barGraphDataListResponse; + } + + NamedList namedList = queryResponse.getResponse(); + if (namedList == null) { + return barGraphDataListResponse; + } + + @SuppressWarnings("unchecked") + SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); + + barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); + return barGraphDataListResponse; + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } + + @SuppressWarnings("unchecked") + public BarGraphDataListResponse getRequestUserLineGraph(SearchCriteria searchCriteria) { + + String from = getFrom((String) searchCriteria.getParamValue("startTime")); + String to = getTo((String) searchCriteria.getParamValue("endTime")); + String unit = getUnit((String) searchCriteria.getParamValue("unit")); + + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + List histogramData = new ArrayList(); + + SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + + String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER, + LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); + + try { + SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); + SolrUtil.setRowCount(solrQuery, 0); + QueryResponse response = auditSolrDao.process(solrQuery); + if (response == null) { + return dataList; + } + SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) response.getResponse().get("facets"); + + if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { + return dataList; + } + extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); + + dataList.setGraphData(histogramData); + return dataList; + + } catch (SolrException | IOException | SolrServerException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + + } + + public String getAuditLogsSchemaFieldsName() { + String excludeArray[] = Arrays.copyOf(solrAuditLogConfig.getExcludeColumnList().toArray(), + solrAuditLogConfig.getExcludeColumnList().size(), String[].class); + List fieldNames = new ArrayList(); + HashMap uiFieldColumnMapping = new HashMap(); + ConfigHelper.getSchemaFieldsName(excludeArray, fieldNames, auditSolrDao); + + for (String fieldName : fieldNames) { + String uiField = solrAuditLogConfig.getSolrAndUiColumns().get(fieldName + LogSearchConstants.SOLR_SUFFIX); + if (uiField == null) { + uiFieldColumnMapping.put(fieldName, fieldName); + } else { + uiFieldColumnMapping.put(fieldName, uiField); + } + } + + uiFieldColumnMapping = BizUtil.sortHashMapByValues(uiFieldColumnMapping); + return convertObjToString(uiFieldColumnMapping); + + } + + public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria) { + searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery); + if (result == null) { + result = new BarGraphDataListResponse(); + } + return result; + + } + + @SuppressWarnings("unchecked") + private void extractValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String outerField, String innerField, + List histogramData) { + NamedList stack = (NamedList) jsonFacetResponse.get(outerField); + ArrayList stackBuckets = (ArrayList) stack.get("buckets"); + for (Object temp : stackBuckets) { + BarGraphData vBarGraphData = new BarGraphData(); + + SimpleOrderedMap level = (SimpleOrderedMap) temp; + String name = ((String) level.getVal(0)).toUpperCase(); + vBarGraphData.setName(name); + + Collection vNameValues = new ArrayList(); + vBarGraphData.setDataCount(vNameValues); + ArrayList levelBuckets = (ArrayList) ((NamedList) level.get(innerField)).get("buckets"); + for (Object temp1 : levelBuckets) { + SimpleOrderedMap countValue = (SimpleOrderedMap) temp1; + String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); + + String count = "" + countValue.getVal(1); + NameValueData vNameValue = new NameValueData(); + vNameValue.setName(value); + vNameValue.setValue(count); + vNameValues.add(vNameValue); + } + histogramData.add(vBarGraphData); + } + } + + @SuppressWarnings({"unchecked"}) + public Response exportUserTableToTextFile(SearchCriteria searchCriteria) { + String jsonUserQuery = + "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}"; + + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + String startTime = (String) searchCriteria.getParamValue("startTime"); + String endTime = (String) searchCriteria.getParamValue("endTime"); + + startTime = startTime == null ? "" : startTime; + endTime = endTime == null ? "" : "_" + endTime; + + SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); + SolrUtil.setRowCount(solrQuery, 0); + + String dataFormat = (String) searchCriteria.getParamValue("format"); + FileOutputStream fis = null; + try { + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + VResponse response = new VResponse(); + response.setMsgDesc("Query was not able to execute " + solrQuery); + throw RESTErrorUtil.createRESTException(response); + } + + NamedList namedList = queryResponse.getResponse(); + if (namedList == null) { + VResponse response = new VResponse(); + response.setMsgDesc("Query was not able to execute " + solrQuery); + throw RESTErrorUtil.createRESTException(response); + } + BarGraphDataListResponse vBarUserDataList = new BarGraphDataListResponse(); + BarGraphDataListResponse vBarResourceDataList = new BarGraphDataListResponse(); + + SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); + vBarUserDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); + vBarResourceDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "y", "x"); + String data = ""; + String summary = ""; + if ("text".equals(dataFormat)) { + int users = 0; + int resources = 0; + summary += "\n\n\n\n"; + data += addBlank("Users") + "Components/Access" + "\n"; + data += "--------------------------------------------------------------------------\n"; + Collection tableUserData = vBarUserDataList.getGraphData(); + for (BarGraphData graphData : tableUserData) { + String userName = graphData.getName(); + String largeUserName = ""; + + if (userName.length() > 45) { + largeUserName = userName.substring(0, 45); + data += addBlank(largeUserName); + } else + data += addBlank(userName); + + Collection vnameValueList = graphData.getDataCount(); + int count = 0; + String blank = ""; + for (NameValueData vNameValue : vnameValueList) { + data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; + if (count == 0) + blank = addBlank(blank); + count++; + + } + while (largeUserName.length() > 0) { + data += largeUserName.substring(0, 45) + "\n"; + } + + users += 1; + } + data += "\n\n\n\n\n\n"; + data += addBlank("Resources") + "Components/Access" + "\n"; + data += "--------------------------------------------------------------------------\n"; + Collection tableResourceData = vBarResourceDataList.getGraphData(); + for (BarGraphData graphData : tableResourceData) { + String resourceName = graphData.getName(); + String largeResourceName = resourceName; + if (largeResourceName.length() > 45) { + resourceName = largeResourceName.substring(0, 45); + largeResourceName = largeResourceName.substring(45, largeResourceName.length()); + } else { + largeResourceName = ""; + } + + //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n"); + data += addBlank(resourceName); + Collection vnameValueList = graphData.getDataCount(); + int count = 0; + String blank = ""; + for (NameValueData vNameValue : vnameValueList) { + data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; + if (count == 0) + blank = addBlank(blank); + count++; + } + String tempLargeResourceName = largeResourceName; + while (largeResourceName.length() > 45) { + largeResourceName = tempLargeResourceName.substring(0, 45); + tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length()); + data += largeResourceName + "\n"; + } + if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) { + data += largeResourceName + "\n"; + } + resources += 1; + } + String header = "--------------------------------SUMMARY-----------------------------------\n"; + summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary; + data = summary + data; + } else { + data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}"; + dataFormat = "json"; + } + String fileName = "Users_Resource" + startTime + endTime + "."; + File file = File.createTempFile(fileName, dataFormat); + + fis = new FileOutputStream(file); + fis.write(data.getBytes()); + return Response + .ok(file, MediaType.APPLICATION_OCTET_STREAM) + .header("Content-Disposition", "attachment;filename=" + fileName + dataFormat) + .build(); + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } finally { + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + } + } + } + } + + private String addBlank(String field) { + int blanks = 50; + int strSize = field.length(); + String fieldWithBlank = field; + for (int i = 0; i < blanks - strSize; i++) { + fieldWithBlank += " "; + } + return fieldWithBlank; + } + + public BarGraphDataListResponse getServiceLoad(SearchCriteria searchCriteria) { + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + Collection vaDatas = new ArrayList(); + dataList.setGraphData(vaDatas); + + SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + + try { + SolrUtil.setFacetField(serivceLoadQuery, LogSearchConstants.AUDIT_COMPONENT); + QueryResponse serviceLoadResponse = auditSolrDao.process(serivceLoadQuery); + if (serviceLoadResponse == null) { + return dataList; + } + FacetField serviceFacetField = serviceLoadResponse.getFacetField(LogSearchConstants.AUDIT_COMPONENT); + if (serviceFacetField == null) { + return dataList; + } + + List serviceLoadFacets = serviceFacetField.getValues(); + if (serviceLoadFacets == null) { + return dataList; + } + for (Count cnt : serviceLoadFacets) { + List valueList = new ArrayList(); + BarGraphData vBarGraphData = new BarGraphData(); + vaDatas.add(vBarGraphData); + NameValueData vNameValue = new NameValueData(); + vNameValue.setName(cnt.getName()); + vBarGraphData.setName(cnt.getName().toUpperCase()); + vNameValue.setValue("" + cnt.getCount()); + valueList.add(vNameValue); + vBarGraphData.setDataCount(valueList); + } + + return dataList; + + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + } + + @Override + protected List convertToSolrBeans(QueryResponse response) { + return response.getBeans(SolrAuditLogData.class); + } + + @Override + protected AuditLogResponse createLogSearchResponse() { + return new AuditLogResponse(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java deleted file mode 100644 index 172ec81..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ambari.logsearch.manager; - -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; - -import org.apache.ambari.logsearch.common.ConfigHelper; -import org.apache.ambari.logsearch.common.LogSearchConstants; -import org.apache.ambari.logsearch.common.ManageStartEndTime; -import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.common.PropertiesHelper; -import org.apache.ambari.logsearch.common.SearchCriteria; -import org.apache.ambari.logsearch.dao.AuditSolrDao; -import org.apache.ambari.logsearch.graph.GraphDataGenerator; -import org.apache.ambari.logsearch.util.BizUtil; -import org.apache.ambari.logsearch.util.DateUtil; -import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; -import org.apache.ambari.logsearch.view.VBarDataList; -import org.apache.ambari.logsearch.view.VBarGraphData; -import org.apache.ambari.logsearch.view.VGroupList; -import org.apache.ambari.logsearch.view.VNameValue; -import org.apache.ambari.logsearch.view.VNameValueList; -import org.apache.ambari.logsearch.view.VResponse; -import org.apache.ambari.logsearch.view.VSolrLogList; -import org.apache.log4j.Logger; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.FacetField.Count; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.RangeFacet; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class AuditMgr extends MgrBase { - private static final Logger logger = Logger.getLogger(AuditMgr.class); - - @Autowired - private AuditSolrDao auditSolrDao; - @Autowired - private GraphDataGenerator graphDataGenerator; - - public String getLogs(SearchCriteria searchCriteria) { - String lastPage = (String) searchCriteria.getParamValue("isLastPage"); - Boolean isLastPage = Boolean.parseBoolean(lastPage); - if (isLastPage) { - SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - VSolrLogList collection = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery); - if(collection == null){ - collection = new VSolrLogList(); - } - return convertObjToString(collection); - } - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - VSolrLogList collection = getLogAsPaginationProvided(solrQuery, auditSolrDao); - return convertObjToString(collection); - - } - - private SolrDocumentList getComponents(SearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrDocumentList docList = new SolrDocumentList(); - try { - SolrUtil.setFacetField(solrQuery, LogSearchConstants.AUDIT_COMPONENT); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - List facetFields = null; - List componentsCount = new ArrayList(); - FacetField facetField = null; - - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return docList; - } - - facetFields = queryResponse.getFacetFields(); - if (facetFields == null) { - return docList; - } - if (!facetFields.isEmpty()) { - facetField = facetFields.get(0); - } - if (facetField != null) { - componentsCount = facetField.getValues(); - } - - for (Count compnonet : componentsCount) { - SolrDocument solrDocument = new SolrDocument(); - solrDocument.addField("type", compnonet.getName()); - docList.add(solrDocument); - } - return docList; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public String getAuditComponents(SearchCriteria searchCriteria) { - VGroupList vGroupList = new VGroupList(); - SolrDocumentList docList = getComponents(searchCriteria); - - vGroupList.setGroupDocuments(docList); - return convertObjToString(vGroupList); - } - - @SuppressWarnings("unchecked") - public String getAuditLineGraphData(SearchCriteria searchCriteria) { - VBarDataList dataList = new VBarDataList(); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - String from = getFrom((String) searchCriteria.getParamValue("startTime")); - String to = getTo((String) searchCriteria.getParamValue("endTime")); - String unit = getUnit((String) searchCriteria.getParamValue("unit")); - - List histogramData = new ArrayList(); - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - try { - SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery, 0); - QueryResponse response = auditSolrDao.process(solrQuery); - if (response == null){ - return convertObjToString(dataList); - } - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) response.getResponse().get("facets"); - - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return convertObjToString(dataList); - } - - extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); - - dataList.setGraphData(histogramData); - return convertObjToString(dataList); - - } catch (SolrServerException | SolrException | IOException e) { - logger.error(e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - - } - } - - public String getTopAuditFieldCount(SearchCriteria searchCriteria) { - int topCounts = 10; - Integer top = (Integer) searchCriteria.getParamValue("top"); - String facetField = (String) searchCriteria.getParamValue("field"); - if (top == null){ - top = new Integer(topCounts); - } - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - try { - - List nameValues = new ArrayList(); - - VNameValueList nameValueList = new VNameValueList(nameValues); - - SolrUtil.setFacetField(solrQuery, facetField); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_COUNT); - SolrUtil.setFacetLimit(solrQuery, top.intValue()); - - List countList = new ArrayList(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return convertObjToString(nameValueList); - } - - if (queryResponse.getFacetField(facetField) != null) { - FacetField queryFacetField = queryResponse.getFacetField(facetField); - if (queryFacetField != null) { - countList = queryFacetField.getValues(); - } - } - - for (Count cnt : countList) { - VNameValue nameValue = new VNameValue(); - nameValue.setName(cnt.getName()); - - nameValue.setValue("" + cnt.getCount()); - nameValues.add(nameValue); - } - return convertObjToString(nameValueList); - - } catch (SolrException | IOException | SolrServerException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - public String getLiveLogCounts() { - VNameValueList nameValueList = new VNameValueList(); - SolrQuery solrQuery = new SolrQuery(); - solrQuery.setParam("event", "/audit/logs/live/count"); - try { - Date[] timeRange = ManageStartEndTime.getStartEndTime(); - String startDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[0]); - String endDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[1]); - - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetRange(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, "+2MINUTE"); - List listCount; - - QueryResponse response = auditSolrDao.process(solrQuery); - - List rangeFacet = response.getFacetRanges(); - if (rangeFacet == null){ - return convertObjToString(nameValueList); - } - RangeFacet range=rangeFacet.get(0); - - if(range == null){ - return convertObjToString(nameValueList); - } - - listCount = range.getCounts(); - - List nameValues = new ArrayList(); - int count = 0; - for (RangeFacet.Count cnt : listCount) { - VNameValue nameValue = new VNameValue(); - nameValue.setName("" + count); - nameValue.setValue("" + cnt.getCount()); - nameValues.add(nameValue); - count++; - } - nameValueList.setVNameValues(nameValues); - return convertObjToString(nameValueList); - - } catch (SolrException | SolrServerException | ParseException - | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public String topTenUsers(SearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - VBarDataList vBarDataList = new VBarDataList(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return convertObjToString(vBarDataList); - } - - NamedList namedList = queryResponse.getResponse(); - - if (namedList == null) { - return convertObjToString(vBarDataList); - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - if (jsonFacetResponse == null) { - return convertObjToString(vBarDataList); - } - if (jsonFacetResponse.toString().equals("{count=0}")) { - return convertObjToString(vBarDataList); - } - vBarDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse,"Repo","Users"); - return convertObjToString(vBarDataList); - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public String topTenResources(SearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - VBarDataList vBarDataList = new VBarDataList(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if(queryResponse == null){ - return convertObjToString(vBarDataList); - } - - NamedList namedList = queryResponse.getResponse(); - if (namedList == null) { - return convertObjToString(vBarDataList); - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - - vBarDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse,"Repo","Users"); - return convertObjToString(vBarDataList); - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - @SuppressWarnings("unchecked") - public String getRequestUserLineGraph(SearchCriteria searchCriteria) { - - String from = getFrom((String) searchCriteria.getParamValue("startTime")); - String to = getTo((String) searchCriteria.getParamValue("endTime")); - String unit = getUnit((String) searchCriteria.getParamValue("unit")); - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - VBarDataList dataList = new VBarDataList(); - List histogramData = new ArrayList(); - - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - try { - SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery, 0); - QueryResponse response = auditSolrDao.process(solrQuery); - if (response == null){ - return convertObjToString(dataList); - } - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) response.getResponse().get("facets"); - - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return convertObjToString(dataList); - } - extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); - - dataList.setGraphData(histogramData); - return convertObjToString(dataList); - - } catch (SolrException | IOException | SolrServerException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - - } - - public String getAuditLogsSchemaFieldsName() { - String excludeArray[] = PropertiesHelper.getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist"); - List fieldNames = new ArrayList(); - HashMap uiFieldColumnMapping = new HashMap(); - ConfigHelper.getSchemaFieldsName(excludeArray, fieldNames,auditSolrDao); - - for (String fieldName : fieldNames) { - String uiField = ConfigHelper.auditLogsColumnMapping.get(fieldName + LogSearchConstants.SOLR_SUFFIX); - if (uiField == null) { - uiFieldColumnMapping.put(fieldName, fieldName); - } else { - uiFieldColumnMapping.put(fieldName, uiField); - } - } - - uiFieldColumnMapping = BizUtil.sortHashMapByValues(uiFieldColumnMapping); - return convertObjToString(uiFieldColumnMapping); - - } - - public String getAnyGraphData(SearchCriteria searchCriteria) { - searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery); - if (result == null) { - result = new VBarDataList(); - } - return convertObjToString(result); - - } - - @SuppressWarnings("unchecked") - private void extractValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String outerField, String innerField, - List histogramData) { - NamedList stack = (NamedList) jsonFacetResponse.get(outerField); - ArrayList stackBuckets = (ArrayList) stack.get("buckets"); - for (Object temp : stackBuckets) { - VBarGraphData vBarGraphData = new VBarGraphData(); - - SimpleOrderedMap level = (SimpleOrderedMap) temp; - String name = ((String) level.getVal(0)).toUpperCase(); - vBarGraphData.setName(name); - - Collection vNameValues = new ArrayList(); - vBarGraphData.setDataCounts(vNameValues); - ArrayList levelBuckets = (ArrayList) ((NamedList) level.get(innerField)).get("buckets"); - for (Object temp1 : levelBuckets) { - SimpleOrderedMap countValue = (SimpleOrderedMap) temp1; - String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); - - String count = "" + countValue.getVal(1); - VNameValue vNameValue = new VNameValue(); - vNameValue.setName(value); - vNameValue.setValue(count); - vNameValues.add(vNameValue); - } - histogramData.add(vBarGraphData); - } - } - - @SuppressWarnings({"unchecked"}) - public Response exportUserTableToTextFile(SearchCriteria searchCriteria) { - String jsonUserQuery = - "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}"; - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - String startTime = (String) searchCriteria.getParamValue("startTime"); - String endTime = (String) searchCriteria.getParamValue("endTime"); - - startTime = startTime == null ? "" : startTime; - endTime = endTime == null ? "" : "_" + endTime; - - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - - String dataFormat = (String) searchCriteria.getParamValue("format"); - FileOutputStream fis = null; - try { - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if(queryResponse == null){ - VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute "+solrQuery); - throw RESTErrorUtil.createRESTException(response); - } - - NamedList namedList = queryResponse.getResponse(); - if (namedList == null) { - VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute "+solrQuery); - throw RESTErrorUtil.createRESTException(response); - } - VBarDataList vBarUserDataList = new VBarDataList(); - VBarDataList vBarResourceDataList = new VBarDataList(); - - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - vBarUserDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse,"Repo","Users"); - vBarResourceDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse,"y","x"); - String data = ""; - String summary = ""; - if ("text".equals(dataFormat)) { - int users = 0; - int resources = 0; - summary += "\n\n\n\n"; - data += addBlank("Users") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection tableUserData = vBarUserDataList.getGraphData(); - for (VBarGraphData graphData : tableUserData) { - String userName = graphData.getName(); - String largeUserName = ""; - - if (userName.length() > 45) { - largeUserName = userName.substring(0, 45); - data += addBlank(largeUserName); - } else - data += addBlank(userName); - - Collection vnameValueList = graphData.getDataCount(); - int count = 0; - String blank = ""; - for (VNameValue vNameValue : vnameValueList) { - data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; - if (count == 0) - blank = addBlank(blank); - count++; - - } - while (largeUserName.length() > 0) { - data += largeUserName.substring(0, 45) + "\n"; - } - - users += 1; - } - data += "\n\n\n\n\n\n"; - data += addBlank("Resources") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection tableResourceData = vBarResourceDataList.getGraphData(); - for (VBarGraphData graphData : tableResourceData) { - String resourceName = graphData.getName(); - String largeResourceName = resourceName; - if (largeResourceName.length() > 45) { - resourceName = largeResourceName.substring(0, 45); - largeResourceName = largeResourceName.substring(45, largeResourceName.length()); - } else { - largeResourceName = ""; - } - - //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n"); - data += addBlank(resourceName); - Collection vnameValueList = graphData.getDataCount(); - int count = 0; - String blank = ""; - for (VNameValue vNameValue : vnameValueList) { - data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; - if (count == 0) - blank = addBlank(blank); - count++; - } - String tempLargeResourceName = largeResourceName; - while (largeResourceName.length() > 45) { - largeResourceName = tempLargeResourceName.substring(0, 45); - tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length()); - data += largeResourceName + "\n"; - } - if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) { - data += largeResourceName + "\n"; - } - resources += 1; - } - String header = "--------------------------------SUMMARY-----------------------------------\n"; - summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary; - data = summary + data; - } else { - data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}"; - dataFormat = "json"; - } - String fileName = "Users_Resource" + startTime + endTime + "."; - File file = File.createTempFile(fileName, dataFormat); - - fis = new FileOutputStream(file); - fis.write(data.getBytes()); - return Response - .ok(file, MediaType.APPLICATION_OCTET_STREAM) - .header("Content-Disposition", "attachment;filename=" + fileName + dataFormat) - .build(); - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } finally { - if (fis != null) { - try { - fis.close(); - } catch (IOException e) { - } - } - } - } - - private String addBlank(String field) { - int blanks = 50; - int strSize = field.length(); - String fieldWithBlank = field; - for (int i = 0; i < blanks - strSize; i++) { - fieldWithBlank += " "; - } - return fieldWithBlank; - } - - public String getServiceLoad(SearchCriteria searchCriteria) { - VBarDataList dataList = new VBarDataList(); - Collection vaDatas = new ArrayList(); - dataList.setGraphData(vaDatas); - - SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - try { - SolrUtil.setFacetField(serivceLoadQuery, LogSearchConstants.AUDIT_COMPONENT); - QueryResponse serviceLoadResponse = auditSolrDao.process(serivceLoadQuery); - if (serviceLoadResponse == null){ - return convertObjToString(dataList); - } - FacetField serviceFacetField =serviceLoadResponse.getFacetField(LogSearchConstants.AUDIT_COMPONENT); - if (serviceFacetField == null) { - return convertObjToString(dataList); - } - - List serviceLoadFacets = serviceFacetField.getValues(); - if (serviceLoadFacets == null) { - return convertObjToString(dataList); - } - for (Count cnt : serviceLoadFacets) { - List valueList = new ArrayList(); - VBarGraphData vBarGraphData = new VBarGraphData(); - vaDatas.add(vBarGraphData); - VNameValue vNameValue = new VNameValue(); - vNameValue.setName(cnt.getName()); - vBarGraphData.setName(cnt.getName().toUpperCase()); - vNameValue.setValue("" + cnt.getCount()); - valueList.add(vNameValue); - vBarGraphData.setDataCounts(valueList); - } - - return convertObjToString(dataList); - - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/JsonManagerBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/JsonManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/JsonManagerBase.java new file mode 100644 index 0000000..94191e0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/JsonManagerBase.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.manager; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import java.util.Date; + +public class JsonManagerBase { + + private JsonSerializer jsonDateSerialiazer = null; + private JsonDeserializer jsonDateDeserialiazer = null; + + public JsonManagerBase() { + jsonDateSerialiazer = new JsonSerializer() { + + @Override + public JsonElement serialize(Date paramT, java.lang.reflect.Type paramType, JsonSerializationContext paramJsonSerializationContext) { + return paramT == null ? null : new JsonPrimitive(paramT.getTime()); + } + }; + + jsonDateDeserialiazer = new JsonDeserializer() { + + @Override + public Date deserialize(JsonElement json, java.lang.reflect.Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + return json == null ? null : new Date(json.getAsLong()); + } + + }; + } + + protected String convertObjToString(Object obj) { + if (obj == null) { + return ""; + } + + Gson gson = new GsonBuilder() + .registerTypeAdapter(Date.class, jsonDateSerialiazer) + .registerTypeAdapter(Date.class, jsonDateDeserialiazer).create(); + + return gson.toJson(obj); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/cd08bde6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java new file mode 100644 index 0000000..405eaef --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.manager; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.model.response.LogFileData; +import org.apache.ambari.logsearch.model.response.LogFileDataListResponse; +import org.apache.ambari.logsearch.model.response.LogListResponse; +import org.apache.ambari.logsearch.model.response.ServiceLogData; +import org.apache.ambari.logsearch.model.response.ServiceLogResponse; +import org.apache.ambari.logsearch.query.model.SearchCriteria; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; +import org.apache.ambari.logsearch.dao.SolrDaoBase; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrException; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; + + +@Component +public class LogFileManager extends ManagerBase { + + private static final Logger logger = Logger.getLogger(LogFileManager.class); + + @Inject + private ServiceLogsSolrDao serviceLogsSolrDao; + @Inject + private AuditSolrDao auditSolrDao; + + public LogFileDataListResponse searchLogFiles(SearchCriteria searchCriteria) { + LogFileDataListResponse logFileList = new LogFileDataListResponse(); + List logFiles = new ArrayList(); + String componentName = (String) searchCriteria.getParamValue("component"); + String host = (String) searchCriteria.getParamValue("host"); + int minCount = 1;// to remove zero count facet + SolrQuery solrQuery = new SolrQuery(); + SolrUtil.setMainQuery(solrQuery, null); + SolrUtil.setFacetFieldWithMincount(solrQuery, LogSearchConstants.SOLR_PATH, minCount); + // adding filter + queryGenerator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_COMPONENT, componentName); + queryGenerator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_HOST, host); + try { + String logType = (String) searchCriteria.getParamValue("logType"); + if (StringUtils.isBlank(logType)) { + logType = LogType.SERVICE.name();// default is service Log + } + SolrDaoBase daoMgr = null; + if (logType.equalsIgnoreCase(LogType.SERVICE.name())) { + daoMgr = serviceLogsSolrDao; + } else if (logType.equalsIgnoreCase(LogType.AUDIT.name())) { + daoMgr = auditSolrDao; + } else { + throw RESTErrorUtil.createRESTException(logType + " is not a valid logType", MessageEnums.INVALID_INPUT_DATA); + } + QueryResponse queryResponse = daoMgr.process(solrQuery); + if (queryResponse.getFacetField(LogSearchConstants.SOLR_PATH) != null) { + FacetField queryFacetField = queryResponse.getFacetField(LogSearchConstants.SOLR_PATH); + if (queryFacetField != null) { + List countList = queryFacetField.getValues(); + for (Count count : countList) { + LogFileData vLogFile = new LogFileData(); + String filePath = count.getName(); + String fileName = FilenameUtils.getName(filePath); + vLogFile.setPath(filePath); + vLogFile.setName(fileName); + logFiles.add(vLogFile); + } + } + } + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error in solr query :" + e.getLocalizedMessage() + "\n Query :" + solrQuery.toQueryString(), e.getCause()); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + logFileList.setLogFiles(logFiles); + + return logFileList; + } + + public LogListResponse getLogFileTail(SearchCriteria searchCriteria) { + String host = (String) searchCriteria.getParamValue("host"); + String logFile = (String) searchCriteria.getParamValue("name"); + String component = (String) searchCriteria.getParamValue("component"); + String tailSize = (String) searchCriteria.getParamValue("tailSize"); + if (StringUtils.isBlank(host)) { + throw RESTErrorUtil.createRESTException("missing Host Name", MessageEnums.ERROR_SYSTEM); + } + tailSize = (StringUtils.isBlank(tailSize)) ? "10" : tailSize; + SolrQuery logFileTailQuery = new SolrQuery(); + try { + int tail = Integer.parseInt(tailSize); + tail = tail > 100 ? 100 : tail; + SolrUtil.setMainQuery(logFileTailQuery, null); + queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_HOST, host); + if (!StringUtils.isBlank(logFile)) { + queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_PATH, SolrUtil.makeSolrSearchString(logFile)); + } else if (!StringUtils.isBlank(component)) { + queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_COMPONENT, component); + } else { + throw RESTErrorUtil.createRESTException("component or logfile parameter must be present", MessageEnums.ERROR_SYSTEM); + } + + SolrUtil.setRowCount(logFileTailQuery, tail); + queryGenerator.setSortOrderDefaultServiceLog(logFileTailQuery, new SearchCriteria()); + return getLogAsPaginationProvided(logFileTailQuery, serviceLogsSolrDao); + + } catch (NumberFormatException ne) { + + throw RESTErrorUtil.createRESTException(ne.getMessage(), + MessageEnums.ERROR_SYSTEM); + + } + } + + @Override + protected List convertToSolrBeans(QueryResponse response) { + throw new UnsupportedOperationException(); + } + + @Override + protected ServiceLogResponse createLogSearchResponse() { + throw new UnsupportedOperationException(); + } +}