Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9D7B3200B95 for ; Tue, 27 Sep 2016 21:54:44 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 9BE56160AB9; Tue, 27 Sep 2016 19:54:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EB33F160AE8 for ; Tue, 27 Sep 2016 21:54:42 +0200 (CEST) Received: (qmail 12994 invoked by uid 500); 27 Sep 2016 19:54:42 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 12689 invoked by uid 99); 27 Sep 2016 19:54:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 27 Sep 2016 19:54:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6318AE1075; Tue, 27 Sep 2016 19:54:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Date: Tue, 27 Sep 2016 19:54:46 -0000 Message-Id: <46074294720443b49ebb25e5d787a881@git.apache.org> In-Reply-To: <67bb6d422aa1484b89fc0c400669cf04@git.apache.org> References: <67bb6d422aa1484b89fc0c400669cf04@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [6/7] ambari git commit: AMBARI-18310. Logsearch - Refactor solr query layer (oleewere) archived-at: Tue, 27 Sep 2016 19:54:44 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java deleted file mode 100644 index c57e0e9..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.graph; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; - -import org.apache.ambari.logsearch.manager.ManagerBase; -import org.apache.ambari.logsearch.model.response.BarGraphData; -import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.util.DateUtil; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; - -class GraphDataGeneratorBase { - - private static final String BUCKETS = "buckets"; - - private static enum DataType { - LONG("long"), - DOUBLE("double"), - FLOAT("long"), - INT("long"); - - private String type; - - DataType(String type) { - this.type = type; - } - - String getType() { - return type; - } - } - - protected static enum GraphType { - UNKNOWN, - NORMAL_GRAPH, - RANGE_NON_STACK_GRAPH, - NON_RANGE_STACK_GRAPH, - RANGE_STACK_GRAPH; - } - - @SuppressWarnings("unchecked") - protected void extractRangeStackValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String outerField, - String innerField, List histogramData) { - if (jsonFacetResponse != null) { - NamedList stack = (NamedList) jsonFacetResponse.get(outerField); - if (stack != null) { - ArrayList stackBuckets = (ArrayList) stack.get(BUCKETS); - if (stackBuckets != null) { - for (Object stackBucket : stackBuckets) { - BarGraphData vBarGraphData = new BarGraphData(); - SimpleOrderedMap level = (SimpleOrderedMap) stackBucket; - if (level != null) { - String name = level.getVal(0) != null ? level.getVal(0).toString().toUpperCase() : ""; - vBarGraphData.setName(name); - Collection vNameValues = new ArrayList(); - NamedList innerFiledValue = (NamedList) level.get(innerField); - if (innerFiledValue != null) { - ArrayList levelBuckets = (ArrayList) innerFiledValue.get(BUCKETS); - if (levelBuckets != null) { - for (Object levelBucket : levelBuckets) { - SimpleOrderedMap countValue = (SimpleOrderedMap) levelBucket; - if (countValue != null) { - String innerName = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); - String innerValue = countValue.getVal(1) != null ? countValue.getVal(1).toString() : ""; - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(innerName); - vNameValue.setValue(innerValue); - vNameValues.add(vNameValue); - } - } - } - } - vBarGraphData.setDataCount(vNameValues); - } - histogramData.add(vBarGraphData); - } - } - } - } - } - - @SuppressWarnings("unchecked") - protected boolean extractNonRangeStackValuesFromBucket(SimpleOrderedMap jsonFacetResponse, String level, - Collection vGraphDatas, String typeXAxis) { - boolean zeroFlag = true; - if (jsonFacetResponse == null || jsonFacetResponse.get(level) == null - || jsonFacetResponse.get(level).toString().equals("{count=0}")) { - return false; - } - NamedList levelList = (NamedList) jsonFacetResponse.get(level); - if (levelList != null) { - ArrayList bucketList = (ArrayList) levelList.get(BUCKETS); - if (bucketList != null) { - for (int index = 0; index < bucketList.size(); index++) { - SimpleOrderedMap valueCount = (SimpleOrderedMap) bucketList.get(index); - if (valueCount != null && valueCount.size() > 2) { - BarGraphData vGraphData = new BarGraphData(); - Collection levelCounts = new ArrayList(); - String name = valueCount.getVal(0) != null ? valueCount.getVal(0).toString().trim() : ""; - if (isTypeNumber(typeXAxis)) { - NameValueData nameValue = new NameValueData(); - Double sumValue = (Double) valueCount.getVal(2); - String value = "0";// default is zero - if (sumValue != null) { - value = "" + sumValue.longValue(); - } - nameValue.setName(name); - nameValue.setValue(value); - levelCounts.add(nameValue); - } else { - SimpleOrderedMap valueCountMap = (SimpleOrderedMap) valueCount.getVal(2); - if (valueCountMap != null) { - ArrayList buckets = (ArrayList) valueCountMap.get(BUCKETS); - if (buckets != null) { - for (int innerIndex = 0; innerIndex < buckets.size(); innerIndex++) { - SimpleOrderedMap innerValueCount = (SimpleOrderedMap) buckets.get(innerIndex); - if (innerValueCount != null) { - String innerName = innerValueCount.getVal(0) != null ? innerValueCount.getVal(0).toString().trim() : ""; - String innerValue = innerValueCount.getVal(1) != null ? innerValueCount.getVal(1).toString().trim() : ""; - NameValueData nameValue = new NameValueData(); - nameValue.setValue(innerValue); - nameValue.setName(innerName); - levelCounts.add(nameValue); - } - } - } - } - } - vGraphData.setName(name); - vGraphData.setDataCount(levelCounts); - vGraphDatas.add(vGraphData); - } - } - } - } - return zeroFlag; - } - - protected boolean isTypeNumber(String typeXAxis) { - if (StringUtils.isBlank(typeXAxis)) { - return false; - } else { - return typeXAxis.contains(DataType.LONG.getType()) || typeXAxis.contains(DataType.INT.getType()) - || typeXAxis.contains(DataType.FLOAT.getType()) || typeXAxis.contains(DataType.DOUBLE.getType()); - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java index 52482c2..56ba2d7 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java @@ -20,398 +20,151 @@ package org.apache.ambari.logsearch.manager; import javax.inject.Inject; +import javax.inject.Named; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.text.ParseException; +import java.io.StringWriter; import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; -import org.apache.ambari.logsearch.common.LogSearchConstants; -import org.apache.ambari.logsearch.common.ManageStartEndTime; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; import org.apache.ambari.logsearch.dao.AuditSolrDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; +import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest; +import org.apache.ambari.logsearch.model.request.impl.FieldAuditLogRequest; +import org.apache.ambari.logsearch.model.request.impl.UserExportRequest; import org.apache.ambari.logsearch.model.response.AuditLogResponse; import org.apache.ambari.logsearch.model.response.BarGraphData; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; import org.apache.ambari.logsearch.model.response.GroupListResponse; import org.apache.ambari.logsearch.model.response.LogData; -import org.apache.ambari.logsearch.model.response.LogSearchResponse; import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; -import org.apache.ambari.logsearch.query.model.AnyGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.AuditBarGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.CommonSearchCriteria; -import org.apache.ambari.logsearch.query.model.FieldAuditBarGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.FieldAuditLogSearchCriteria; -import org.apache.ambari.logsearch.query.model.UserExportSearchCriteria; +import org.apache.ambari.logsearch.model.response.TemplateData; +import org.apache.ambari.logsearch.solr.SolrConstants; import org.apache.ambari.logsearch.solr.model.SolrAuditLogData; import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; -import org.apache.ambari.logsearch.util.BizUtil; -import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.util.DownloadUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.ambari.logsearch.view.VResponse; -import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.RangeFacet; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.stereotype.Component; +import org.springframework.core.convert.ConversionService; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.SimpleQuery; -@Component +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT; + +@Named public class AuditLogsManager extends ManagerBase { private static final Logger logger = Logger.getLogger(AuditLogsManager.class); + private static final String AUDIT_LOG_TEMPLATE = "audit_log_txt.ftl"; + @Inject private AuditSolrDao auditSolrDao; @Inject private GraphDataGenerator graphDataGenerator; @Inject - private SolrAuditLogPropsConfig solrAuditLogPropsConfig; + private ConversionService conversionService; + @Inject + private Configuration freemarkerConfiguration; - public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) { - Boolean isLastPage = searchCriteria.isLastPage(); - if (isLastPage) { - SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - LogSearchResponse logResponse = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery); - if (logResponse == null) { - logResponse = new AuditLogResponse(); - } - return (AuditLogResponse) logResponse; - } - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - return getLogAsPaginationProvided(solrQuery, auditSolrDao); + public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) { + return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs"); } - private List getComponents(CommonSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + private List getComponents(AuditComponentRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); List docList = new ArrayList<>(); - try { - SolrUtil.setFacetField(solrQuery, LogSearchConstants.AUDIT_COMPONENT); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - List facetFields = null; - List componentsCount = new ArrayList(); - FacetField facetField = null; + QueryResponse queryResponse = auditSolrDao.process(facetQuery); + List componentsCount = graphDataGenerator.generateCount(queryResponse); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return docList; - } - - facetFields = queryResponse.getFacetFields(); - if (facetFields == null) { - return docList; - } - if (!facetFields.isEmpty()) { - facetField = facetFields.get(0); - } - if (facetField != null) { - componentsCount = facetField.getValues(); - } - - for (Count component : componentsCount) { - SolrComponentTypeLogData logData = new SolrComponentTypeLogData(); - logData.setType(component.getName()); - docList.add(logData); - } - return docList; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + for (Count component : componentsCount) { + SolrComponentTypeLogData logData = new SolrComponentTypeLogData(); + logData.setType(component.getName()); + docList.add(logData); } + return docList; } - public GroupListResponse getAuditComponents(CommonSearchCriteria searchCriteria) { + public GroupListResponse getAuditComponents(AuditComponentRequest request) { GroupListResponse componentResponse = new GroupListResponse(); - List docList = getComponents(searchCriteria); + List docList = getComponents(request); componentResponse.setGroupList(docList); return componentResponse; } - @SuppressWarnings("unchecked") - public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphSearchCriteria searchCriteria) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - String from = getFrom(searchCriteria.getStartTime()); - String to = getTo(searchCriteria.getEndTime()); - String unit = getUnit(searchCriteria.getUnit()); - - List histogramData = new ArrayList(); - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public NameValueDataListResponse getLiveLogCounts() { - NameValueDataListResponse nameValueList = new NameValueDataListResponse(); - SolrQuery solrQuery = new SolrQuery(); - solrQuery.setParam("event", "/audit/logs/live/count"); - try { - Date[] timeRange = ManageStartEndTime.getStartEndTime(); - String startDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[0]); - String endDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[1]); - - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetRange(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, "+2MINUTE"); - List listCount; - - QueryResponse response = auditSolrDao.process(solrQuery); - - List rangeFacet = response.getFacetRanges(); - if (rangeFacet == null) { - return nameValueList; - } - RangeFacet range = rangeFacet.get(0); - - if (range == null) { - return nameValueList; - } - - listCount = range.getCounts(); - - List nameValues = new ArrayList<>(); - int count = 0; - for (RangeFacet.Count cnt : listCount) { - NameValueData nameValue = new NameValueData(); - nameValue.setName("" + count); - nameValue.setValue("" + cnt.getCount()); - nameValues.add(nameValue); - count++; - } - nameValueList.setvNameValues(nameValues); - return nameValueList; - - } catch (SolrException | SolrServerException | ParseException - | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public BarGraphDataListResponse topTenUsers(FieldAuditBarGraphSearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return barGraphDataListResponse; - } - - NamedList namedList = queryResponse.getResponse(); - - if (namedList == null) { - return barGraphDataListResponse; - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - if (jsonFacetResponse == null) { - return barGraphDataListResponse; - } - if (jsonFacetResponse.toString().equals("{count=0}")) { - return barGraphDataListResponse; - } - barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - return barGraphDataListResponse; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public BarGraphDataListResponse topTenResources(FieldAuditLogSearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return barGraphDataListResponse; - } - - NamedList namedList = queryResponse.getResponse(); - if (namedList == null) { - return barGraphDataListResponse; - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - - barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - return barGraphDataListResponse; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } + public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphRequest request) { + SolrQuery solrQuery = conversionService.convert(request, SolrQuery.class); + QueryResponse response = auditSolrDao.process(solrQuery); + return graphDataGenerator.generateBarGraphDataResponseWithRanges(response, SolrConstants.AuditLogConstants.AUDIT_COMPONENT, true); } - @SuppressWarnings("unchecked") - public BarGraphDataListResponse getRequestUserLineGraph(FieldAuditBarGraphSearchCriteria searchCriteria) { - - String from = getFrom(searchCriteria.getStartTime()); - String to = getTo(searchCriteria.getEndTime()); - String unit = getUnit(searchCriteria.getUnit()); - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - List histogramData = new ArrayList(); - - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery); + public BarGraphDataListResponse topResources(FieldAuditLogRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse queryResponse = auditSolrDao.process(facetQuery); + return graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0); } public String getAuditLogsSchemaFieldsName() { - return convertObjToString(auditSolrDao.schemaFieldNameMap); + return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap()); } - public BarGraphDataListResponse getAnyGraphData(AnyGraphSearchCriteria searchCriteria) { - searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery); - if (result == null) { - result = new BarGraphDataListResponse(); - } - return result; - + public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse response = auditSolrDao.process(facetQuery); + return graphDataGenerator.generateBarGraphFromFieldFacet(response, AUDIT_COMPONENT); } - @SuppressWarnings({"unchecked"}) - public Response exportUserTableToTextFile(UserExportSearchCriteria searchCriteria) { - String jsonUserQuery = - "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}"; - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - String startTime = searchCriteria.getStartTime(); - String endTime = searchCriteria.getEndTime(); + public Response export(UserExportRequest request) throws TemplateException { + String startTime = request.getFrom(); + String endTime = request.getTo(); + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); startTime = startTime == null ? "" : startTime; endTime = endTime == null ? "" : "_" + endTime; - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - - String dataFormat = searchCriteria.getFormat(); + String dataFormat = request.getFormat(); FileOutputStream fis = null; try { - QueryResponse queryResponse = auditSolrDao.process(solrQuery); + QueryResponse queryResponse = auditSolrDao.process(facetQuery); if (queryResponse == null) { VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute " + solrQuery); + response.setMsgDesc("Query was not able to execute " + facetQuery); throw RESTErrorUtil.createRESTException(response); } - - NamedList namedList = queryResponse.getResponse(); - if (namedList == null) { - VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute " + solrQuery); - throw RESTErrorUtil.createRESTException(response); - } - BarGraphDataListResponse vBarUserDataList = new BarGraphDataListResponse(); - BarGraphDataListResponse vBarResourceDataList = new BarGraphDataListResponse(); - - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) namedList.get("facets"); - vBarUserDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - vBarResourceDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "y", "x"); + BarGraphDataListResponse vBarUserDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0); + BarGraphDataListResponse vBarResourceDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 1); String data = ""; - String summary = ""; if ("text".equals(dataFormat)) { - int users = 0; - int resources = 0; - summary += "\n\n\n\n"; - data += addBlank("Users") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection tableUserData = vBarUserDataList.getGraphData(); - for (BarGraphData graphData : tableUserData) { - String userName = graphData.getName(); - String largeUserName = ""; - - if (userName.length() > 45) { - largeUserName = userName.substring(0, 45); - data += addBlank(largeUserName); - } else - data += addBlank(userName); - - Collection vnameValueList = graphData.getDataCount(); - data = appendNameValueData(data, vnameValueList); - while (largeUserName.length() > 0) { - data += largeUserName.substring(0, 45) + "\n"; - } - - users += 1; - } - data += "\n\n\n\n\n\n"; - data += addBlank("Resources") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection tableResourceData = vBarResourceDataList.getGraphData(); - for (BarGraphData graphData : tableResourceData) { - String resourceName = graphData.getName(); - String largeResourceName = resourceName; - if (largeResourceName.length() > 45) { - resourceName = largeResourceName.substring(0, 45); - largeResourceName = largeResourceName.substring(45, largeResourceName.length()); - } else { - largeResourceName = ""; - } + StringWriter stringWriter = new StringWriter(); + Template template = freemarkerConfiguration.getTemplate(AUDIT_LOG_TEMPLATE); + Map models = new HashMap<>(); + DownloadUtil.fillUserResourcesModel(models, vBarUserDataList, vBarResourceDataList); + template.process(models, stringWriter); + data = stringWriter.toString(); - //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n"); - data += addBlank(resourceName); - Collection vnameValueList = graphData.getDataCount(); - data = appendNameValueData(data, vnameValueList); - String tempLargeResourceName = largeResourceName; - while (largeResourceName.length() > 45) { - largeResourceName = tempLargeResourceName.substring(0, 45); - tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length()); - data += largeResourceName + "\n"; - } - if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) { - data += largeResourceName + "\n"; - } - resources += 1; - } - String header = "--------------------------------SUMMARY-----------------------------------\n"; - summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary; - data = summary + data; } else { data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}"; dataFormat = "json"; } String fileName = "Users_Resource" + startTime + endTime + "."; File file = File.createTempFile(fileName, dataFormat); - fis = new FileOutputStream(file); fis.write(data.getBytes()); return Response @@ -419,8 +172,8 @@ public class AuditLogsManager extends ManagerBase histogramData, String jsonHistogramQuery) { - try { - SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery, 0); - QueryResponse response = auditSolrDao.process(solrQuery); - if (response == null) { - return dataList; - } - SimpleOrderedMap jsonFacetResponse = (SimpleOrderedMap) response.getResponse().get("facets"); - - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return dataList; - } - - auditSolrDao.extractValuesFromBuckets(jsonFacetResponse, "x", "y", histogramData); - - dataList.setGraphData(histogramData); - return dataList; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error(e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - - } - } - - private String appendNameValueData(String data, Collection vnameValueList) { - int count = 0; - String blank = ""; - for (NameValueData vNameValue : vnameValueList) { - data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; - if (count == 0) - blank = addBlank(blank); - count++; - } - return data; - } - - private String addBlank(String field) { - int blanks = 50; - int strSize = field.length(); - String fieldWithBlank = field; - for (int i = 0; i < blanks - strSize; i++) { - fieldWithBlank += " "; - } - return fieldWithBlank; - } - - public BarGraphDataListResponse getServiceLoad(CommonSearchCriteria searchCriteria) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - Collection vaDatas = new ArrayList(); - dataList.setGraphData(vaDatas); - - SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - try { - SolrUtil.setFacetField(serivceLoadQuery, LogSearchConstants.AUDIT_COMPONENT); - QueryResponse serviceLoadResponse = auditSolrDao.process(serivceLoadQuery); - if (serviceLoadResponse == null) { - return dataList; - } - FacetField serviceFacetField = serviceLoadResponse.getFacetField(LogSearchConstants.AUDIT_COMPONENT); - if (serviceFacetField == null) { - return dataList; - } - - List serviceLoadFacets = serviceFacetField.getValues(); - if (serviceLoadFacets == null) { - return dataList; - } - for (Count cnt : serviceLoadFacets) { - List valueList = new ArrayList(); - BarGraphData vBarGraphData = new BarGraphData(); - vaDatas.add(vBarGraphData); - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(cnt.getName()); - vBarGraphData.setName(cnt.getName().toUpperCase()); - vNameValue.setValue("" + cnt.getCount()); - valueList.add(vNameValue); - vBarGraphData.setDataCount(valueList); - } - - return dataList; - - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - @Override protected List convertToSolrBeans(QueryResponse response) { return response.getBeans(SolrAuditLogData.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java index 13df470..a7b9cb5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java @@ -28,17 +28,13 @@ import java.util.Date; import java.util.List; import java.util.Scanner; -import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.model.response.LogData; import org.apache.ambari.logsearch.model.response.LogSearchResponse; -import org.apache.ambari.logsearch.query.model.SearchCriteria; import org.apache.ambari.logsearch.dao.SolrDaoBase; -import org.apache.ambari.logsearch.query.QueryGeneration; import org.apache.ambari.logsearch.util.DateUtil; import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; @@ -46,30 +42,13 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; - -import javax.inject.Inject; +import org.springframework.data.solr.core.DefaultQueryParser; +import org.springframework.data.solr.core.query.SimpleQuery; +import org.springframework.data.solr.core.query.SolrDataQuery; public abstract class ManagerBase extends JsonManagerBase { private static final Logger logger = Logger.getLogger(ManagerBase.class); - @Inject - protected QueryGeneration queryGenerator; - - public enum LogType { - SERVICE("Service"), - AUDIT("Audit"); - - private String label; - - private LogType(String label) { - this.label = label; - } - - public String getLabel() { - return label; - } - } - public ManagerBase() { super(); } @@ -103,77 +82,52 @@ public abstract class ManagerBase docList = logResponse.getLogList(); - List lastPageDocList = new ArrayList<>(); - logResponse.setLogList(lastPageDocList); - int cnt = 0; - for(LOG_DATA_TYPE doc:docList){ - if(cnt docList = logResponse.getLogList(); + List lastPageDocList = new ArrayList<>(); + logResponse.setLogList(lastPageDocList); + int cnt = 0; + for (LOG_DATA_TYPE doc : docList) { + if (cnt < numberOfLogsOnLastPage) { + lastPageDocList.add(doc); } - Collections.reverse(lastPageDocList); - - } catch (SolrException | SolrServerException | IOException | NumberFormatException e) { - logger.error("Count Query was not executed successfully",e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + cnt++; } + Collections.reverse(lastPageDocList); return logResponse; } - protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase) { - try { - QueryResponse response = solrDaoBase.process(solrQuery); - SEARCH_RESPONSE logResponse = createLogSearchResponse(); - SolrDocumentList docList = response.getResults(); - List serviceLogDataList = convertToSolrBeans(response); - if (docList != null && !docList.isEmpty()) { - logResponse.setLogList(serviceLogDataList); - logResponse.setStartIndex((int) docList.getStart()); - logResponse.setTotalCount(docList.getNumFound()); - Integer rowNumber = solrQuery.getRows(); - if (rowNumber == null) { - logger.error("No RowNumber was set in solrQuery"); - return createLogSearchResponse(); - } - logResponse.setPageSize(rowNumber); + protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrDataQuery solrQuery, SolrDaoBase solrDaoBase, String event) { + SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(solrQuery); + return getLogAsPaginationProvided(query, solrDaoBase, event); + } + + + protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase, String event) { + QueryResponse response = solrDaoBase.process(solrQuery, event); + SEARCH_RESPONSE logResponse = createLogSearchResponse(); + SolrDocumentList docList = response.getResults(); + List serviceLogDataList = convertToSolrBeans(response); + if (docList != null && !docList.isEmpty()) { + logResponse.setLogList(serviceLogDataList); + logResponse.setStartIndex((int) docList.getStart()); + logResponse.setTotalCount(docList.getNumFound()); + Integer rowNumber = solrQuery.getRows(); + if (rowNumber == null) { + logger.error("No RowNumber was set in solrQuery"); + return createLogSearchResponse(); } - return logResponse; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + logResponse.setPageSize(rowNumber); } + return logResponse; } protected Long countQuery(SolrQuery query,SolrDaoBase solrDaoBase) throws SolrException, SolrServerException, IOException { @@ -189,13 +143,6 @@ public abstract class ManagerBase