metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [2/2] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Date Sat, 16 Jun 2018 00:32:29 GMT
Merge branch 'master' into feature/METRON-1416-upgrade-solr


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a89a72c3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a89a72c3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a89a72c3

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: a89a72c3b2ceb5066fd7a310636507ce274c66ad
Parents: 10271c4 b081e80
Author: cstella <cestella@gmail.com>
Authored: Fri Jun 15 20:32:18 2018 -0400
Committer: cstella <cestella@gmail.com>
Committed: Fri Jun 15 20:32:18 2018 -0400

----------------------------------------------------------------------
 .../alert-details/alert-details.component.html  |  4 +--
 .../alert-details/alert-details.component.ts    | 24 ++++++++++++++---
 .../alerts-list/alerts-list.component.html      |  4 +--
 .../alerts/alerts-list/alerts-list.component.ts | 13 ++++++----
 .../src/app/alerts/alerts-list/query-builder.ts |  5 ++--
 .../table-view/table-view.component.html        | 18 ++++++-------
 .../table-view/table-view.component.ts          | 19 ++++++++++++--
 .../tree-view/tree-view.component.html          | 16 ++++++------
 .../tree-view/tree-view.component.ts            | 27 +++++++++++++++++---
 .../meta-alerts/meta-alerts.component.html      |  2 +-
 .../alerts/meta-alerts/meta-alerts.component.ts | 11 +++++---
 .../src/app/model/group-request.ts              |  2 +-
 .../src/app/service/global-config.service.ts    | 16 +++++++++---
 .../metron-alerts/src/app/utils/constants.ts    |  3 +--
 .../ElasticsearchMetaAlertIntegrationTest.java  |  5 ++--
 15 files changed, 120 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------
diff --cc metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index ffd7dd0,c939f04..39d47ac
--- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@@ -29,8 -30,8 +30,9 @@@ import {Patch} from '../../model/patch'
  import {AlertComment} from './alert-comment';
  import {AuthenticationService} from '../../service/authentication.service';
  import {MetronDialogBox} from '../../shared/metron-dialog-box';
- import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
 -import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
 +import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
++import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+ import { GlobalConfigService } from '../../service/global-config.service';
  
  export enum AlertState {
    NEW, OPEN, ESCALATE, DISMISS, RESOLVE

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
----------------------------------------------------------------------
diff --cc metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index 98459db,4496e37..07f7015
--- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@@ -36,10 -36,10 +36,10 @@@ import {AlertSearchDirective} from '../
  import {SearchResponse} from '../../model/search-response';
  import {ElasticsearchUtils} from '../../utils/elasticsearch-utils';
  import {Filter} from '../../model/filter';
- import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants';
+ import {TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants';
  import {TableViewComponent} from './table-view/table-view.component';
  import {Pagination} from '../../model/pagination';
 -import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from '../../utils/constants';
 +import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
  import {MetaAlertService} from '../../service/meta-alert.service';
  import {Facets} from '../../model/facets';
  import { GlobalConfigService } from '../../service/global-config.service';

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.html
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/utils/constants.ts
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 224ec56,933fa2a..c05efc1
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@@ -38,24 -41,36 +38,25 @@@ import java.util.HashMap
  import java.util.List;
  import java.util.Map;
  import java.util.Optional;
 -import java.util.Set;
 +import java.util.function.Function;
  import java.util.stream.Collectors;
+ 
+ import com.google.common.collect.ImmutableList;
  import org.adrianwalker.multilinestring.Multiline;
  import org.apache.metron.common.Constants;
  import org.apache.metron.common.utils.JSONUtils;
  import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
  import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao;
  import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
- import org.apache.metron.guava.collect.ImmutableList;
  import org.apache.metron.indexing.dao.AccessConfig;
  import org.apache.metron.indexing.dao.IndexDao;
 -import org.apache.metron.indexing.dao.MetaAlertDao;
 -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
 -import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
 +import org.apache.metron.indexing.dao.metaalert.MetaAlertIntegrationTest;
  import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
  import org.apache.metron.indexing.dao.search.GetRequest;
 -import org.apache.metron.indexing.dao.search.Group;
 -import org.apache.metron.indexing.dao.search.GroupRequest;
 -import org.apache.metron.indexing.dao.search.GroupResponse;
 -import org.apache.metron.indexing.dao.search.GroupResult;
 -import org.apache.metron.indexing.dao.search.InvalidSearchException;
  import org.apache.metron.indexing.dao.search.SearchRequest;
  import org.apache.metron.indexing.dao.search.SearchResponse;
 -import org.apache.metron.indexing.dao.search.SearchResult;
  import org.apache.metron.indexing.dao.search.SortField;
 -import org.apache.metron.indexing.dao.update.Document;
 -import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
 -import org.apache.metron.indexing.dao.update.PatchRequest;
  import org.junit.After;
  import org.junit.AfterClass;
  import org.junit.Assert;
@@@ -291,87 -1136,131 +292,87 @@@ public class ElasticsearchMetaAlertInte
      return cnt;
    }
  
 -  protected long getMatchingMetaAlertCount(String fieldName, String fieldValue) throws IOException,
InterruptedException {
 +  @Override
 +  protected long getMatchingMetaAlertCount(String fieldName, String fieldValue)
 +          throws IOException, InterruptedException {
      long cnt = 0;
      for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS))
{
 -      List<Map<String, Object>> docs = es.getAllIndexedDocs(METAALERTS_INDEX,
MetaAlertDao.METAALERT_DOC);
 +      List<Map<String, Object>> docs = es
 +              .getAllIndexedDocs(METAALERTS_INDEX, METAALERT_DOC);
        cnt = docs
 -          .stream()
 -          .filter(d -> {
 -            List<Map<String, Object>> alerts = (List<Map<String, Object>>)
d
 -                .get(MetaAlertDao.ALERT_FIELD);
 +              .stream()
 +              .filter(d -> {
 +                @SuppressWarnings("unchecked")
 +                List<Map<String, Object>> alerts = (List<Map<String, Object>>)
d
 +                        .get(ALERT_FIELD);
 +
 +                for (Map<String, Object> alert : alerts) {
 +                  Object newField = alert.get(fieldName);
 +                  if (newField != null && newField.equals(fieldValue)) {
 +                    return true;
 +                  }
 +                }
 +
 +                return false;
 +              }).count();
 +    }
 +    return cnt;
 +  }
  
 -            for (Map<String, Object> alert : alerts) {
 -              Object newField = alert.get(fieldName);
 -              if (newField != null && newField.equals(fieldValue)) {
 -                return true;
 +  @Override
 +  protected void addRecords(List<Map<String, Object>> inputData, String index,
String docType)
 +          throws IOException {
 +    es.add(index, docType, inputData.stream().map(m -> {
 +              try {
 +                return JSONUtils.INSTANCE.toJSON(m, true);
 +              } catch (JsonProcessingException e) {
 +                throw new IllegalStateException(e.getMessage(), e);
                }
              }
 -
 -            return false;
 -          }).count();
 -    }
 -    return cnt;
 +            ).collect(Collectors.toList())
 +    );
    }
  
 -  protected void findUpdatedDoc(Map<String, Object> message0, String guid, String
sensorType)
 -      throws InterruptedException, IOException, OriginalNotFoundException {
 -    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
 -      Document doc = metaDao.getLatest(guid, sensorType);
 -      if (doc != null && compareDocs(message0, doc.getDocument())) {
 -        return;
 -      }
 -    }
 -    throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES
+ " tries");
 +  @Override
 +  protected void setupTypings() {
 +    ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX)
 +            .setType("test_doc")
 +            .setSource(nestedAlertMapping)
 +            .get();
    }
  
 -  private boolean compareDocs(Map<String, Object> expected, Map<String, Object>
actual) {
 -    if (expected.size() != actual.size()) {
 -      return false;
 -    }
 -    for(String key: expected.keySet()) {
 -      if (ALERT_FIELD.equals(key)) {
 -        List<Map<String, Object>> expectedAlerts = (List<Map<String, Object>>)
expected.get(MetaAlertDao.ALERT_FIELD);
 -        ArrayList<Map<String, Object>> actualAlerts = (ArrayList<Map<String,
Object>>) actual.get(MetaAlertDao.ALERT_FIELD);
 -        if (!expectedAlerts.containsAll(actualAlerts) || !actualAlerts.containsAll(expectedAlerts))
{
 -          return false;
 -        }
 -      } else if (!expected.get(key).equals(actual.get(key))){
 -        return false;
 -      }
 -    }
 -    return true;
 +  @Override
 +  protected String getTestIndexName() {
 +    return INDEX_RAW;
    }
  
 -  protected boolean findCreatedDoc(String guid, String sensorType)
 -      throws InterruptedException, IOException, OriginalNotFoundException {
 -    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
 -      Document doc = metaDao.getLatest(guid, sensorType);
 -      if (doc != null) {
 -        return true;
 -      }
 -    }
 -    throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES
+ "tries");
 +  @Override
 +  protected String getTestIndexFullName() {
 +    return INDEX;
    }
  
 -  protected boolean findCreatedDocs(List<GetRequest> getRequests)
 -      throws InterruptedException, IOException, OriginalNotFoundException {
 -    for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
 -      Iterable<Document> docs = metaDao.getAllLatest(getRequests);
 -      if (docs != null) {
 -        int docCount = 0;
 -        for (Document doc: docs) {
 -          docCount++;
 -        }
 -        if (getRequests.size() == docCount) {
 -          return true;
 -        }
 -      }
 -    }
 -    throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries");
 +  @Override
 +  protected String getMetaAlertIndex() {
 +    return METAALERTS_INDEX;
    }
  
 -  protected List<Map<String, Object>> buildAlerts(int count) {
 -    List<Map<String, Object>> inputData = new ArrayList<>();
 -    for (int i = 0; i < count; ++i) {
 -      final String guid = "message_" + i;
 -      Map<String, Object> alerts = new HashMap<>();
 -      alerts.put(Constants.GUID, guid);
 -      alerts.put("source:type", SENSOR_NAME);
 -      alerts.put(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, i);
 -      alerts.put("timestamp", System.currentTimeMillis());
 -      inputData.add(alerts);
 -    }
 -    return inputData;
 +  @Override
 +  protected String getSourceTypeField() {
 +    return ElasticsearchMetaAlertDao.SOURCE_TYPE_FIELD;
    }
  
 -  protected List<Map<String, Object>> buildMetaAlerts(int count, MetaAlertStatus
status, Optional<List<Map<String, Object>>> alerts) {
 -    List<Map<String, Object>> inputData = new ArrayList<>();
 -    for (int i = 0; i < count; ++i) {
 -      final String guid = "meta_" + status.getStatusString() + "_" + i;
 -      inputData.add(buildMetaAlert(guid, status, alerts));
 -    }
 -    return inputData;
 +  @Override
 +  protected void setEmptiedMetaAlertField(Map<String, Object> docMap) {
 +    docMap.put(METAALERT_FIELD, new ArrayList<>());
    }
  
 -  protected Map<String, Object> buildMetaAlert(String guid, MetaAlertStatus status,
Optional<List<Map<String, Object>>> alerts) {
 -    Map<String, Object> metaAlert = new HashMap<>();
 -    metaAlert.put(Constants.GUID, guid);
 -    metaAlert.put("source:type", METAALERT_TYPE);
 -    metaAlert.put(MetaAlertDao.STATUS_FIELD, status.getStatusString());
 -    if (alerts.isPresent()) {
 -      List<Map<String, Object>> alertsList = alerts.get();
 -      metaAlert.put(ALERT_FIELD, alertsList);
 -    }
 -    return metaAlert;
 +  @Override
 +  protected boolean isFiniteDoubleOnly() {
 +    return true;
    }
  
 -  protected void elasticsearchAdd(List<Map<String, Object>> inputData, String
index, String docType)
 -      throws IOException {
 -    es.add(index, docType, inputData.stream().map(m -> {
 -          try {
 -            return JSONUtils.INSTANCE.toJSON(m, true);
 -          } catch (JsonProcessingException e) {
 -            throw new IllegalStateException(e.getMessage(), e);
 -          }
 -        }
 -        ).collect(Collectors.toList())
 -    );
 +  @Override
 +  protected boolean isEmptyMetaAlertList() {
 +    return true;
    }
- }
+ }


Mime
View raw message