metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject metron git commit: METRON-1482 Update REST to work with Solr (merrimanr) closes apache/metron#957
Date Thu, 08 Mar 2018 20:08:43 GMT
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr a7a41a55a -> cdbe7ac1f


METRON-1482 Update REST to work with Solr (merrimanr) closes apache/metron#957


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/cdbe7ac1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/cdbe7ac1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/cdbe7ac1

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: cdbe7ac1f291c8d5be505973aaf01aa1d8a91920
Parents: a7a41a5
Author: merrimanr <merrimanr@gmail.com>
Authored: Thu Mar 8 14:08:22 2018 -0600
Committer: merrimanr <merrimanr@gmail.com>
Committed: Thu Mar 8 14:08:22 2018 -0600

----------------------------------------------------------------------
 .../CURRENT/package/scripts/rest_commands.py    |   1 +
 .../metron-rest/src/main/scripts/metron-rest.sh |  29 +++--
 .../metron/solr/dao/SolrMetaAlertDao.java       | 126 +++++++++++++++++++
 .../apache/metron/solr/dao/SolrSearchDao.java   |   5 +-
 4 files changed, 150 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 4f717bb..c3d3ec1 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -101,6 +101,7 @@ class RestCommands:
           "export METRON_INDEX_CP={metron_indexing_classpath};"
           "export METRON_LOG_DIR={metron_log_dir};"
           "export METRON_PID_FILE={pid_file};"
+          "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};"
           "{metron_home}/bin/metron-rest.sh;"
           "unset METRON_JDBC_PASSWORD;"
         ))

http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index f9a2b69..e0eeaa4 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -84,19 +84,32 @@ if [ ${METRON_JDBC_CLIENT_PATH} ]; then
     METRON_REST_CLASSPATH+=":${METRON_JDBC_CLIENT_PATH}"
 fi
 
-# Use a custom indexing jar if provided, else pull the metron-elasticsearch uber jar
+# Use metron-elasticsearch uber jar by default
+indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar"
+# Use metron-solr uber jar if ra indexing writer set to Solr
+if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
+    indexing_jar_pattern="${METRON_HOME}/lib/metron-solr*uber.jar"
+fi
+# Use a custom indexing jar if provided
 if [ ${METRON_INDEX_CP} ]; then
-    echo "Default metron indexing jar is: ${METRON_INDEX_CP}"
-    METRON_REST_CLASSPATH+=":${METRON_INDEX_CP}"
-else
-    indexing_jar_pattern="${METRON_HOME}/lib/metron-elasticsearch*uber.jar"
-    indexing_files=( ${indexing_jar_pattern} )
-    echo "Default metron indexing jar is: ${indexing_files[0]}"
-    METRON_REST_CLASSPATH+=":${indexing_files[0]}"
+    indexing_jar_pattern="${METRON_INDEX_CP}"
 fi
+indexing_files=( ${indexing_jar_pattern} )
+echo "Metron indexing jar is: ${indexing_files[0]}"
+METRON_REST_CLASSPATH+=":${indexing_files[0]}"
 
 echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
 
+#Use Solr daos if ra indexing writer set to Solr
+if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
+    METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao"
+    METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao"
+    echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}"
+    echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}"
+    METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO}
+    METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO}
+fi
+
 echo "Starting application"
 ${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \
 -cp ${METRON_REST_CLASSPATH} \

http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
new file mode 100644
index 0000000..389cb4e
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.solr.dao;
+
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MetaAlertDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
+import org.apache.metron.indexing.dao.search.*;
+import org.apache.metron.indexing.dao.update.Document;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class SolrMetaAlertDao implements MetaAlertDao {
+
+    private SolrDao solrDao;
+
+    @Override
+    public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException
{
+        return null;
+    }
+
+    @Override
+    public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) throws
InvalidCreateException, IOException {
+        return null;
+    }
+
+    @Override
+    public boolean addAlertsToMetaAlert(String metaAlertGuid, List<GetRequest> getRequests)
throws IOException {
+        return false;
+    }
+
+    @Override
+    public boolean removeAlertsFromMetaAlert(String metaAlertGuid, List<GetRequest>
getRequests) throws IOException {
+        return false;
+    }
+
+    @Override
+    public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus status) throws
IOException {
+        return false;
+    }
+
+    @Override
+    public void init(IndexDao indexDao) {
+
+    }
+
+    @Override
+    public void init(IndexDao indexDao, Optional<String> threatSort) {
+        if (indexDao instanceof MultiIndexDao) {
+            MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao;
+            for (IndexDao childDao : multiIndexDao.getIndices()) {
+                if (childDao instanceof SolrDao) {
+                    this.solrDao = (SolrDao) childDao;
+                }
+            }
+        } else if (indexDao instanceof SolrDao) {
+            this.solrDao = (SolrDao) indexDao;
+        } else {
+            throw new IllegalArgumentException(
+                    "Need an SolrDao when using SolrMetaAlertDao"
+            );
+        }
+    }
+
+    @Override
+    public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException
{
+        return solrDao.search(searchRequest);
+    }
+
+    @Override
+    public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException {
+        return solrDao.group(groupRequest);
+    }
+
+    @Override
+    public void init(AccessConfig config) {
+
+    }
+
+    @Override
+    public Document getLatest(String guid, String sensorType) throws IOException {
+        return solrDao.getLatest(guid, sensorType);
+    }
+
+    @Override
+    public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws
IOException {
+        return solrDao.getAllLatest(getRequests);
+    }
+
+    @Override
+    public void update(Document update, Optional<String> index) throws IOException
{
+        solrDao.update(update, index);
+    }
+
+    @Override
+    public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException
{
+        solrDao.batchUpdate(updates);
+    }
+
+    @Override
+    public Map<String, FieldType> getColumnMetadata(List<String> indices) throws
IOException {
+        return solrDao.getColumnMetadata(indices);
+    }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/cdbe7ac1/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index 378c976..2acf3c9 100644
--- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -223,12 +223,11 @@ public class SolrSearchDao implements SearchDao {
   private SearchResult getSearchResult(SolrDocument solrDocument, Optional<List<String>>
fields) {
     SearchResult searchResult = new SearchResult();
     searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID));
-    Map<String, Object> source;
+    final Map<String, Object> source = new HashMap<>();
     if (fields.isPresent()) {
-      source = new HashMap<>();
       fields.get().forEach(field -> source.put(field, solrDocument.getFieldValue(field)));
     } else {
-      source = solrDocument.getFieldValueMap();
+      solrDocument.getFieldNames().forEach(field -> source.put(field, solrDocument.getFieldValue(field)));
     }
     searchResult.setSource(source);
     return searchResult;


Mime
View raw message