metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kylerichard...@apache.org
Subject incubator-metron git commit: METRON-700 Add hadoop container to metron-docker (kylerichardson) closes apache/incubator-metron#472
Date Wed, 29 Mar 2017 00:40:09 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 88d72339f -> c13ee8265


METRON-700 Add hadoop container to metron-docker (kylerichardson) closes apache/incubator-metron#472


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/c13ee826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/c13ee826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/c13ee826

Branch: refs/heads/master
Commit: c13ee82656830e131f929e443930465c475ff4eb
Parents: 88d7233
Author: kylerichardson <kylerichardson2@gmail.com>
Authored: Tue Mar 28 20:38:37 2017 -0400
Committer: Kyle Richardson <kylerichardson@apache.org>
Committed: Tue Mar 28 20:38:37 2017 -0400

----------------------------------------------------------------------
 metron-docker/.gitignore                        |  4 +--
 metron-docker/README.md                         |  2 ++
 metron-docker/compose/docker-compose.yml        | 12 ++++++-
 metron-docker/compose/hadoop/Dockerfile         | 34 ++++++++++++++++++
 metron-docker/compose/hadoop/conf/core-site.xml | 24 +++++++++++++
 metron-docker/compose/hadoop/conf/hdfs-site.xml | 32 +++++++++++++++++
 .../compose/hadoop/docker-entrypoint.sh         | 38 ++++++++++++++++++++
 metron-docker/compose/kafkazk/Dockerfile        | 10 ++----
 metron-docker/compose/kafkazk/bin/init-zk.sh    |  1 +
 metron-docker/compose/storm/Dockerfile          |  6 ++--
 metron-docker/pom.xml                           | 24 +++++++++++--
 11 files changed, 169 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/.gitignore
----------------------------------------------------------------------
diff --git a/metron-docker/.gitignore b/metron-docker/.gitignore
index 2001ab7..05cbde5 100644
--- a/metron-docker/.gitignore
+++ b/metron-docker/.gitignore
@@ -1,8 +1,6 @@
 /compose/.env
 /compose/elasticsearch/es_templates
-/compose/kafkazk/common
-/compose/kafkazk/parser
-/compose/kafkazk/enrichment
+/compose/kafkazk/packages/
 /compose/hbase/data-management
 /compose/storm/elasticsearch
 /compose/storm/enrichment

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/README.md
----------------------------------------------------------------------
diff --git a/metron-docker/README.md b/metron-docker/README.md
index eb0138b..32892f8 100644
--- a/metron-docker/README.md
+++ b/metron-docker/README.md
@@ -13,6 +13,7 @@ Metron Docker includes these images that have been customized for Metron:
   - Storm (with all topologies deployed)
   - Elasticsearch
   - Kibana
+  - HDFS
 
 Setup
 -----
@@ -80,6 +81,7 @@ Then, assuming a host ip of `192.168.99.100`, the UIs and APIs are available
at:
 * HBase - http://192.168.99.100:16010/
 * Elasticsearch - http://192.168.99.100:9200/_plugin/head/
 * Kibana - http://192.168.99.100:5601/
+* HDFS (Namenode) - http://192.168.99.100:50070/
 
 The Storm logs can be useful when troubleshooting topologies.  They can be found on the Storm
container in `/usr/share/apache-storm/logs`.
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/docker-compose.yml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/docker-compose.yml b/metron-docker/compose/docker-compose.yml
index 8ef2d85..efc4227 100644
--- a/metron-docker/compose/docker-compose.yml
+++ b/metron-docker/compose/docker-compose.yml
@@ -37,6 +37,14 @@ services:
       - "/opt/hbase-1.1.6/conf"
     depends_on:
       - kafkazk
+  hadoop:
+    build:
+      context: ./hadoop
+    ports:
+      - "50070:50070"
+    volumes:
+      - "/opt/hadoop-2.7.3/etc/hadoop"
+    command: tail -f /dev/null
   storm:
     build:
       context: ./storm
@@ -48,13 +56,15 @@ services:
       - "8081:8081"
     environment:
       ZOOKEEPER_ADDR: kafkazk
-      CONFIG_TOPOLOGY_CLASSPATH: /opt/hbase-1.1.6/conf
+      CONFIG_TOPOLOGY_CLASSPATH: "/opt/hbase-1.1.6/conf:/opt/hadoop-2.7.3/etc/hadoop"
     volumes_from:
       - hbase
+      - hadoop
     depends_on:
       - kafkazk
       - hbase
       - elasticsearch
+      - hadoop
     command: --daemon nimbus supervisor ui logviewer
   elasticsearch:
     build:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/Dockerfile b/metron-docker/compose/hadoop/Dockerfile
new file mode 100644
index 0000000..caddbd9
--- /dev/null
+++ b/metron-docker/compose/hadoop/Dockerfile
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+FROM openjdk:7
+
+ENV HADOOP_PREFIX=/opt/hadoop-2.7.3
+ENV HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
+ENV HADOOP_IDENT_STRING=root
+
+RUN curl -sL http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz
| tar -xzC /opt
+
+COPY ./conf/* $HADOOP_CONF_DIR/
+RUN mkdir -p /hadoop/dfs/name \
+    && mkdir -p /hadoop/dfs/data \
+    && $HADOOP_PREFIX/bin/hdfs namenode -format
+
+WORKDIR $HADOOP_PREFIX
+EXPOSE 9000 50010 50020 50070 50075
+
+COPY ./docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/conf/core-site.xml b/metron-docker/compose/hadoop/conf/core-site.xml
new file mode 100644
index 0000000..f7a9465
--- /dev/null
+++ b/metron-docker/compose/hadoop/conf/core-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://hadoop:9000</value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/conf/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/conf/hdfs-site.xml b/metron-docker/compose/hadoop/conf/hdfs-site.xml
new file mode 100644
index 0000000..c75a464
--- /dev/null
+++ b/metron-docker/compose/hadoop/conf/hdfs-site.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>/hadoop/dfs/name</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/hadoop/dfs/data</value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/hadoop/docker-entrypoint.sh
----------------------------------------------------------------------
diff --git a/metron-docker/compose/hadoop/docker-entrypoint.sh b/metron-docker/compose/hadoop/docker-entrypoint.sh
new file mode 100755
index 0000000..97b9809
--- /dev/null
+++ b/metron-docker/compose/hadoop/docker-entrypoint.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# exit immediately on error
+set -e
+
+# start namenode
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_PREFIX/etc/hadoop --script hdfs start
namenode
+
+# start datanode
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_PREFIX/etc/hadoop --script hdfs start
datanode
+
+# create metron base directory
+$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /apps/metron
+
+# create directory for geo database
+$HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /apps/metron/geo/default
+
+# download geo database to hdfs
+curl http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | $HADOOP_PREFIX/bin/hdfs
dfs -put - /apps/metron/geo/default/GeoLite2-City.mmdb.gz
+
+# pass through CMD as PID 1
+exec "$@"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/kafkazk/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/kafkazk/Dockerfile b/metron-docker/compose/kafkazk/Dockerfile
index 401b7fd..77b6b7e 100644
--- a/metron-docker/compose/kafkazk/Dockerfile
+++ b/metron-docker/compose/kafkazk/Dockerfile
@@ -34,13 +34,9 @@ RUN yum install -y java-1.8.0-openjdk lsof
 RUN mkdir -p $METRON_HOME
 ADD ./bin /opt/kafka_2.11-0.10.0.0/bin
 RUN chmod 755 /opt/kafka_2.11-0.10.0.0/bin/wait-for-it.sh
-ADD ./common /common
-ADD ./parser /parser
-ADD ./enrichment /enrichment
-ADD ./data /data
-RUN tar -xzf /common/metron-common-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
-RUN tar -xzf /parser/metron-parsers-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
-RUN tar -xzf /enrichment/metron-enrichment-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
+COPY ./data /data/
+COPY ./packages/* /packages/
+RUN find /packages -type f -name '*.tar.gz' -exec tar -xzf {} -C /usr/metron/$METRON_VERSION/
\;
 ADD ./conf /$METRON_HOME/config/zookeeper
 
 EXPOSE 2181 9092

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/kafkazk/bin/init-zk.sh
----------------------------------------------------------------------
diff --git a/metron-docker/compose/kafkazk/bin/init-zk.sh b/metron-docker/compose/kafkazk/bin/init-zk.sh
index 6e2dd3f..4974683 100755
--- a/metron-docker/compose/kafkazk/bin/init-zk.sh
+++ b/metron-docker/compose/kafkazk/bin/init-zk.sh
@@ -19,4 +19,5 @@ echo "create /metron metron" | ./bin/zookeeper-shell.sh localhost:2181
 echo "create /metron/topology topology" | ./bin/zookeeper-shell.sh localhost:2181
 echo "create /metron/topology/parsers parsers" | ./bin/zookeeper-shell.sh localhost:2181
 echo "create /metron/topology/enrichments enrichments" | ./bin/zookeeper-shell.sh localhost:2181
+echo "create /metron/topology/indexing indexing" | ./bin/zookeeper-shell.sh localhost:2181
 $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/compose/storm/Dockerfile
----------------------------------------------------------------------
diff --git a/metron-docker/compose/storm/Dockerfile b/metron-docker/compose/storm/Dockerfile
index 4d80479..78aeba4 100644
--- a/metron-docker/compose/storm/Dockerfile
+++ b/metron-docker/compose/storm/Dockerfile
@@ -20,7 +20,6 @@ ARG METRON_VERSION
 
 ENV METRON_VERSION $METRON_VERSION
 ENV METRON_HOME /usr/metron/$METRON_VERSION/
-ENV GEO_DIR /apps/metron/geo/default/
 
 ADD ./bin $METRON_HOME/bin
 ADD ./parser /parser
@@ -28,7 +27,6 @@ ADD ./enrichment /enrichment
 ADD ./indexing /indexing
 ADD ./elasticsearch /elasticsearch
 RUN mkdir -p $METRON_HOME
-RUN mkdir -p $GEO_DIR
 RUN tar -xzf /parser/metron-parsers-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
 
 RUN tar -xzf /enrichment/metron-enrichment-$METRON_VERSION-archive.tar.gz -C /usr/metron/$METRON_VERSION/
@@ -49,10 +47,10 @@ RUN tar -xzf /elasticsearch/metron-elasticsearch-$METRON_VERSION-archive.tar.gz
 RUN sed -i -e "s/kafka.zk=.*:/kafka.zk=kafkazk:/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
 RUN sed -i -e "s/kafka.broker=.*/kafka.broker=kafkazk:9092/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
 RUN sed -i -e "s/es.ip=.*/es.ip=metron-elasticsearch/g" /usr/metron/$METRON_VERSION/config/elasticsearch.properties
-RUN sed -i -e "s/bolt.hdfs.file.system.url=.*:8020/bolt.hdfs.file.system.url=file\:\/\/\//g"
/usr/metron/$METRON_VERSION/config/elasticsearch.properties
+RUN sed -i -e "s/bolt.hdfs.file.system.url=.*/bolt.hdfs.file.system.url=hdfs:\/\/hadoop:9000/g"
/usr/metron/$METRON_VERSION/config/elasticsearch.properties
+RUN sed -i -e "s/index.hdfs.output=.*/index.hdfs.output=\/apps\/metron\/indexing\/indexed/g"
/usr/metron/$METRON_VERSION/config/elasticsearch.properties
 
 EXPOSE 8080 8000
 EXPOSE 8081 8081
 
 WORKDIR $METRON_HOME
-RUN cd $GEO_DIR && wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c13ee826/metron-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-docker/pom.xml b/metron-docker/pom.xml
index 1351cdf..0f4b1f8 100644
--- a/metron-docker/pom.xml
+++ b/metron-docker/pom.xml
@@ -41,7 +41,7 @@
                             <goal>copy-resources</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.basedir}/compose/kafkazk/common</outputDirectory>
+                            <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
                             <resources>
                                 <resource>
                                     <directory>../metron-platform/metron-common/target/</directory>
@@ -59,7 +59,7 @@
                             <goal>copy-resources</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.basedir}/compose/kafkazk/parser</outputDirectory>
+                            <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
                             <resources>
                                 <resource>
                                     <directory>../metron-platform/metron-parsers/target/</directory>
@@ -77,7 +77,7 @@
                             <goal>copy-resources</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.basedir}/compose/kafkazk/enrichment</outputDirectory>
+                            <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
                             <resources>
                                 <resource>
                                     <directory>../metron-platform/metron-enrichment/target/</directory>
@@ -89,6 +89,24 @@
                         </configuration>
                     </execution>
                     <execution>
+                        <id>copy-indexing-to-kafkazk</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.basedir}/compose/kafkazk/packages</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>../metron-platform/metron-indexing/target/</directory>
+                                    <includes>
+                                        <include>*.tar.gz</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                    <execution>
                         <id>copy-templates-to-elasticsearch</id>
                         <phase>prepare-package</phase>
                         <goals>


Mime
View raw message