airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lah...@apache.org
Subject [3/6] airavata git commit: Add more docker files for kafka, logstash
Date Wed, 07 Sep 2016 08:27:11 GMT
Add more docker files for kafka, logstash


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/724f9591
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/724f9591
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/724f9591

Branch: refs/heads/lahiru/AIRAVATA-2065
Commit: 724f95917ebfe68cebf5311805e04040422a08d9
Parents: 9b372ac
Author: Lahiru Ginnaliya Gamathige <lahiru@apache.org>
Authored: Tue Aug 30 00:18:35 2016 -0700
Committer: Lahiru Ginnaliya Gamathige <lahiru@apache.org>
Committed: Fri Sep 2 09:03:44 2016 -0700

----------------------------------------------------------------------
 deploy/images/kafka/Dockerfile                  |   8 ++
 deploy/images/kafka/start-kafka.sh              |  17 ++++
 deploy/images/logstash/Dockerfile               |   6 ++
 .../logstash-output-amazon_es-0.3-java.gem      | Bin 0 -> 22016 bytes
 .../logstash-output-dogstatsd-2.0.0.gem         | Bin 0 -> 9728 bytes
 deploy/images/logstash/logstash.conf            |  86 +++++++++++++++++++
 6 files changed, 117 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/kafka/Dockerfile
----------------------------------------------------------------------
diff --git a/deploy/images/kafka/Dockerfile b/deploy/images/kafka/Dockerfile
new file mode 100644
index 0000000..59770e1
--- /dev/null
+++ b/deploy/images/kafka/Dockerfile
@@ -0,0 +1,8 @@
+FROM scigap/java:8
+ENV ZOOKEEPER localhost:2181/kafka
+ENV LOG_DIRS /var/lib/kafka
+ENV JMX_PORT 9999
+ADD start-kafka.sh /start-kafka.sh
+RUN mkdir -p /opt/kafka && curl http://mirrors.sonic.net/apache/kafka/0.8.2.2/kafka_2.11-0.8.2.2.tgz
> /tmp/kafka.tgz && \
+    tar -zxf /tmp/kafka.tgz  -C /opt/kafka --strip-components=1 && rm -f /tmp/kafka.tgz
+ENTRYPOINT /start-kafka.sh
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/kafka/start-kafka.sh
----------------------------------------------------------------------
diff --git a/deploy/images/kafka/start-kafka.sh b/deploy/images/kafka/start-kafka.sh
new file mode 100644
index 0000000..3b3ee86
--- /dev/null
+++ b/deploy/images/kafka/start-kafka.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if test -z "${BROKER_ID}"; then
+  BROKER_ID=$(ifconfig eth0 | awk '/inet addr/{print substr($2,6)}' | head -n1  | tr -d '.')
+fi
+
+mkdir -p /opt/kafka/etc
+cat <<EOF > /opt/kafka/etc/server.properties
+broker.id=${BROKER_ID}
+zookeeper.connect=${ZOOKEEPER}
+log.dirs=${LOG_DIRS}
+num.partitions=2
+default.replication.factor=2
+advertised.host.name=${ADVERTISED_HOST_NAME}
+EOF
+
+exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/etc/server.properties

http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/Dockerfile b/deploy/images/logstash/Dockerfile
new file mode 100644
index 0000000..c3b8963
--- /dev/null
+++ b/deploy/images/logstash/Dockerfile
@@ -0,0 +1,6 @@
+FROM logstash:2
+
+COPY *.gem /tmp/
+RUN logstash-plugin install /tmp/*.gem && rm -f /tmp/*.gem
+RUN logstash-plugin install logstash-codec-avro logstash-codec-cloudtrail logstash-input-journald
+ENTRYPOINT []
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem b/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem
new file mode 100644
index 0000000..d3c913a
Binary files /dev/null and b/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem
differ

http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem b/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem
new file mode 100644
index 0000000..ec177a6
Binary files /dev/null and b/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem differ

http://git-wip-us.apache.org/repos/asf/airavata/blob/724f9591/deploy/images/logstash/logstash.conf
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash.conf b/deploy/images/logstash/logstash.conf
new file mode 100644
index 0000000..1f94cb2
--- /dev/null
+++ b/deploy/images/logstash/logstash.conf
@@ -0,0 +1,86 @@
+input {
+  kafka {
+    topic_id = > "${ENV_NAME}_all_logs"
+    zk_connect = > "${KAFKA_ZK}"
+    auto_offset_reset = > "smallest"
+    type = > "all_logs"
+  }
+  kafka {
+    topic_id = > "${ENV_NAME}_apiserver_logs"
+    zk_connect = > "${KAFKA_ZK}"
+    auto_offset_reset = > "smallest"
+    type = > "apiserver_logs"
+  }
+  kafka {
+    topic_id = > "${ENV_NAME}_gfac_logs"
+    zk_connect = > "${KAFKA_ZK}"
+    auto_offset_reset = > "smallest"
+    type = > "gfac_logs"
+  }
+  kafka {
+    topic_id = > "${ENV_NAME}_orchestrator_logs"
+    zk_connect = > "${KAFKA_ZK}"
+    auto_offset_reset = > "smallest"
+    type = > "orchestrator_logs"
+  }
+  kafka {
+    topic_id = > "${ENV_NAME}_credentialstore_logs"
+    zk_connect = > "${KAFKA_ZK}"
+    auto_offset_reset = > "smallest"
+    type = > "credentialstore_logs"
+  }
+}
+
+filter {
+  mutate {add_field = > {"[@metadata][level]" = > "%{[level]}"}}
+  mutate {lowercase = > ["[@metadata][level]"]}
+  mutate {gsub = > ["level", "LOG_", ""]}
+  mutate {
+    add_tag = > ["${ENV_NAME}", "${OS_NAME}-${OS_VERSION}"]
+  }
+  date {
+    match = > ["timestamp", "ISO8601"]
+  }
+}
+
+output {
+  if [type] == "apiserver_logs" {
+    if [@metadata][level] == "debug" {
+      amazon_es {
+        hosts = > ["${ES_ENDPOINT}"]
+        region = > "us-east-1"
+        index = > "${ENV_NAME}-apiserver-logs-logstash-%{+YYYY.MM.dd}"
+      }
+    }
+  } else if [type] == "gfac_logs" {
+    if [@metadata][level] == "debug" {
+      amazon_es {
+        hosts = > ["${ES_ENDPOINT}"]
+        region = > "us-east-1"
+        index = > "${ENV_NAME}-gfac-logs-logstash-%{+YYYY.MM.dd}"
+      }
+    }
+  } else if [type] == "orchestrator_logs" {
+    if [@metadata][level] == "debug" {
+      amazon_es {
+        hosts = > ["${ES_ENDPOINT}"]
+        region = > "us-east-1"
+        index = > "${ENV_NAME}-orchestrator-logs-logstash-%{+YYYY.MM.dd}"
+      }
+    }
+  } else if [type] == "credentialstore_logs" {
+    if [@metadata][level] == "debug" {
+      amazon_es {
+        hosts = > ["${ES_ENDPOINT}"]
+        region = > "us-east-1"
+        index = > "${ENV_NAME}-credentialstore-logs-logstash-%{+YYYY.MM.dd}"
+      }
+    }
+  } else {
+    amazon_es {
+      hosts => ["${ES_ENDPOINT}"]
+      region => "us-east-1"
+      index => "${ENV_NAME}-airavata-logs-logstash-%{+YYYY.MM.dd}"
+    }
+  }
+}


Mime
View raw message