bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ofle...@apache.org
Subject [35/51] [abbrv] bigtop git commit: BIGTOP-2357: Add Apache Flink to Bigtop / Create puppet recipes
Date Fri, 29 Jul 2016 17:03:16 GMT
BIGTOP-2357: Add Apache Flink to Bigtop / Create puppet recipes


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3e0de8c0
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3e0de8c0
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3e0de8c0

Branch: refs/heads/BIGTOP-2253
Commit: 3e0de8c04bc8f6cb5993d3c08d1f56a837958db1
Parents: b5d001d
Author: Robert Metzger <rmetzger@apache.org>
Authored: Tue May 24 14:51:48 2016 +0200
Committer: Roman Shaposhnik <rvs@apache.org>
Committed: Fri Jun 17 16:29:40 2016 -0700

----------------------------------------------------------------------
 .../puppet/hieradata/bigtop/cluster.yaml        |  9 ++-
 bigtop-deploy/puppet/hieradata/site.yaml        |  1 +
 bigtop-deploy/puppet/manifests/cluster.pp       |  5 ++
 .../puppet/modules/flink/manifests/init.pp      | 62 ++++++++++++++++++++
 .../modules/flink/templates/flink-conf.yaml     | 31 ++++++++++
 bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile  |  1 +
 6 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index f592f64..5c2c5f1 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -16,7 +16,7 @@
 # "$components" list. If $components isn't set then everything in the stack will
 # be installed as usual. Otherwise only a specified list will be set
 # Possible elements:
-# hadoop,yarn,hbase,alluxio,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs,
+# hadoop,yarn,hbase,alluxio,flink,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs,
 # hue,mahout,giraph,crunch,pig,hive,zookeeper,ycsb,qfs
 # Example (to deploy only HDFS and YARN server and gateway parts)
 # This can be a comma-separated list or an array.
@@ -191,3 +191,10 @@ zeppelin::server::spark_master_url: "yarn-client"
 zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
 zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}"
 zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}"
+
+# Flink
+flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}"
+flink::common::jobmanager_port: "6123"
+
+flink::common::ui_port: "8081"
+flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/site.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index d429269..b9ed6f9 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -15,6 +15,7 @@ hadoop::hadoop_storage_dirs:
 #  - alluxio
 #  - apex
 #  - crunch
+#  - flink
 #  - flume
 #  - giraph
 #  - ignite_hadoop

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index ba980bd..9ff424c 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -56,6 +56,10 @@ $roles_map = {
     master => ["alluxio-master"],
     worker => ["alluxio-worker"],
   },
+  flink => {
+    master => ["flink-jobmanager"],
+    worker => ["flink-taskmanager"],
+  },
   flume => {
     worker => ["flume-agent"],
   },
@@ -161,6 +165,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_
     "alluxio",
     "apex",
     "crunch",
+    "flink",
     "giraph",
     "hadoop",
     "hadoop_hbase",

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
new file mode 100644
index 0000000..ae79fe4
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
@@ -0,0 +1,62 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class flink {
+
+  class deploy ($roles) {
+    if ("flink-jobmanager" in $roles) {
+      include flink::jobmanager
+    }
+
+    if ("flink-taskmanager" in $roles) {
+      include flink::taskmanager
+    }
+  }
+
+  class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) {
+    # make sure flink is installed
+    package { "flink":
+      ensure => latest
+    }
+
+    # set values in flink-conf.yaml
+    file { "/etc/flink/conf/flink-conf.yaml":
+        content => template("flink/flink-conf.yaml"),
+        require => Package["flink"]
+    }
+  }
+
+  class jobmanager {
+    include common
+
+    service { "flink-jobmanager":
+      ensure => running,
+      require => Package["flink"],
+      subscribe => File["/etc/flink/conf/flink-conf.yaml"],
+      hasrestart => true,
+      hasstatus => true
+     
+    }
+
+  }
+
+  class taskmanager {
+    include common
+
+    service { "flink-taskmanager":
+      ensure => running,
+      require => Package["flink"],
+      subscribe => File["/etc/flink/conf/flink-conf.yaml"],
+      hasrestart => true,
+      hasstatus => true,
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
new file mode 100644
index 0000000..14e8853
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
@@ -0,0 +1,31 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+# Configuration values managed by puppet:
+jobmanager.rpc.address: <%= @jobmanager_host %>
+jobmanager.rpc.port: <%= @jobmanager_port %>
+jobmanager.web.port: <%= @ui_port %>
+
+<% if defined?(storage_dirs) %>
+taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %>
+<% end %>
+
+
+# For performance reasons its highly recommended to allocate as much memory to the
+# Flink TaskManager as possible by setting 'taskmanager.heap.mb'.

http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index 63aff20..b4017d0 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -67,6 +67,7 @@ bigtop_master = "bigtop1.vagrant"
 
 $script = <<SCRIPT
 service iptables stop
+service firewalld stop
 chkconfig iptables off
 # Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN,
 # which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP


Mime
View raw message