bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject [12/13] bigtop git commit: BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
Date Fri, 18 Mar 2016 02:22:44 GMT
BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ

thrift-devel is still an issue for Ubuntu


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b8ab3931
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b8ab3931
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b8ab3931

Branch: refs/heads/BIGTOP-2320
Commit: b8ab3931e7167af5a34d876717195e89e2812fa1
Parents: 8a946b7
Author: Konstantin Boudnik <cos@apache.org>
Authored: Thu Feb 11 19:33:30 2016 +0300
Committer: Konstantin Boudnik <cos@apache.org>
Committed: Thu Mar 17 19:20:01 2016 -0700

----------------------------------------------------------------------
 bigtop_toolchain/manifests/installer.pp |  2 ++
 bigtop_toolchain/manifests/libhdfs.pp   | 45 ++++++++++++++++++++++++++++
 bigtop_toolchain/manifests/packages.pp  | 11 ++++---
 bigtop_toolchain/manifests/protobuf.pp  | 11 ++++---
 4 files changed, 61 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/installer.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
index 246fcaf..060d4bd 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -20,6 +20,8 @@ class bigtop_toolchain::installer {
   include bigtop_toolchain::gradle
   include bigtop_toolchain::protobuf
   include bigtop_toolchain::packages
+## The hack workaround until such times when the lib is embedded into Hawq
+  include bigtop_toolchain::libhdfs
   include bigtop_toolchain::env
   include bigtop_toolchain::user
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/libhdfs.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp
new file mode 100644
index 0000000..4698108
--- /dev/null
+++ b/bigtop_toolchain/manifests/libhdfs.pp
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class bigtop_toolchain::libhdfs {
+
+  include bigtop_toolchain::packages
+
+
+  case $operatingsystem {
+    /(?i:(centos|fedora|amazon))/: {
+
+      $apache_prefix = nearest_apache_mirror()
+      $hackrepourl = "https://bintray.com/artifact/download/wangzw/rpm/centos7/x86_64"
+      $libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm"
+      $libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm"
+
+      exec {"download":
+        path    => "/usr/bin",
+        cwd     => "/usr/src",
+        command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev
-o $libhdfs_dev",
+        creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev",
+      }
+
+      exec {"install":
+        path    => "/usr/bin",
+        cwd     => "/usr/src",
+        command => "yum install -y $libhdfs $libhdfs_dev",
+        require => [ Exec[ 'download' ], Package[ $packages::pkgs ] ],
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/packages.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index 26959e2..edd01da 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -22,16 +22,19 @@ class bigtop_toolchain::packages {
        } else {
          $mysql_devel="mariadb-devel"
        }
-       $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake",
"libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel",
"openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel",
"sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs",
"asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel",
"libuuid-devel" ]
      }
-    /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake",
"autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel",
"cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel",
"python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel",
"sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel",
"snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
+       $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf",
"automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel",
"bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools",
"libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel",
"sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs",
"asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel", "boost-devel",
"xfsprogs-devel", "libuuid-devel", "thrift-devel" ]
+       exec { '/usr/bin/yum install -y epel-release':
+       } -> Package <| |>
+     }
+    /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git",
"make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++",
"fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf",
"libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools",
"libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel",
"sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel",
"readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ]
       # fix package dependencies: BIGTOP-2120 and BIGTOP-2152
       exec { '/usr/bin/zypper remove -y krb5-mini':
       } -> exec {'/usr/bin/zypper install -y libopenssl-devel':
       } -> Package <| |>
     }
-    Amazon: {                 $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake",
"autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel",
"openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel"
] }
-    /(Ubuntu|Debian)/: {      $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake",
"autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev",
"libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts",
"build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev",
"libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev",
"libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev",
"libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ]
+    Amazon: {                 $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git",
"make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++",
"fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel",
"libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config",
"fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] }
+    /(Ubuntu|Debian)/: {      $pkgs = [ "unzip", "curl", "libcurl4-openssl-dev", "wget",
"git-core", "make", "cmake", "autoconf", "automake", "libtool", "libapr1-dev", "bison", "gcc",
"g++", "fuse", "reprepro", "liblzo2-dev", "libbz2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev",
"libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make",
"libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev",
"zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf",
"uuid", "libgsasl7-dev", "libevent-dev", "libjson-c-dev", "python-setuptools", "libkrb5-dev",
"asciidoc", "libyaml-dev", "libgmp-dev", "libreadline-dev", "libsnappy-dev", "libboost-regex-dev",
"xfslibs-dev" ]
 
       exec { "apt-update":
         command => "/usr/bin/apt-get update"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/b8ab3931/bigtop_toolchain/manifests/protobuf.pp
----------------------------------------------------------------------
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index c71dec2..53d539b 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -21,13 +21,15 @@ class bigtop_toolchain::protobuf {
     /Ubuntu|Debian/: {
       case $architecture {
         'amd64' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5585371/+files/"
+                    $url1= "http://launchpadlibrarian.net/166035876/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb"
+                    $url2= "http://launchpadlibrarian.net/166035877/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"
                     $arch= "amd64" }
         'ppc64le' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5604345/+files"
                     $arch= "ppc64el" }
       }
     }
   }
- 
+
   case $operatingsystem{
     /Ubuntu|Debian/: {
       $libprotobuf8 = "libprotobuf8_2.5.0-9ubuntu1_$arch.deb"
@@ -35,13 +37,14 @@ class bigtop_toolchain::protobuf {
       $protobuf_compiler = "protobuf-compiler_2.5.0-9ubuntu1_$arch.deb"
 
       exec { "download protobuf":
+        path    => "/usr/bin",
         cwd     => "/usr/src",
-        command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl
-L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler",
-        creates  => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler"
]
+        command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl
-L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler;
curl -L $url1 -o libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb; curl -L $url2 -o libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
+        creates  => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler",
"/usr/src/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb", "/usr/src/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"
]
       }
       exec { "install protobuf":
         cwd     => "/usr/src",
-        command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler",
+        command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb
libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb",
         require => EXEC["download protobuf"],
       }
     }


Mime
View raw message