impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbap...@apache.org
Subject incubator-impala git commit: IMPALA-6045: Make build scripts more friendly to Ubuntu 16.04
Date Fri, 13 Oct 2017 03:50:45 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master 9ba923948 -> 2187d36b3


IMPALA-6045: Make build scripts more friendly to Ubuntu 16.04

This commit bundles two changes.

The first extracts from bootstrap_development.sh the commands to
prepare a system to build-and-test without actually doing so. This
enables custom build commands that might not load the test data, for
instance.

The second changes bootstrap_build.sh to work with Ubuntu 16.04. It
should still work with Ubuntu 14.04, but I don't anticipate that being
part of the Jenkins pre-merge job anymore, so I removed that from the
comment at the top of the file explaining what it does.

Change-Id: I8196a2a87bce5893a349a1b290c3f3d04fd80317
Reviewed-on: http://gerrit.cloudera.org:8080/8262
Reviewed-by: Michael Brown <mikeb@cloudera.com>
Tested-by: Jim Apple <jbapple-impala@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/2187d36b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/2187d36b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/2187d36b

Branch: refs/heads/master
Commit: 2187d36b36f5b2fc4318e57265cc31ee11d22423
Parents: 9ba9239
Author: Jim Apple <jbapple-impala@apache.org>
Authored: Wed Oct 11 13:20:49 2017 -0700
Committer: Jim Apple <jbapple-impala@apache.org>
Committed: Thu Oct 12 20:54:36 2017 +0000

----------------------------------------------------------------------
 bin/bootstrap_build.sh       |  18 +++-
 bin/bootstrap_development.sh | 172 +------------------------------
 bin/bootstrap_system.sh      | 211 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 227 insertions(+), 174 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2187d36b/bin/bootstrap_build.sh
----------------------------------------------------------------------
diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh
index 9c53ed8..82f1f37 100755
--- a/bin/bootstrap_build.sh
+++ b/bin/bootstrap_build.sh
@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# This script builds Impala from scratch. It is known to work on Ubuntu 14.04. To run it
+# This script builds Impala from scratch. It is known to work on Ubuntu 16.04. To run it
 # you need to have:
 #
 # 1. At least 8GB of free disk space
@@ -29,10 +29,20 @@
 # Set up some logging and exit conditions:
 set -euxo pipefail
 
-# Install dependencies:
+# Install non-java dependencies:
 sudo apt-get update
-sudo apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make maven openjdk-7-jdk \
+sudo apt-get --yes install g++ gcc git libsasl2-dev libssl-dev make maven \
     python-dev python-setuptools libffi-dev libkrb5-dev
 
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
+
+source /etc/lsb-release
+
+JDK_VERSION=8
+if [[ $DISTRIB_RELEASE = 14.04 ]]
+then
+  JDK_VERSION=7
+fi
+sudo apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source
+export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
+
 ./buildall.sh -notests -so

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2187d36b/bin/bootstrap_development.sh
----------------------------------------------------------------------
diff --git a/bin/bootstrap_development.sh b/bin/bootstrap_development.sh
index 612d765..0e31a3f 100755
--- a/bin/bootstrap_development.sh
+++ b/bin/bootstrap_development.sh
@@ -39,177 +39,9 @@
 
 set -eu -o pipefail
 
-if [[ -t 1 ]] # if on an interactive terminal
-then
-  echo "This script will clobber some system settings. Are you sure you want to"
-  echo -n "continue? "
-  while true
-  do
-    read -p "[yes/no] " ANSWER
-    ANSWER=$(echo "$ANSWER" | tr /a-z/ /A-Z/)
-    if [[ $ANSWER = YES ]]
-    then
-      break
-    elif [[ $ANSWER = NO ]]
-    then
-      echo "OK, Bye!"
-      exit 1
-    fi
-  done
-else
-  export DEBIAN_FRONTEND=noninteractive
-fi
+BINDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-set -x
-
-source /etc/lsb-release
-
-if ! [[ $DISTRIB_ID = Ubuntu ]]
-then
-  echo "This script only supports Ubuntu" >&2
-  exit 1
-fi
-
-if ! [[ $DISTRIB_RELEASE = 14.04 || $DISTRIB_RELEASE = 16.04 ]]
-then
-  echo "This script only supports Ubuntu 14.04 and 16.04" >&2
-  exit 1
-fi
-
-REAL_APT_GET=$(which apt-get)
-function apt-get {
-  for ITER in $(seq 1 20); do
-    echo "ATTEMPT: ${ITER}"
-    if sudo "${REAL_APT_GET}" "$@"
-    then
-      return 0
-    fi
-    sleep "${ITER}"
-  done
-  echo "NO MORE RETRIES"
-  return 1
-}
-
-apt-get update
-apt-get --yes install apt-utils
-apt-get --yes install git
-
-# If there is no Impala git repo, get one now
-if ! [[ -d ~/Impala ]]
-then
-  time -p git clone https://git-wip-us.apache.org/repos/asf/incubator-impala.git ~/Impala
-fi
-cd ~/Impala
-SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"
-echo "$SET_IMPALA_HOME" >> ~/.bashrc
-eval "$SET_IMPALA_HOME"
-
-apt-get --yes install ccache g++ gcc libffi-dev liblzo2-dev libkrb5-dev libsasl2-dev \
-        libssl-dev make maven ninja-build ntp ntpdate python-dev python-setuptools \
-        postgresql ssh wget vim-common
-
-if ! { service --status-all | grep -E '^ \[ \+ \]  ssh$'; }
-then
-  sudo service ssh start
-fi
-
-# TODO: config ccache to give it plenty of space
-# TODO: check that there is enough space on disk to do a build and data load
-# TODO: make this work with non-bash shells
-
-JDK_VERSION=8
-if [[ $DISTRIB_RELEASE = 14.04 ]]
-then
-  JDK_VERSION=7
-fi
-apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source
-SET_JAVA_HOME="export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64"
-echo "$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
-eval "$SET_JAVA_HOME"
-
-sudo service ntp stop
-sudo ntpdate us.pool.ntp.org
-# If on EC2, use Amazon's ntp servers
-if which dmidecode && { sudo dmidecode -s bios-version | grep amazon; }
-then
-  sudo sed -i 's/ubuntu\.pool/amazon\.pool/' /etc/ntp.conf
-  grep amazon /etc/ntp.conf
-  grep ubuntu /etc/ntp.conf
-fi
-# While it is nice to have ntpd running to keep the clock in sync, that does not work in
a
-# --privileged docker container, and a non-privileged container cannot run ntpdate, which
-# is strictly needed by Kudu.
-# TODO: Make privileged docker start ntpd
-sudo service ntp start || grep docker /proc/1/cgroup
-
-# IMPALA-3932, IMPALA-3926
-if [[ $DISTRIB_RELEASE = 16.04 ]]
-then
-  SET_LD_LIBRARY_PATH='export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}'
-elif [[ $DISTRIB_RELEASE = 14.04 ]]
-then
-  SET_LD_LIBRARY_PATH="unset LD_LIBRARY_PATH"
-fi
-echo "$SET_LD_LIBRARY_PATH" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
-eval "$SET_LD_LIBRARY_PATH"
-
-# TODO: What are the security implications of this?
-for PG_AUTH_FILE in /etc/postgresql/*/main/pg_hba.conf
-do
-  sudo sed -ri 's/local +all +all +peer/local all all trust/g' $PG_AUTH_FILE
-done
-sudo service postgresql restart
-sudo /etc/init.d/postgresql reload
-sudo service postgresql restart
-
-# Set up postgress for HMS
-if ! [[ 1 = $(sudo -u postgres psql -At -c "SELECT count(*) FROM pg_roles WHERE rolname =
'hiveuser';") ]]
-then
-  sudo -u postgres psql -c "CREATE ROLE hiveuser LOGIN PASSWORD 'password';"
-fi
-sudo -u postgres psql -c "ALTER ROLE hiveuser WITH CREATEDB;"
-sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';"
-
-# Setup ssh to ssh to localhost
-mkdir -p ~/.ssh
-chmod go-rwx ~/.ssh
-if ! [[ -f ~/.ssh/id_rsa ]]
-then
-  ssh-keygen -t rsa -N '' -q -f ~/.ssh/id_rsa
-fi
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
-echo "NoHostAuthenticationForLocalhost yes" >> ~/.ssh/config
-ssh localhost whoami
-
-# Workarounds for HDFS networking issues
-echo "127.0.0.1 $(hostname -s) $(hostname)" | sudo tee -a /etc/hosts
-# In Docker, one can change /etc/hosts as above but not with sed -i. The error message is
-# "sed: cannot rename /etc/sedc3gPj8: Device or resource busy". The following lines are
-# basically sed -i but with cp instead of mv for -i part.
-NEW_HOSTS=$(mktemp)
-sed 's/127.0.1.1/127.0.0.1/g' /etc/hosts > "${NEW_HOSTS}"
-diff -u /etc/hosts "${NEW_HOSTS}" || true
-sudo cp "${NEW_HOSTS}" /etc/hosts
-rm "${NEW_HOSTS}"
-
-sudo mkdir -p /var/lib/hadoop-hdfs
-sudo chown $(whoami) /var/lib/hadoop-hdfs/
-
-# TODO: restrict this to only the users it is needed for
-echo "* - nofile 1048576" | sudo tee -a /etc/security/limits.conf
-
-# LZO is not needed to compile or run Impala, but it is needed for the data load
-if ! [[ -d ~/Impala-lzo ]]
-then
-  git clone https://github.com/cloudera/impala-lzo.git ~/Impala-lzo
-fi
-if ! [[ -d ~/hadoop-lzo ]]
-then
-  git clone https://github.com/cloudera/hadoop-lzo.git ~/hadoop-lzo
-fi
-cd ~/hadoop-lzo/
-time -p ant package
-cd "$IMPALA_HOME"
+source "${BINDIR}/bootstrap_system.sh"
 
 export MAX_PYTEST_FAILURES=0
 source bin/impala-config.sh

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2187d36b/bin/bootstrap_system.sh
----------------------------------------------------------------------
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
new file mode 100644
index 0000000..43cf02d
--- /dev/null
+++ b/bin/bootstrap_system.sh
@@ -0,0 +1,211 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script bootstraps a system for Impala development from almost nothing; it is known
+# to work on Ubuntu 14.04 and 16.04. It clobbers some local environment and system
+# configurations, so it is best to run this in a fresh install. It also sets up the
+# ~/.bashrc for the calling user and impala-config-local.sh with some environment
+# variables to make Impala compile and run after this script is complete.
+#
+# The intended user is a person who wants to start contributing code to Impala. This
+# script serves as an executable reference point for how to get started.
+#
+# To run this in a Docker container:
+#
+#   1. Run with --privileged
+#   2. Give the container a non-root sudoer wih NOPASSWD:
+#      apt-get update
+#      apt-get install sudo
+#      adduser --disabled-password --gecos '' impdev
+#      echo 'impdev ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+#   3. Run this script as that user: su - impdev -c /bootstrap_development.sh
+
+set -eu -o pipefail
+
+if [[ -t 1 ]] # if on an interactive terminal
+then
+  echo "This script will clobber some system settings. Are you sure you want to"
+  echo -n "continue? "
+  while true
+  do
+    read -p "[yes/no] " ANSWER
+    ANSWER=$(echo "$ANSWER" | tr /a-z/ /A-Z/)
+    if [[ $ANSWER = YES ]]
+    then
+      break
+    elif [[ $ANSWER = NO ]]
+    then
+      echo "OK, Bye!"
+      exit 1
+    fi
+  done
+else
+  export DEBIAN_FRONTEND=noninteractive
+fi
+
+set -x
+
+source /etc/lsb-release
+
+if ! [[ $DISTRIB_ID = Ubuntu ]]
+then
+  echo "This script only supports Ubuntu" >&2
+  exit 1
+fi
+
+if ! [[ $DISTRIB_RELEASE = 14.04 || $DISTRIB_RELEASE = 16.04 ]]
+then
+  echo "This script only supports Ubuntu 14.04 and 16.04" >&2
+  exit 1
+fi
+
+REAL_APT_GET=$(which apt-get)
+function apt-get {
+  for ITER in $(seq 1 20); do
+    echo "ATTEMPT: ${ITER}"
+    if sudo "${REAL_APT_GET}" "$@"
+    then
+      return 0
+    fi
+    sleep "${ITER}"
+  done
+  echo "NO MORE RETRIES"
+  return 1
+}
+
+apt-get update
+apt-get --yes install apt-utils
+apt-get --yes install git
+
+# If there is no Impala git repo, get one now
+if ! [[ -d ~/Impala ]]
+then
+  time -p git clone https://git-wip-us.apache.org/repos/asf/incubator-impala.git ~/Impala
+fi
+cd ~/Impala
+SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"
+echo "$SET_IMPALA_HOME" >> ~/.bashrc
+eval "$SET_IMPALA_HOME"
+
+apt-get --yes install ccache g++ gcc libffi-dev liblzo2-dev libkrb5-dev libsasl2-dev \
+        libssl-dev make maven ninja-build ntp ntpdate python-dev python-setuptools \
+        postgresql ssh wget vim-common
+
+if ! { service --status-all | grep -E '^ \[ \+ \]  ssh$'; }
+then
+  sudo service ssh start
+fi
+
+# TODO: config ccache to give it plenty of space
+# TODO: check that there is enough space on disk to do a build and data load
+# TODO: make this work with non-bash shells
+
+JDK_VERSION=8
+if [[ $DISTRIB_RELEASE = 14.04 ]]
+then
+  JDK_VERSION=7
+fi
+apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source
+SET_JAVA_HOME="export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64"
+echo "$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
+eval "$SET_JAVA_HOME"
+
+sudo service ntp stop
+sudo ntpdate us.pool.ntp.org
+# If on EC2, use Amazon's ntp servers
+if which dmidecode && { sudo dmidecode -s bios-version | grep amazon; }
+then
+  sudo sed -i 's/ubuntu\.pool/amazon\.pool/' /etc/ntp.conf
+  grep amazon /etc/ntp.conf
+  grep ubuntu /etc/ntp.conf
+fi
+# While it is nice to have ntpd running to keep the clock in sync, that does not work in
a
+# --privileged docker container, and a non-privileged container cannot run ntpdate, which
+# is strictly needed by Kudu.
+# TODO: Make privileged docker start ntpd
+sudo service ntp start || grep docker /proc/1/cgroup
+
+# IMPALA-3932, IMPALA-3926
+if [[ $DISTRIB_RELEASE = 16.04 ]]
+then
+  SET_LD_LIBRARY_PATH='export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}'
+elif [[ $DISTRIB_RELEASE = 14.04 ]]
+then
+  SET_LD_LIBRARY_PATH="unset LD_LIBRARY_PATH"
+fi
+echo "$SET_LD_LIBRARY_PATH" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
+eval "$SET_LD_LIBRARY_PATH"
+
+# TODO: What are the security implications of this?
+for PG_AUTH_FILE in /etc/postgresql/*/main/pg_hba.conf
+do
+  sudo sed -ri 's/local +all +all +peer/local all all trust/g' $PG_AUTH_FILE
+done
+sudo service postgresql restart
+sudo /etc/init.d/postgresql reload
+sudo service postgresql restart
+
+# Set up postgress for HMS
+if ! [[ 1 = $(sudo -u postgres psql -At -c "SELECT count(*) FROM pg_roles WHERE rolname =
'hiveuser';") ]]
+then
+  sudo -u postgres psql -c "CREATE ROLE hiveuser LOGIN PASSWORD 'password';"
+fi
+sudo -u postgres psql -c "ALTER ROLE hiveuser WITH CREATEDB;"
+sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';"
+
+# Setup ssh to ssh to localhost
+mkdir -p ~/.ssh
+chmod go-rwx ~/.ssh
+if ! [[ -f ~/.ssh/id_rsa ]]
+then
+  ssh-keygen -t rsa -N '' -q -f ~/.ssh/id_rsa
+fi
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+echo "NoHostAuthenticationForLocalhost yes" >> ~/.ssh/config
+ssh localhost whoami
+
+# Workarounds for HDFS networking issues
+echo "127.0.0.1 $(hostname -s) $(hostname)" | sudo tee -a /etc/hosts
+# In Docker, one can change /etc/hosts as above but not with sed -i. The error message is
+# "sed: cannot rename /etc/sedc3gPj8: Device or resource busy". The following lines are
+# basically sed -i but with cp instead of mv for -i part.
+NEW_HOSTS=$(mktemp)
+sed 's/127.0.1.1/127.0.0.1/g' /etc/hosts > "${NEW_HOSTS}"
+diff -u /etc/hosts "${NEW_HOSTS}" || true
+sudo cp "${NEW_HOSTS}" /etc/hosts
+rm "${NEW_HOSTS}"
+
+sudo mkdir -p /var/lib/hadoop-hdfs
+sudo chown $(whoami) /var/lib/hadoop-hdfs/
+
+# TODO: restrict this to only the users it is needed for
+echo "* - nofile 1048576" | sudo tee -a /etc/security/limits.conf
+
+# LZO is not needed to compile or run Impala, but it is needed for the data load
+if ! [[ -d ~/Impala-lzo ]]
+then
+  git clone https://github.com/cloudera/impala-lzo.git ~/Impala-lzo
+fi
+if ! [[ -d ~/hadoop-lzo ]]
+then
+  git clone https://github.com/cloudera/hadoop-lzo.git ~/hadoop-lzo
+fi
+cd ~/hadoop-lzo/
+time -p ant package
+cd "$IMPALA_HOME"


Mime
View raw message