airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: Pin Hive and Hadoop to a specific version and create writable warehouse dir
Date Fri, 16 Jun 2017 12:39:06 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/v1-8-test 7cff6cde4 -> 33a9dcbb6


Pin Hive and Hadoop to a specific version and create writable warehouse dir

(cherry picked from commit 38b2747c5b50afc5f21af5b44e8a0ccf9a440559)
Signed-off-by: Bolke de Bruin <bolke@xs4all.nl>


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/33a9dcbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/33a9dcbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/33a9dcbb

Branch: refs/heads/v1-8-test
Commit: 33a9dcbb673713aecdca6febf04ab422c447ed95
Parents: 7cff6cd
Author: Bolke de Bruin <bolke@xs4all.nl>
Authored: Wed Jun 14 18:19:10 2017 -0400
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Fri Jun 16 08:04:58 2017 -0400

----------------------------------------------------------------------
 scripts/ci/setup_env.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/33a9dcbb/scripts/ci/setup_env.sh
----------------------------------------------------------------------
diff --git a/scripts/ci/setup_env.sh b/scripts/ci/setup_env.sh
index e093ff4..1a6d242 100755
--- a/scripts/ci/setup_env.sh
+++ b/scripts/ci/setup_env.sh
@@ -77,11 +77,16 @@ mkdir -p ${TRAVIS_CACHE}/minicluster
 mkdir -p ${TRAVIS_CACHE}/hive
 mkdir -p ${HIVE_HOME}
 chmod -R 777 ${HIVE_HOME}
-mkdir -p /user/hive/warehouse
+sudo mkdir -p /user/hive/warehouse
+sudo chown -R ${USER} /user/
+sudo chmod -R 777 /user/
+ls -l /
 
 if [ $HADOOP_DISTRO = "cdh" ]; then
-    URL="http://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz"
-    HIVE_URL="http://archive.cloudera.com/cdh5/cdh/5/hive-latest.tar.gz"
+    # URL="http://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz"
+    URL="https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.11.0.tar.gz"
+    # HIVE_URL="http://archive.cloudera.com/cdh5/cdh/5/hive-latest.tar.gz"
+    HIVE_URL="https://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.11.0.tar.gz"
 elif [ $HADOOP_DISTRO = "hdp" ]; then
     URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/tars/hadoop-2.7.1.2.3.2.0-2950.tar.gz"
     HIVE_URL="http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/tars/apache-hive-1.2.1.2.3.2.0-2950-bin.tar.gz"
@@ -105,6 +110,8 @@ if $ONLY_DOWNLOAD; then
 fi
 
 echo "Extracting ${HADOOP_HOME}/hadoop.tar.gz into $HADOOP_HOME"
+# TODO: remove this below after first successful build
+rm -rf ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz
 tar zxf ${TRAVIS_CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz --strip-components 1 -C $HADOOP_HOME
 
 if [ $? != 0 ]; then
@@ -121,6 +128,8 @@ fi
 
 echo "Downloading and unpacking hive"
 curl -z ${TRAVIS_CACHE}/hive/hive.tar.gz -o ${TRAVIS_CACHE}/hive/hive.tar.gz -L ${HIVE_URL}
+# TODO: remove this below after first successful build
+rm -rf ${TRAVIS_CACHE}/hive/hive.tar.gz
 tar zxf ${TRAVIS_CACHE}/hive/hive.tar.gz --strip-components 1 -C ${HIVE_HOME}
 
 if [ $? != 0 ]; then


Mime
View raw message