kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [22/50] [abbrv] kylin git commit: KYLIN-2605 Execute sample.sh script failure in hdfs federation environment
Date Fri, 19 May 2017 23:58:00 GMT
KYLIN-2605 Execute sample.sh script failure in hdfs federation environment

Signed-off-by: Billy Liu <billyliu@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d449c40d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d449c40d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d449c40d

Branch: refs/heads/KYLIN-2624
Commit: d449c40d58b8af10ebc638be4aadb80e09787a9d
Parents: b3310dc
Author: 10069681 <peng.jianhua@zte.com.cn>
Authored: Thu May 11 11:19:01 2017 +0800
Committer: Billy Liu <billyliu@apache.org>
Committed: Thu May 11 13:34:24 2017 +0800

----------------------------------------------------------------------
 build/bin/sample.sh                           | 25 ++++++++++++++++------
 examples/sample_cube/create_sample_tables.sql | 10 ++++-----
 2 files changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 617b671..10900bb 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -31,9 +31,20 @@ else
     hadoop_conf_param="--config ${kylin_hadoop_conf_dir}"
 fi
 
-echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
-hadoop ${hadoop_conf_param} fs -mkdir -p /tmp/kylin/sample_cube/data
-hadoop ${hadoop_conf_param} fs -put * /tmp/kylin/sample_cube/data/
+if [ -z "$1" ]; then
+    hdfs_tmp_dir=/tmp/kylin
+else
+    hdfs_tmp_dir=$1
+fi
+
+echo "Loading sample data into HDFS tmp path: ${hdfs_tmp_dir}/sample_cube/data"
+hadoop ${hadoop_conf_param} fs -mkdir -p ${hdfs_tmp_dir}/sample_cube/data
+if [ $? != 0 ]
+then
+    quit "Failed to create ${hdfs_tmp_dir}/sample_cube/data. Please make sure the user has
right to access ${hdfs_tmp_dir}/sample_cube/data or usage: sample.sh hdfs_tmp_dir"
+fi
+
+hadoop ${hadoop_conf_param} fs -put * ${hdfs_tmp_dir}/sample_cube/data/
 
 hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
 sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.database-for-flat-table`
@@ -51,14 +62,14 @@ then
     else
         beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}}
     fi
-    beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql  || {
exit 1; }
+    beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql
 || { exit 1; }
 else
     hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-    hive --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql
 || { exit 1; }
+    hive --database $sample_database --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql
 || { exit 1; }
 fi
 
 echo "Sample hive tables are created successfully; Going to create sample cube..."
-hadoop ${hadoop_conf_param} fs -rm -r /tmp/kylin/sample_cube
+hadoop ${hadoop_conf_param} fs -rm -r ${hdfs_tmp_dir}/sample_cube
 
 # set engine type and storage type to cube desc
 default_engine_type=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default`
@@ -85,4 +96,4 @@ ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & $sample_database\
 cd ${KYLIN_HOME}
 hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool
upload ${KYLIN_HOME}/sample_cube/metadata  || { exit 1; }
 echo "Sample cube is created successfully in project 'learn_kylin'."
-echo "Restart Kylin server or reload the metadata from web UI to see the change."
+echo "Restart Kylin server or reload the metadata from web UI to see the change."
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/d449c40d/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 24f26f8..c33a99e 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -216,8 +216,8 @@ COMMENT 'Sales order table, fact table'
 ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
 STORED AS TEXTFILE;
 
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE
KYLIN_SALES;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE
KYLIN_ACCOUNT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE
KYLIN_COUNTRY;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE
KYLIN_CAL_DT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE
INTO TABLE KYLIN_CATEGORY_GROUPINGS;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO
TABLE KYLIN_SALES;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO
TABLE KYLIN_ACCOUNT;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO
TABLE KYLIN_COUNTRY;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO
TABLE KYLIN_CAL_DT;
+LOAD DATA INPATH '${hdfs_tmp_dir}/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv'
OVERWRITE INTO TABLE KYLIN_CATEGORY_GROUPINGS;


Mime
View raw message