kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject kylin git commit: KYLIN-1402 StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job
Date Fri, 05 Feb 2016 08:40:40 GMT
Repository: kylin
Updated Branches:
  refs/heads/1.x-staging 474dbfa67 -> 85d55f4f8


KYLIN-1402 StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d55f4f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d55f4f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d55f4f

Branch: refs/heads/1.x-staging
Commit: 85d55f4f8e4de440c7764cd1820917ddbef3836b
Parents: 474dbfa
Author: shaofengshi <shaofengshi@apache.org>
Authored: Fri Feb 5 16:35:25 2016 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Fri Feb 5 16:35:25 2016 +0800

----------------------------------------------------------------------
 .../HiveColumnCardinalityUpdateJob.java         | 44 +++++++++++---------
 1 file changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/85d55f4f/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
index 115753e..cad3c0b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -18,14 +18,6 @@
 
 package org.apache.kylin.job.hadoop.cardinality;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
@@ -41,11 +33,21 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
 import org.apache.kylin.metadata.MetadataConstants;
 import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 /**
  * This job will update save the cardinality result into Kylin table metadata store.
- * @author shaoshi
  *
+ * @author shaoshi
  */
 public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
     public static final String JOB_TITLE = "Kylin Hive Column Cardinality Update Job";
@@ -53,6 +55,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
     @SuppressWarnings("static-access")
     protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The
hive table name").create("table");
 
+    private static final Logger logger = LoggerFactory.getLog(HiveColumnCardinalityUpdateJob.class);
     private String table;
 
     public HiveColumnCardinalityUpdateJob() {
@@ -73,7 +76,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
             this.table = getOptionValue(OPTION_TABLE).toUpperCase();
             // start job
             String jobName = JOB_TITLE + getOptionsAsString();
-            System.out.println("Starting: " + jobName);
+            logger.info("Starting: " + jobName);
             Configuration conf = getConf();
             Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
 
@@ -91,30 +94,33 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob
{
         try {
             columns = readLines(new Path(outPath), config);
         } catch (Exception e) {
-            e.printStackTrace();
-            System.out.println("Failed to resolve cardinality for " + tableName + " from
" + outPath);
+            logger.error("Failed to resolve cardinality for " + tableName + " from " + outPath,
e);
             return;
         }
 
         StringBuffer cardi = new StringBuffer();
         Iterator<String> it = columns.iterator();
         while (it.hasNext()) {
-            String string = (String) it.next();
+            String string = it.next();
             String[] ss = StringUtils.split(string, "\t");
 
             if (ss.length != 2) {
-                System.out.println("The hadoop cardinality value is not valid " + string);
+                logger.info("The hadoop cardinality value is not valid " + string);
                 continue;
             }
             cardi.append(ss[1]);
             cardi.append(",");
         }
         String scardi = cardi.toString();
-        scardi = scardi.substring(0, scardi.length() - 1);
-        MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-        Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
-        tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
-        metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+        if (scardi.length() > 0) {
+            scardi = scardi.substring(0, scardi.length() - 1);
+            MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+            Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
+            tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
+            metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+        } else {
+            throw new IllegalArgumentException("No cardinality data is collected for table
" + tableName);
+        }
     }
 
     private static List<String> readLines(Path location, Configuration conf) throws
Exception {


Mime
View raw message