kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [15/50] incubator-kylin git commit: Use LongWritable as key type in fact distinct job.
Date Sat, 28 Mar 2015 00:04:52 GMT
Use LongWritable as key type in fact distinct job.

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d40a578
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d40a578
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d40a578

Branch: refs/heads/streaming-localdict
Commit: 8d40a578170da66c61503d7b42fe70c3a930dadd
Parents: 7658a50
Author: Shao Feng, Shi <shaoshi@ebay.com>
Authored: Thu Mar 26 23:32:56 2015 +0800
Committer: Shao Feng, Shi <shaoshi@ebay.com>
Committed: Thu Mar 26 23:32:56 2015 +0800

----------------------------------------------------------------------
 .../job/hadoop/cube/FactDistinctColumnsCombiner.java     |  6 +++---
 .../kylin/job/hadoop/cube/FactDistinctColumnsJob.java    |  4 ++--
 .../job/hadoop/cube/FactDistinctColumnsMapperBase.java   | 11 +++++------
 3 files changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
index c0cdd46..59ccd5a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
@@ -21,9 +21,9 @@ package org.apache.kylin.job.hadoop.cube;
 import java.io.IOException;
 import java.util.HashSet;
 
+import org.apache.hadoop.io.LongWritable;
 import org.apache.kylin.common.mr.KylinReducer;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.ShortWritable;
 import org.apache.hadoop.io.Text;
 
 import org.apache.kylin.common.util.ByteArray;
@@ -31,7 +31,7 @@ import org.apache.kylin.common.util.ByteArray;
 /**
  * @author yangli9
  */
-public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable, Text, ShortWritable,
Text> {
+public class FactDistinctColumnsCombiner extends KylinReducer<LongWritable, Text, LongWritable,
Text> {
 
     private Text outputValue = new Text();
 
@@ -41,7 +41,7 @@ public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable,
Tex
     }
 
     @Override
-    public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws
IOException, InterruptedException {
+    public void reduce(LongWritable key, Iterable<Text> values, Context context) throws
IOException, InterruptedException {
 
         if(key.get() >= 0) {
             HashSet<ByteArray> set = new HashSet<ByteArray>();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 17c5e9b..5903c7b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -22,8 +22,8 @@ import java.io.IOException;
 
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -103,7 +103,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
         job.setInputFormatClass(HCatInputFormat.class);
         job.setMapperClass(FactDistinctHiveColumnsMapper.class);
         job.setCombinerClass(FactDistinctColumnsCombiner.class);
-        job.setMapOutputKeyClass(ShortWritable.class);
+        job.setMapOutputKeyClass(LongWritable.class);
         job.setMapOutputValueClass(Text.class);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
index 9945769..2f046ab 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
@@ -1,12 +1,7 @@
 package org.apache.kylin.job.hadoop.cube;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.ShortWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hive.hcatalog.data.HCatRecord;
 import org.apache.kylin.common.KylinConfig;
@@ -21,6 +16,10 @@ import org.apache.kylin.job.constant.BatchConstants;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
 import org.apache.kylin.metadata.model.TblColRef;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * Created by Hongbin Ma(Binmahone) on 3/26/15.
  */
@@ -35,7 +34,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends
KylinMapper<K
 
     protected LongWritable outputKey = new LongWritable();
     protected Text outputValue = new Text();
-    protected int errorRecordCounter =0;
+    protected int errorRecordCounter = 0;
 
     @Override
     protected void setup(Context context) throws IOException {


Mime
View raw message