kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [02/10] kylin git commit: KYLIN-1323 further fix on read-write separation
Date Sun, 22 May 2016 08:29:42 GMT
KYLIN-1323 further fix on read-write separation


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa7558ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa7558ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa7558ab

Branch: refs/heads/master
Commit: aa7558ab7b53f94b1b926035eba7c66cabbd5779
Parents: c96f827
Author: Li Yang <liyang@apache.org>
Authored: Fri May 20 12:37:39 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Fri May 20 12:37:39 2016 +0800

----------------------------------------------------------------------
 .../kylin/storage/hbase/steps/CreateHTableJob.java  | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/aa7558ab/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 29bcb62..7a32e37 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -30,7 +30,11 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
@@ -44,7 +48,6 @@ import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.engine.mr.common.CuboidShardUtil;
@@ -280,7 +283,9 @@ public class CreateHTableJob extends AbstractHadoopJob {
             return;
         }
 
-        FileSystem fs = FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration());
+        // note read-write separation, respect HBase FS here
+        Configuration hbaseConf = HBaseConnection.getCurrentHBaseConfiguration();
+        FileSystem fs = FileSystem.get(hbaseConf);
         if (fs.exists(outputFolder) == false) {
             fs.mkdirs(outputFolder);
         }
@@ -294,7 +299,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
         if (hfileSizeMB > 0.0 && kylinConfig.isDevEnv()) {
             hfileSizeMB = mbPerRegion / 2;
         }
-        int compactionThreshold = Integer.valueOf(HBaseConnection.getCurrentHBaseConfiguration().get("hbase.hstore.compactionThreshold",
"3"));
+        int compactionThreshold = Integer.valueOf(hbaseConf.get("hbase.hstore.compactionThreshold",
"3"));
         logger.info("hbase.hstore.compactionThreshold is " + compactionThreshold);
         if (hfileSizeMB > 0.0  && hfileSizeMB * compactionThreshold < mbPerRegion)
{
             hfileSizeMB = mbPerRegion / compactionThreshold;
@@ -338,8 +343,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
 
         }
 
-        Configuration conf = HadoopUtil.getCurrentConfiguration();
-        SequenceFile.Writer hfilePartitionWriter = SequenceFile.createWriter(conf, SequenceFile.Writer.file(hfilePartitionFile),
SequenceFile.Writer.keyClass(ImmutableBytesWritable.class), SequenceFile.Writer.valueClass(NullWritable.class));
+        SequenceFile.Writer hfilePartitionWriter = SequenceFile.createWriter(hbaseConf, SequenceFile.Writer.file(hfilePartitionFile),
SequenceFile.Writer.keyClass(ImmutableBytesWritable.class), SequenceFile.Writer.valueClass(NullWritable.class));
 
         for (int i = 0; i < splits.size(); i++) {
             hfilePartitionWriter.append(new ImmutableBytesWritable(splits.get(i)), NullWritable.get());


Mime
View raw message