kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [6/9] incubator-kylin git commit: KYLIN-968 CubeSegment.lastBuildJobID is null in new instance but used for rowkey_stats path
Date Fri, 18 Sep 2015 23:44:15 GMT
KYLIN-968 CubeSegment.lastBuildJobID is null in new instance but used for rowkey_stats path

Signed-off-by: shaofengshi <shaofengshi@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8fd1404d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8fd1404d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8fd1404d

Branch: refs/heads/1.x
Commit: 8fd1404dd748b6761ef2e51c197ba5ef4c56c11c
Parents: b003921
Author: sunyerui <sunyerui@gmail.com>
Authored: Thu Sep 10 11:23:16 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Wed Sep 16 21:01:59 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/job/cube/CubingJobBuilder.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8fd1404d/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index 5c3c277..ff79286 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -206,9 +206,9 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
         final String jobId = result.getId();
         final String cuboidPath = cuboidRootPath + "*";
 
-        result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath));
+        result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath, jobId));
         // create htable step
-        result.addTask(createCreateHTableStep(seg));
+        result.addTask(createCreateHTableStep(seg, jobId));
         // generate hfiles step
         final MapReduceExecutable convertCuboidToHfileStep = createConvertCuboidToHfileStep(seg,
cuboidPath, jobId);
         result.addTask(convertCuboidToHfileStep);
@@ -265,8 +265,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
         return getJobWorkingDir(seg.getLastBuildJobID()) + "/" + seg.getCubeInstance().getName()
+ "/cuboid/*";
     }
 
-    private String getRowkeyDistributionOutputPath(CubeSegment seg) {
-        return getJobWorkingDir(seg.getLastBuildJobID()) + "/" + seg.getCubeInstance().getName()
+ "/rowkey_stats";
+    private String getRowkeyDistributionOutputPath(CubeSegment seg, String jobId) {
+        return getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats";
     }
 
     private String getFactDistinctColumnsPath(CubeSegment seg, String jobUuid) {
@@ -347,14 +347,14 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
         return ndCuboidStep;
     }
 
-    private MapReduceExecutable createRangeRowkeyDistributionStep(CubeSegment seg, String
inputPath) {
+    private MapReduceExecutable createRangeRowkeyDistributionStep(CubeSegment seg, String
inputPath, String jobId) {
         MapReduceExecutable rowkeyDistributionStep = new MapReduceExecutable();
         rowkeyDistributionStep.setName(ExecutableConstants.STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION);
         StringBuilder cmd = new StringBuilder();
 
         appendMapReduceParameters(cmd, seg);
         appendExecCmdParameters(cmd, "input", inputPath);
-        appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(seg));
+        appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(seg, jobId));
         appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
         appendExecCmdParameters(cmd, "jobname", "Kylin_Region_Splits_Calculator_" + seg.getCubeInstance().getName()
+ "_Step");
 
@@ -363,12 +363,12 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
         return rowkeyDistributionStep;
     }
 
-    private HadoopShellExecutable createCreateHTableStep(CubeSegment seg) {
+    private HadoopShellExecutable createCreateHTableStep(CubeSegment seg, String jobId) {
         HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
         createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
         StringBuilder cmd = new StringBuilder();
         appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
-        appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg) + "/part-r-00000");
+        appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg, jobId)
+ "/part-r-00000");
         appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
 
         createHtableStep.setJobParams(cmd.toString());


Mime
View raw message