kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [03/12] kylin git commit: minor, move JobInfoConverter to kylin-engine-mr
Date Fri, 31 Mar 2017 12:23:18 GMT
minor, move JobInfoConverter to kylin-engine-mr


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/598b32ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/598b32ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/598b32ef

Branch: refs/heads/master
Commit: 598b32efa21ebd87e76ac5798b30f472c6e29de2
Parents: 997d429
Author: Yifan Zhang <event.dimlas@gmail.com>
Authored: Wed Mar 29 10:29:16 2017 +0800
Committer: Dong Li <lidong@apache.org>
Committed: Wed Mar 29 10:47:10 2017 +0800

----------------------------------------------------------------------
 .../engine/mr/common/JobInfoConverter.java      | 128 ++++++++++++++++++
 .../apache/kylin/rest/service/JobService.java   |   2 +-
 .../kylin/rest/util/JobInfoConverter.java       | 130 -------------------
 3 files changed, 129 insertions(+), 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
new file mode 100644
index 0000000..189e019
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.util.Map;
+
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.JobStatusEnum;
+import org.apache.kylin.job.constant.JobStepStatusEnum;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.Output;
+
+import com.google.common.base.Preconditions;
+
+public class JobInfoConverter {
+    public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output>
outputs) {
+        if (job == null) {
+            return null;
+        }
+        Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
+        CubingJob cubeJob = (CubingJob) job;
+        Output output = outputs.get(job.getId());
+        final JobInstance result = new JobInstance();
+        result.setName(job.getName());
+        result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+        result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
+        result.setLastModified(output.getLastModified());
+        result.setSubmitter(cubeJob.getSubmitter());
+        result.setUuid(cubeJob.getId());
+        result.setType(CubeBuildTypeEnum.BUILD);
+        result.setStatus(parseToJobStatus(output.getState()));
+        result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME,
0L) / 1000);
+        result.setExecStartTime(AbstractExecutable.getStartTime(output));
+        result.setExecEndTime(AbstractExecutable.getEndTime(output));
+        result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime())
/ 1000);
+        for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
+            AbstractExecutable task = cubeJob.getTasks().get(i);
+            result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
+        }
+        return result;
+    }
+
+    public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output
stepOutput) {
+        Preconditions.checkNotNull(stepOutput);
+        JobInstance.JobStep result = new JobInstance.JobStep();
+        result.setId(task.getId());
+        result.setName(task.getName());
+        result.setSequenceID(i);
+        result.setStatus(parseToJobStepStatus(stepOutput.getState()));
+        for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
+            if (entry.getKey() != null && entry.getValue() != null) {
+                result.putInfo(entry.getKey(), entry.getValue());
+            }
+        }
+        result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
+        result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
+        if (task instanceof ShellExecutable) {
+            result.setExecCmd(((ShellExecutable) task).getCmd());
+        }
+        if (task instanceof MapReduceExecutable) {
+            result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
+            result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME,
0L) / 1000);
+        }
+        if (task instanceof HadoopShellExecutable) {
+            result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
+        }
+        return result;
+    }
+
+    public static JobStatusEnum parseToJobStatus(ExecutableState state) {
+        switch (state) {
+        case READY:
+            return JobStatusEnum.PENDING;
+        case RUNNING:
+            return JobStatusEnum.RUNNING;
+        case ERROR:
+            return JobStatusEnum.ERROR;
+        case DISCARDED:
+            return JobStatusEnum.DISCARDED;
+        case SUCCEED:
+            return JobStatusEnum.FINISHED;
+        case STOPPED:
+            return JobStatusEnum.STOPPED;
+        default:
+            throw new RuntimeException("invalid state:" + state);
+        }
+    }
+
+    public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
+        switch (state) {
+        case READY:
+            return JobStepStatusEnum.PENDING;
+        case RUNNING:
+            return JobStepStatusEnum.RUNNING;
+        case ERROR:
+            return JobStepStatusEnum.ERROR;
+        case DISCARDED:
+            return JobStepStatusEnum.DISCARDED;
+        case SUCCEED:
+            return JobStepStatusEnum.FINISHED;
+        case STOPPED:
+            return JobStepStatusEnum.STOPPED;
+        default:
+            throw new RuntimeException("invalid state:" + state);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1e91b43..4ba426e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -37,6 +37,7 @@ import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.cube.model.CubeBuildTypeEnum;
 import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.JobInfoConverter;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.Scheduler;
@@ -55,7 +56,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.rest.constant.Constant;
 import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.util.JobInfoConverter;
 import org.apache.kylin.source.ISource;
 import org.apache.kylin.source.SourceFactory;
 import org.apache.kylin.source.SourcePartition;

http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java b/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
deleted file mode 100644
index e13c9c3..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.util.Map;
-
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
-
-import com.google.common.base.Preconditions;
-
-public class JobInfoConverter {
-    public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output>
outputs) {
-        if (job == null) {
-            return null;
-        }
-        Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
-        CubingJob cubeJob = (CubingJob) job;
-        Output output = outputs.get(job.getId());
-        final JobInstance result = new JobInstance();
-        result.setName(job.getName());
-        result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
-        result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
-        result.setLastModified(output.getLastModified());
-        result.setSubmitter(cubeJob.getSubmitter());
-        result.setUuid(cubeJob.getId());
-        result.setType(CubeBuildTypeEnum.BUILD);
-        result.setStatus(parseToJobStatus(output.getState()));
-        result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME,
0L) / 1000);
-        result.setExecStartTime(AbstractExecutable.getStartTime(output));
-        result.setExecEndTime(AbstractExecutable.getEndTime(output));
-        result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime())
/ 1000);
-        for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
-            AbstractExecutable task = cubeJob.getTasks().get(i);
-            result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
-        }
-        return result;
-    }
-
-    public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output
stepOutput) {
-        Preconditions.checkNotNull(stepOutput);
-        JobInstance.JobStep result = new JobInstance.JobStep();
-        result.setId(task.getId());
-        result.setName(task.getName());
-        result.setSequenceID(i);
-        result.setStatus(parseToJobStepStatus(stepOutput.getState()));
-        for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
-            if (entry.getKey() != null && entry.getValue() != null) {
-                result.putInfo(entry.getKey(), entry.getValue());
-            }
-        }
-        result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
-        result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
-        if (task instanceof ShellExecutable) {
-            result.setExecCmd(((ShellExecutable) task).getCmd());
-        }
-        if (task instanceof MapReduceExecutable) {
-            result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
-            result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME,
0L) / 1000);
-        }
-        if (task instanceof HadoopShellExecutable) {
-            result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
-        }
-        return result;
-    }
-
-    public static JobStatusEnum parseToJobStatus(ExecutableState state) {
-        switch (state) {
-        case READY:
-            return JobStatusEnum.PENDING;
-        case RUNNING:
-            return JobStatusEnum.RUNNING;
-        case ERROR:
-            return JobStatusEnum.ERROR;
-        case DISCARDED:
-            return JobStatusEnum.DISCARDED;
-        case SUCCEED:
-            return JobStatusEnum.FINISHED;
-        case STOPPED:
-            return JobStatusEnum.STOPPED;
-        default:
-            throw new RuntimeException("invalid state:" + state);
-        }
-    }
-
-    public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
-        switch (state) {
-        case READY:
-            return JobStepStatusEnum.PENDING;
-        case RUNNING:
-            return JobStepStatusEnum.RUNNING;
-        case ERROR:
-            return JobStepStatusEnum.ERROR;
-        case DISCARDED:
-            return JobStepStatusEnum.DISCARDED;
-        case SUCCEED:
-            return JobStepStatusEnum.FINISHED;
-        case STOPPED:
-            return JobStepStatusEnum.STOPPED;
-        default:
-            throw new RuntimeException("invalid state:" + state);
-        }
-    }
-}


Mime
View raw message