kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [5/5] kylin git commit: KYLIN-2187 code review
Date Wed, 16 Nov 2016 03:07:40 GMT
KYLIN-2187 code review


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73ec4a4c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73ec4a4c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73ec4a4c

Branch: refs/heads/master
Commit: 73ec4a4c20f8d8cf372a15d2100c7bfd919b4ba5
Parents: 6b35e5e
Author: Li Yang <liyang@apache.org>
Authored: Wed Nov 16 11:07:31 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Wed Nov 16 11:07:31 2016 +0800

----------------------------------------------------------------------
 .../kylin/metadata/model/TableExtDesc.java      | 20 +-----
 .../apache/kylin/rest/service/BasicService.java | 70 --------------------
 .../apache/kylin/rest/service/CubeService.java  | 11 +--
 .../apache/kylin/rest/service/JobService.java   | 59 +++++++++++++++++
 4 files changed, 69 insertions(+), 91 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index fc2edd4..0086917 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -26,8 +26,6 @@ import java.util.Map;
 
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonBackReference;
@@ -36,18 +34,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 
 @SuppressWarnings("serial")
 @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE,
isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class TableExtDesc extends RootPersistentEntity implements ISourceAware {
-
-    private static final Logger logger = LoggerFactory.getLogger(TableExtDesc.class);
+public class TableExtDesc extends RootPersistentEntity {
 
     @JsonProperty("table_name")
     private String tableName;
     @JsonProperty("last_build_job_id")
     private String jodID;
 
-    @JsonProperty("source_type")
-    private int sourceType;
-
     @JsonProperty("columns_stats")
     private List<ColumnStats> columnStats = new ArrayList<>();
 
@@ -66,6 +59,8 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware
{
     private String partitionColumn;
     @JsonProperty("total_file_size")
     private String totalFileSize;
+    @JsonProperty("total_rows")
+    private String totalRows;
     @JsonProperty("data_source_properties")
     private Map<String, String> dataSourceProps = new HashMap<>();
 
@@ -136,10 +131,6 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware
{
         }
     }
 
-    public void setSourceType(int sourceType) {
-        this.sourceType = sourceType;
-    }
-
     public List<ColumnStats> getColumnStats() {
         return this.columnStats;
     }
@@ -231,11 +222,6 @@ public class TableExtDesc extends RootPersistentEntity implements ISourceAware
{
         return "TableExtDesc{" + "name='" + (null == tableName ? "NULL" : tableName) + '\''
+ ", columns_samples=" + (null == columnStats ? "null" : Arrays.toString(columnStats.toArray()));
     }
 
-    @Override
-    public int getSourceType() {
-        return -1;
-    }
-
     @JsonIgnoreProperties(ignoreUnknown = true)
     public static class ColumnStats implements Comparable<ColumnStats> {
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index d83ae28..3091325 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -19,20 +19,11 @@
 package org.apache.kylin.rest.service;
 
 import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.execution.AbstractExecutable;
 import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
 import org.apache.kylin.metadata.project.ProjectManager;
@@ -40,12 +31,6 @@ import org.apache.kylin.metadata.streaming.StreamingManager;
 import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.storage.hybrid.HybridManager;
 
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Lists;
-
 public abstract class BasicService {
 
     public KylinConfig getConfig() {
@@ -94,59 +79,4 @@ public abstract class BasicService {
         return BadQueryHistoryManager.getInstance(getConfig());
     }
 
-    protected List<CubingJob> listAllCubingJobs(final String cubeName, final String
projectName, final Set<ExecutableState> statusList, final Map<String, Output>
allOutputs) {
-        return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs);
-    }
-
-    protected List<CubingJob> listAllCubingJobs(final String cubeName, final String
projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis,
final Map<String, Output> allOutputs) {
-        List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables(timeStartInMillis,
timeEndInMillis)).filter(new Predicate<AbstractExecutable>() {
-            @Override
-            public boolean apply(AbstractExecutable executable) {
-                if (executable instanceof CubingJob) {
-                    if (cubeName == null) {
-                        return true;
-                    }
-                    return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName);
-                } else {
-                    return false;
-                }
-            }
-        }).transform(new Function<AbstractExecutable, CubingJob>() {
-            @Override
-            public CubingJob apply(AbstractExecutable executable) {
-                return (CubingJob) executable;
-            }
-        }).filter(Predicates.and(new Predicate<CubingJob>() {
-            @Override
-            public boolean apply(CubingJob executable) {
-                if (null == projectName || null == getProjectManager().getProject(projectName))
{
-                    return true;
-                } else {
-                    return projectName.equals(executable.getParams().get("projectName"));
-                }
-            }
-        }, new Predicate<CubingJob>() {
-            @Override
-            public boolean apply(CubingJob executable) {
-                try {
-                    Output output = allOutputs.get(executable.getId());
-                    ExecutableState state = output.getState();
-                    boolean ret = statusList.contains(state);
-                    return ret;
-                } catch (Exception e) {
-                    throw e;
-                }
-            }
-        })));
-        return results;
-    }
-
-    protected List<CubingJob> listAllCubingJobs(final String cubeName, final String
projectName, final Set<ExecutableState> statusList) {
-        return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs());
-    }
-
-    protected List<CubingJob> listAllCubingJobs(final String cubeName, final String
projectName) {
-        return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class),
getExecutableManager().getAllOutputs());
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index c95f0ca..5252f59 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -94,6 +94,9 @@ public class CubeService extends BasicService {
     @Autowired
     private AccessService accessService;
 
+    @Autowired
+    private JobService jobService;
+    
     @PostFilter(Constant.ACCESS_POST_FILTER_READ)
     public List<CubeInstance> listAllCubes(final String cubeName, final String projectName,
final String modelName) {
         List<CubeInstance> cubeInstances = null;
@@ -228,7 +231,7 @@ public class CubeService extends BasicService {
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')
or hasPermission(#cube, 'MANAGEMENT')")
     public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName,
boolean forceUpdate) throws IOException, JobException {
 
-        final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+        final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(),
null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
         if (!cubingJobs.isEmpty()) {
             throw new JobException("Cube schema shouldn't be changed with running job.");
         }
@@ -258,7 +261,7 @@ public class CubeService extends BasicService {
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')
or hasPermission(#cube, 'MANAGEMENT')")
     public void deleteCube(CubeInstance cube) throws IOException, JobException {
-        final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+        final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(),
null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
         if (!cubingJobs.isEmpty()) {
             throw new JobException("The cube " + cube.getName() + " has running job, please
discard it and try again.");
         }
@@ -358,7 +361,7 @@ public class CubeService extends BasicService {
             throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY
segment");
         }
 
-        final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+        final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(),
null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
         if (!cubingJobs.isEmpty()) {
             throw new JobException("Enable is not allowed with a running job.");
         }
@@ -528,7 +531,7 @@ public class CubeService extends BasicService {
     }
 
     private void releaseAllJobs(CubeInstance cube) {
-        final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
+        final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(),
null);
         for (CubingJob cubingJob : cubingJobs) {
             final ExecutableState status = cubingJob.getStatus();
             if (status != ExecutableState.SUCCEED && status != ExecutableState.STOPPED
&& status != ExecutableState.DISCARDED) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/73ec4a4c/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 7383c52..fa578fe 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -75,6 +75,8 @@ import org.springframework.stereotype.Component;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -502,4 +504,61 @@ public class JobService extends BasicService implements InitializingBean
{
         }
         return serverName;
     }
+    
+    public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName,
final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
+        return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs);
+    }
+
+    public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName,
final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis,
final Map<String, Output> allOutputs) {
+        List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables(timeStartInMillis,
timeEndInMillis)).filter(new Predicate<AbstractExecutable>() {
+            @Override
+            public boolean apply(AbstractExecutable executable) {
+                if (executable instanceof CubingJob) {
+                    if (cubeName == null) {
+                        return true;
+                    }
+                    return CubingExecutableUtil.getCubeName(executable.getParams()).equalsIgnoreCase(cubeName);
+                } else {
+                    return false;
+                }
+            }
+        }).transform(new Function<AbstractExecutable, CubingJob>() {
+            @Override
+            public CubingJob apply(AbstractExecutable executable) {
+                return (CubingJob) executable;
+            }
+        }).filter(Predicates.and(new Predicate<CubingJob>() {
+            @Override
+            public boolean apply(CubingJob executable) {
+                if (null == projectName || null == getProjectManager().getProject(projectName))
{
+                    return true;
+                } else {
+                    return projectName.equals(executable.getProjectName());
+                }
+            }
+        }, new Predicate<CubingJob>() {
+            @Override
+            public boolean apply(CubingJob executable) {
+                try {
+                    Output output = allOutputs.get(executable.getId());
+                    ExecutableState state = output.getState();
+                    boolean ret = statusList.contains(state);
+                    return ret;
+                } catch (Exception e) {
+                    throw e;
+                }
+            }
+        })));
+        return results;
+    }
+
+    public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName,
final Set<ExecutableState> statusList) {
+        return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs());
+    }
+
+    public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName)
{
+        return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class),
getExecutableManager().getAllOutputs());
+    }
+
+
 }


Mime
View raw message