kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [01/10] kylin git commit: minor, add extract all projects command [Forced Update!]
Date Wed, 15 Mar 2017 09:46:41 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang23-hbase0.98 6b54dbdeb -> 33534515f (forced update)


minor, add extract all projects command

Signed-off-by: Hongbin Ma <mahongbin@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfbffa4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfbffa4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfbffa4

Branch: refs/heads/yang23-hbase0.98
Commit: 2dfbffa46eeb27b4c6aec982116912a545ca55f8
Parents: 17a4bcf
Author: Roger Shi <rogershijicheng@hotmail.com>
Authored: Tue Mar 14 19:22:58 2017 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Tue Mar 14 19:27:00 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/tool/CubeMetaExtractor.java    | 45 +++++++++++++-------
 1 file changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfbffa4/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 35831f7..e370e48 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,8 +23,6 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.OptionGroup;
@@ -52,6 +50,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.project.RealizationEntry;
 import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.metadata.streaming.StreamingConfig;
 import org.apache.kylin.metadata.streaming.StreamingManager;
@@ -61,8 +60,10 @@ import org.apache.kylin.storage.hybrid.HybridManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -80,6 +81,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
     private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify
which hybrid to extract").create("hybrid");
     @SuppressWarnings("static-access")
     private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify
realizations in which project to extract").create("project");
+    @SuppressWarnings("static-access")
+    private static final Option OPTION_All_PROJECT = OptionBuilder.withArgName("allProjects").hasArg(false).isRequired(false).withDescription("Specify
realizations in all projects to extract").create("allProjects");
 
     @SuppressWarnings("static-access")
     private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify
the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@@ -130,6 +133,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
         realizationOrProject.addOption(OPTION_CUBE);
         realizationOrProject.addOption(OPTION_PROJECT);
         realizationOrProject.addOption(OPTION_HYBRID);
+        realizationOrProject.addOption(OPTION_All_PROJECT);
         realizationOrProject.setRequired(true);
 
         options.addOptionGroup(realizationOrProject);
@@ -160,23 +164,15 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
         realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
         badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
 
-        if (optionsHelper.hasOption(OPTION_PROJECT)) {
+        if (optionsHelper.hasOption(OPTION_All_PROJECT)) {
+            for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
+                requireProject(projectInstance);
+            }
+        } else if (optionsHelper.hasOption(OPTION_PROJECT)) {
             String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
             for (String projectName : projectNames.split(",")) {
                 ProjectInstance projectInstance = projectManager.getProject(projectName);
-                if (projectInstance == null) {
-                    throw new IllegalArgumentException("Project " + projectName + " does
not exist");
-                }
-                addRequired(projectInstance.getResourcePath());
-                List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
-                for (RealizationEntry realizationEntry : realizationEntries) {
-                    retrieveResourcePath(getRealization(realizationEntry));
-                }
-                List<DataModelDesc> modelDescs = metadataManager.getModels(projectName);
-                for (DataModelDesc modelDesc : modelDescs) {
-                    addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
-                }
-                addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
+                requireProject(projectInstance);
             }
         } else if (optionsHelper.hasOption(OPTION_CUBE)) {
             String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
@@ -205,6 +201,22 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
         engineOverwrite(new File(exportDir.getAbsolutePath()));
     }
 
+    private void requireProject(ProjectInstance projectInstance) throws IOException {
+        if (projectInstance == null) {
+            throw new IllegalArgumentException("Project " + projectInstance.getName() + "
does not exist");
+        }
+        addRequired(projectInstance.getResourcePath());
+        List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+        for (RealizationEntry realizationEntry : realizationEntries) {
+            retrieveResourcePath(getRealization(realizationEntry));
+        }
+        List<DataModelDesc> modelDescs = metadataManager.getModels(projectInstance.getName());
+        for (DataModelDesc modelDesc : modelDescs) {
+            addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+        }
+        addOptional(badQueryHistoryManager.getBadQueriesForProject(projectInstance.getName()).getResourcePath());
+    }
+
     private void executeExtraction(String dest) {
         logger.info("The resource paths going to be extracted:");
         for (String s : requiredResources) {
@@ -359,6 +371,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
                     logger.warn("It's useless to set includeJobs to true when includeSegments
is set to false");
                 }
 
+                cube.setStatus(RealizationStatusEnum.DISABLED);
                 cubesToTrimAndSave.add(cube);
             }
         } else if (realization instanceof HybridInstance) {


Mime
View raw message