kylin-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (KYLIN-3259) When a cube is deleted, remove it from the hybrid cube definition
Date Thu, 26 Jul 2018 01:45:00 GMT

    [ https://issues.apache.org/jira/browse/KYLIN-3259?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16556763#comment-16556763
] 

ASF GitHub Bot commented on KYLIN-3259:
---------------------------------------

shaofengshi closed pull request #176: KYLIN-3259 when delete cube, remove it from hybrid
URL: https://github.com/apache/kylin/pull/176
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 9fbb7f3fbe..286af5f360 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -108,7 +108,7 @@ private void init() {
                 return;
 
             if (realizationEntries == null || realizationEntries.size() == 0)
-                throw new IllegalArgumentException();
+                return;
 
             RealizationRegistry registry = RealizationRegistry.getInstance(config);
             List<IRealization> realizationList = Lists.newArrayList();
@@ -301,7 +301,7 @@ public boolean supportsLimitPushDown() {
 
     public IRealization[] getRealizations() {
         init();
-        return realizations;
+        return realizations == null ? new IRealization[0] : realizations;
     }
 
     public String getResourcePath() {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index 72ae1ce270..1c8a46acb9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -65,6 +65,8 @@
 
     private static final Option OPTION_CUBES = OptionBuilder.withArgName("cubes").hasArg().isRequired(false).withDescription("the
cubes used in HybridCube, seperated by comma, empty if to delete HybridCube").create("cubes");
 
+    private static final Option OPTION_CHECK = OptionBuilder.withArgName("check").hasArg().isRequired(false).withDescription("whether
to check cube size, default true.").create("check");
+
     private final Options options;
 
     private KylinConfig kylinConfig;
@@ -80,6 +82,7 @@ public HybridCubeCLI() {
         options.addOption(OPTION_PROJECT);
         options.addOption(OPTION_MODEL);
         options.addOption(OPTION_CUBES);
+        options.addOption(OPTION_CHECK);
 
         this.kylinConfig = KylinConfig.getInstanceFromEnv();
         this.store = ResourceStore.getStore(kylinConfig);
@@ -105,6 +108,8 @@ protected void execute(OptionsHelper optionsHelper) throws Exception {
         String projectName = optionsHelper.getOptionValue(OPTION_PROJECT);
         String modelName = optionsHelper.getOptionValue(OPTION_MODEL);
         String cubeNamesStr = optionsHelper.getOptionValue(OPTION_CUBES);
+        boolean checkCubeSize = optionsHelper.hasOption(OPTION_CHECK) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_CHECK))
: true;
+
         String[] cubeNames = new String[] {};
         if (cubeNamesStr != null)
             cubeNames = cubeNamesStr.split(",");
@@ -141,7 +146,7 @@ protected void execute(OptionsHelper optionsHelper) throws Exception {
                 throw new RuntimeException("The Hybrid Cube doesn't exist, could not update:
" + hybridName);
             }
             // Update the Hybrid
-            update(hybridInstance, realizationEntries, projectName, owner);
+            update(hybridInstance, realizationEntries, projectName, owner, checkCubeSize);
         } else if ("delete".equals(action)) {
             if (hybridInstance == null) {
                 throw new RuntimeException("The Hybrid Cube doesn't exist, could not delete:
" + hybridName);
@@ -162,8 +167,9 @@ private HybridInstance create(String hybridName, List<RealizationEntry>
realizat
         return hybridInstance;
     }
 
-    private void update(HybridInstance hybridInstance, List<RealizationEntry> realizationEntries,
String projectName, String owner) throws IOException {
-        checkSegmentOffset(realizationEntries);
+    private void update(HybridInstance hybridInstance, List<RealizationEntry> realizationEntries,
String projectName, String owner, boolean checkCubeSize) throws IOException {
+        if (checkCubeSize)
+            checkSegmentOffset(realizationEntries);
         hybridInstance.setRealizationEntries(realizationEntries);
         store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
         ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID,
hybridInstance.getName(), projectName, owner);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index da907719fa..8608bafc41 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -73,6 +73,7 @@
 import org.apache.kylin.rest.response.MetricsResponse;
 import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.rest.util.ValidateUtil;
+import org.apache.kylin.storage.hybrid.HybridInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -117,6 +118,9 @@
     @Autowired
     private AclEvaluate aclEvaluate;
 
+    @Autowired
+    private HybridService hybridService;
+
     public boolean isCubeNameVaildate(final String cubeName) {
         if (StringUtils.isEmpty(cubeName) || !ValidateUtil.isAlphanumericUnderscore(cubeName))
{
             return false;
@@ -302,6 +306,34 @@ public void deleteCube(CubeInstance cube) throws IOException {
             //ignore the exception
         }
 
+        // remove from hybrid definition
+        ProjectInstance projectInstance = cube.getProjectInstance();
+        List<RealizationEntry> hybridRealizationEntries = projectInstance.getRealizationEntries(RealizationType.HYBRID);
+
+        if (hybridRealizationEntries != null) {
+            for (RealizationEntry entry : hybridRealizationEntries) {
+                HybridInstance instance = getHybridManager().getHybridInstance(entry.getRealization());
+                List<RealizationEntry> cubeRealizationEntries = instance.getRealizationEntries();
+
+                boolean needUpdateHybrid = false;
+                for (RealizationEntry cubeRealizationEntry : cubeRealizationEntries){
+                    if (cube.getName().equals(cubeRealizationEntry.getRealization())){
+                        needUpdateHybrid = true;
+                        cubeRealizationEntries.remove(cubeRealizationEntry);
+                        break;
+                    }
+                }
+
+                if (needUpdateHybrid){
+                    String[] cubeNames = new String[cubeRealizationEntries.size()];
+                    for (int i = 0; i < cubeRealizationEntries.size(); i++){
+                        cubeNames[i] = cubeRealizationEntries.get(i).getRealization();
+                    }
+                    hybridService.updateHybridCubeNoCheck(instance.getName(), projectInstance.getName(),
cube.getModel().getName(), cubeNames);
+                }
+            }
+        }
+
         int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
         getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when
no other cube is using it
     }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
index 2fb7f84389..1b48082853 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -89,6 +89,29 @@ public HybridInstance updateHybridCube(String hybridName, String projectName,
St
         return getHybridInstance(hybridName);
     }
 
+    public void updateHybridCubeNoCheck(String hybridName, String projectName, String modelName,
+                                           String[] cubeNames) {
+        List<String> args = new ArrayList<String>();
+        args.add("-name");
+        args.add(hybridName);
+        args.add("-project");
+        args.add(projectName);
+        args.add("-model");
+        args.add(modelName);
+        args.add("-cubes");
+        args.add(StringUtils.join(cubeNames, ","));
+        args.add("-check");
+        args.add("false");
+        args.add("-action");
+        args.add("update");
+        try {
+            HybridCubeCLI.main(args.toArray(new String[args.size()]));
+        } catch (Exception e) {
+            logger.warn("Update Hybrid Failed", e);
+            throw e;
+        }
+    }
+
     public void deleteHybridCube(String hybridName, String projectName, String modelName)
{
         aclEvaluate.checkProjectWritePermission(projectName);
         List<String> args = new ArrayList<String>();


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> When a cube is deleted, remove it from the hybrid cube definition
> -----------------------------------------------------------------
>
>                 Key: KYLIN-3259
>                 URL: https://issues.apache.org/jira/browse/KYLIN-3259
>             Project: Kylin
>          Issue Type: Improvement
>          Components: Metadata
>    Affects Versions: v2.2.0
>         Environment: HDP 2.5.6, Kylin 2.2
>            Reporter: Vsevolod Ostapenko
>            Assignee:  Kaige Liu
>            Priority: Major
>             Fix For: v2.5.0
>
>
> When a cube is deleted, its references are not automatically removed from existing hybrid
cube definition. That can lead to errors down the road, if user (or application) retrieves
the list of cubes via REST API call and later tries to update the hybrid cube.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message