kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject kylin git commit: KYLIN-1540 REST API for deleting segment
Date Mon, 28 Mar 2016 10:15:15 GMT
Repository: kylin
Updated Branches:
  refs/heads/1.3.x 569c711f8 -> 125b34b9e


KYLIN-1540 REST API for deleting segment

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/125b34b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/125b34b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/125b34b9

Branch: refs/heads/1.3.x
Commit: 125b34b9e534be66db16a081483372ce8560c738
Parents: 569c711
Author: shaofengshi <shaofengshi@apache.org>
Authored: Mon Mar 28 18:14:08 2016 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Mon Mar 28 18:14:08 2016 +0800

----------------------------------------------------------------------
 ...st_kylin_cube_with_slr_ready_3_segments.json | 88 ++++++++++++++++++++
 .../kylin/rest/controller/CubeController.java   | 27 ++++++
 .../apache/kylin/rest/service/CubeService.java  | 21 +++++
 .../rest/controller/CubeControllerTest.java     | 45 ++++++++++
 4 files changed, 181 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
new file mode 100644
index 0000000..e39de87
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
@@ -0,0 +1,88 @@
+{
+  "uuid" : "3eaca32a-a33e-4b69-83dd-0bb8b1f8c53c",
+  "last_modified" : 1404098141020,
+  "name" : "test_kylin_cube_with_slr_ready_3_segments",
+  "owner" : null,
+  "descriptor" : "test_kylin_cube_with_slr_desc",
+  "cost" : 50,
+  "segments" : [ {
+    "uuid" : "1aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+    "name" : "19691231160000_20131112000000",
+    "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
+    "date_range_start" : 1384240200000,
+    "date_range_end" : 1384243200000,
+    "status" : "READY",
+    "size_kb" : 7801,
+    "source_records" : 10000,
+    "source_records_size" : 608012,
+    "last_build_time" : 1404098140902,
+    "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
+    "binary_signature" : null,
+      "dictionaries" : {
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
+        "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
+        "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
+        "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
+        "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
+        "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
+        "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
+        "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
+      },
+      "snapshots" : {
+        "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
+        "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
+        "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
+      }
+  }, {
+    "uuid" : "2aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+    "name" : "20131112000000_20131212000000",
+    "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
+    "date_range_start" : 1384243200000,
+    "date_range_end" : 1386835200000,
+    "status" : "READY",
+    "size_kb" : 7801,
+    "source_records" : 10000,
+    "source_records_size" : 608012,
+    "last_build_time" : 1404098140902,
+    "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb",
+    "binary_signature" : null,
+      "dictionaries" : {
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
+        "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
+        "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
+        "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
+        "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
+        "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
+        "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
+        "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
+      },
+      "snapshots" : {
+        "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
+        "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
+        "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
+        "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
+      }
+  }, {
+    "uuid" : "3aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+    "name" : "20131212000000_20140112000000",
+    "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
+    "date_range_start" : 1386835200000,
+    "date_range_end" : 1389484800000,
+    "status" : "NEW",
+    "size_kb" : 0,
+    "source_records" : 0,
+    "source_records_size" : 0,
+    "last_build_time" : 1404098140902,
+    "last_build_job_id" : "",
+    "binary_signature" : null
+  } ],
+  "status" : "READY",
+  "create_time" : null,
+  "notify_list" : null,
+  "auto_merge_time_ranges" : [2595000000]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 880a736..23e22d0 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -200,6 +200,33 @@ public class CubeController extends BasicController {
     }
 
     /**
+     * Delete a cube segment
+     *
+     * @throws IOException
+     */
+    @RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE
})
+    @ResponseBody
+    public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String
segmentName) {
+        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+        if (cube == null) {
+            throw new InternalErrorException("Cannot find cube " + cubeName);
+        }
+
+        CubeSegment segment = cube.getSegment(segmentName, null);
+        if (segment == null) {
+            throw new InternalErrorException("Cannot find segment '" + segmentName + "'");
+        }
+
+        try {
+            return cubeService.deleteSegment(cube, segmentName);
+        } catch (Exception e) {
+            logger.error(e.getLocalizedMessage(), e);
+            throw new InternalErrorException(e.getLocalizedMessage());
+        }
+    }
+
+    /**
      * Send a rebuild cube job
      *
      * @param cubeName Cube ID

http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 5020dc3..6f634cb 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -495,6 +495,27 @@ public class CubeService extends BasicService {
         return cube;
     }
 
+    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')
or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
+    public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException
{
+
+        if (!segmentName.equals(cube.getSegments().get(0).getName()) && !segmentName.equals(cube.getSegments().get(cube.getSegments().size()
- 1).getName())) {
+            throw new IllegalArgumentException("Cannot delete segment '" + segmentName +
"' as it is neither the first nor the last segment.");
+        }
+        CubeSegment toDelete = null;
+        for (CubeSegment seg : cube.getSegments()) {
+            if (seg.getName().equals(segmentName)) {
+                toDelete = seg;
+            }
+        }
+
+        if (toDelete.getStatus() != SegmentStatusEnum.READY) {
+            throw new IllegalArgumentException("Cannot delete segment '" + segmentName +
"' as its status is not READY. Discard the on-going job for it.");
+        }
+
+        cube.getSegments().remove(toDelete);
+        return CubeManager.getInstance(getConfig()).updateCube(cube);
+    }
+
     /**
      * purge the cube
      *

http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 0d14711..e939a54 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -27,6 +27,7 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.service.AccessService;
 import org.apache.kylin.rest.service.CubeService;
@@ -225,4 +226,48 @@ public class CubeControllerTest extends ServiceTestBase {
         CubeRequest res = cubeController.updateCubeDesc(req);
         Assert.assertFalse(res.getSuccessful());
     }
+
+    @Test (expected=InternalErrorException.class)
+    public void testDeleteSegmentNew() throws IOException {
+        String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+        CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+        Assert.assertNotNull(cubes);
+
+        cubeController.deleteSegment(cubeName, "20131212000000_20140112000000");
+    }
+
+    @Test (expected=InternalErrorException.class)
+    public void testDeleteSegmentNotExist() throws IOException {
+        String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+        CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+        Assert.assertNotNull(cubes);
+
+        cubeController.deleteSegment(cubeName, "not_exist_segment");
+    }
+
+
+    @Test (expected=InternalErrorException.class)
+    public void testDeleteSegmentInMiddle() throws IOException {
+        String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+        CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+        Assert.assertNotNull(cubes);
+
+        cubeController.deleteSegment(cubeName, "20131112000000_20131212000000");
+    }
+
+    @Test
+    public void testDeleteSegmentFromHead() throws IOException {
+        String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+        CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+        Assert.assertNotNull(cubes);
+
+        int segNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size();
+
+        cubeController.deleteSegment(cubeName, "19691231160000_20131112000000");
+
+        int newSegNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size();
+
+        Assert.assertTrue(segNumber == newSegNumber + 1);
+    }
+
 }


Mime
View raw message