Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8132F18419 for ; Mon, 28 Mar 2016 10:15:15 +0000 (UTC) Received: (qmail 99158 invoked by uid 500); 28 Mar 2016 10:15:15 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 99127 invoked by uid 500); 28 Mar 2016 10:15:15 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 99118 invoked by uid 99); 28 Mar 2016 10:15:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 28 Mar 2016 10:15:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 51615DFDE0; Mon, 28 Mar 2016 10:15:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shaofengshi@apache.org To: commits@kylin.apache.org Message-Id: <4e1b5280fef949a1a0e558727c979892@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: kylin git commit: KYLIN-1540 REST API for deleting segment Date: Mon, 28 Mar 2016 10:15:15 +0000 (UTC) Repository: kylin Updated Branches: refs/heads/1.3.x 569c711f8 -> 125b34b9e KYLIN-1540 REST API for deleting segment Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/125b34b9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/125b34b9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/125b34b9 Branch: refs/heads/1.3.x Commit: 125b34b9e534be66db16a081483372ce8560c738 Parents: 569c711 Author: shaofengshi Authored: Mon Mar 28 18:14:08 2016 +0800 Committer: shaofengshi Committed: Mon Mar 28 18:14:08 2016 +0800 ---------------------------------------------------------------------- ...st_kylin_cube_with_slr_ready_3_segments.json | 88 ++++++++++++++++++++ .../kylin/rest/controller/CubeController.java | 27 ++++++ .../apache/kylin/rest/service/CubeService.java | 21 +++++ .../rest/controller/CubeControllerTest.java | 45 ++++++++++ 4 files changed, 181 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json new file mode 100644 index 0000000..e39de87 --- /dev/null +++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json @@ -0,0 +1,88 @@ +{ + "uuid" : "3eaca32a-a33e-4b69-83dd-0bb8b1f8c53c", + "last_modified" : 1404098141020, + "name" : "test_kylin_cube_with_slr_ready_3_segments", + "owner" : null, + "descriptor" : "test_kylin_cube_with_slr_desc", + "cost" : 50, + "segments" : [ { + "uuid" : "1aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "19691231160000_20131112000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA", + "date_range_start" : 1384240200000, + "date_range_end" : 1384243200000, + "status" : "READY", + "size_kb" : 7801, + "source_records" : 10000, + "source_records_size" : 608012, + "last_build_time" : 1404098140902, + "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa", + "binary_signature" : null, + "dictionaries" : { + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict", + "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict", + "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict", + "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict", + "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict", + "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict", + "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict", + "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict" + }, + "snapshots" : { + "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot", + "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot", + "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot", + "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot" + } + }, { + "uuid" : "2aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "20131112000000_20131212000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB", + "date_range_start" : 1384243200000, + "date_range_end" : 1386835200000, + "status" : "READY", + "size_kb" : 7801, + "source_records" : 10000, + "source_records_size" : 608012, + "last_build_time" : 1404098140902, + "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb", + "binary_signature" : null, + "dictionaries" : { + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict", + "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict", + "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict", + "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict", + "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict", + "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict", + "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict", + "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict", + "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict" + }, + "snapshots" : { + "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot", + "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot", + "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot", + "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot" + } + }, { + "uuid" : "3aeca32a-a33e-4b69-83dd-xxe8b1f8dddd", + "name" : "20131212000000_20140112000000", + "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB", + "date_range_start" : 1386835200000, + "date_range_end" : 1389484800000, + "status" : "NEW", + "size_kb" : 0, + "source_records" : 0, + "source_records_size" : 0, + "last_build_time" : 1404098140902, + "last_build_job_id" : "", + "binary_signature" : null + } ], + "status" : "READY", + "create_time" : null, + "notify_list" : null, + "auto_merge_time_ranges" : [2595000000] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 880a736..23e22d0 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -200,6 +200,33 @@ public class CubeController extends BasicController { } /** + * Delete a cube segment + * + * @throws IOException + */ + @RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE }) + @ResponseBody + public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String segmentName) { + CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); + + if (cube == null) { + throw new InternalErrorException("Cannot find cube " + cubeName); + } + + CubeSegment segment = cube.getSegment(segmentName, null); + if (segment == null) { + throw new InternalErrorException("Cannot find segment '" + segmentName + "'"); + } + + try { + return cubeService.deleteSegment(cube, segmentName); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + throw new InternalErrorException(e.getLocalizedMessage()); + } + } + + /** * Send a rebuild cube job * * @param cubeName Cube ID http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 5020dc3..6f634cb 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -495,6 +495,27 @@ public class CubeService extends BasicService { return cube; } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") + public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException { + + if (!segmentName.equals(cube.getSegments().get(0).getName()) && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) { + throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as it is neither the first nor the last segment."); + } + CubeSegment toDelete = null; + for (CubeSegment seg : cube.getSegments()) { + if (seg.getName().equals(segmentName)) { + toDelete = seg; + } + } + + if (toDelete.getStatus() != SegmentStatusEnum.READY) { + throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as its status is not READY. Discard the on-going job for it."); + } + + cube.getSegments().remove(toDelete); + return CubeManager.getInstance(getConfig()).updateCube(cube); + } + /** * purge the cube * http://git-wip-us.apache.org/repos/asf/kylin/blob/125b34b9/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java index 0d14711..e939a54 100644 --- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java +++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java @@ -27,6 +27,7 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.RowKeyColDesc; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.request.CubeRequest; import org.apache.kylin.rest.service.AccessService; import org.apache.kylin.rest.service.CubeService; @@ -225,4 +226,48 @@ public class CubeControllerTest extends ServiceTestBase { CubeRequest res = cubeController.updateCubeDesc(req); Assert.assertFalse(res.getSuccessful()); } + + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentNew() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "20131212000000_20140112000000"); + } + + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentNotExist() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "not_exist_segment"); + } + + + @Test (expected=InternalErrorException.class) + public void testDeleteSegmentInMiddle() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + cubeController.deleteSegment(cubeName, "20131112000000_20131212000000"); + } + + @Test + public void testDeleteSegmentFromHead() throws IOException { + String cubeName = "test_kylin_cube_with_slr_ready_3_segments"; + CubeDesc[] cubes = cubeDescController.getCube(cubeName); + Assert.assertNotNull(cubes); + + int segNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size(); + + cubeController.deleteSegment(cubeName, "19691231160000_20131112000000"); + + int newSegNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size(); + + Assert.assertTrue(segNumber == newSegNumber + 1); + } + }