carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: enhancement of VT for LCM.
Date Wed, 12 Oct 2016 15:46:34 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 5177754e7 -> 39dd489e3


enhancement of VT for LCM.

enhancing vt

adding the blocklet test case.


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ea4d8990
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ea4d8990
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ea4d8990

Branch: refs/heads/master
Commit: ea4d8990cfcfe2f88c2f589bc72200ac7c8522a1
Parents: 5177754
Author: ravikiran <ravikiran.sn042@gmail.com>
Authored: Thu Sep 8 17:52:37 2016 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Wed Oct 12 22:40:03 2016 +0800

----------------------------------------------------------------------
 .../test/resources/compaction/compaction1.csv   |   2 +-
 .../compaction/compaction1_forhive.csv          |   5 +
 .../compaction/compactioncard2_forhive.csv      | 256 +++++++++++++++++++
 .../DataCompactionBlockletBoundryTest.scala     | 108 ++++++++
 .../DataCompactionBoundaryConditionsTest.scala  | 105 ++++++++
 .../DataCompactionMinorThresholdTest.scala      | 118 +++++++++
 6 files changed, 593 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compaction1.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/compaction/compaction1.csv b/integration/spark/src/test/resources/compaction/compaction1.csv
index 2271e18..b669eca 100644
--- a/integration/spark/src/test/resources/compaction/compaction1.csv
+++ b/integration/spark/src/test/resources/compaction/compaction1.csv
@@ -3,4 +3,4 @@ ID,date,country,name,phonetype,serialname,salary
 2,07/24/2015,chile,aaa2,phone756,ASD42892,15001
 3,07/25/2015,america,aaa3,phone1904,ASD37014,15002
 4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003
-5,07/27/2015,england,aaa5,phone2441,ASD90633,15004
+5,07/27/2015,england,aaa5,phone2441,ASD90633,15004
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compaction1_forhive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/compaction/compaction1_forhive.csv b/integration/spark/src/test/resources/compaction/compaction1_forhive.csv
new file mode 100644
index 0000000..07c2508
--- /dev/null
+++ b/integration/spark/src/test/resources/compaction/compaction1_forhive.csv
@@ -0,0 +1,5 @@
+1,07/23/2015,china,aaa1,phone197,ASD69643,15000
+2,07/24/2015,chile,aaa2,phone756,ASD42892,15001
+3,07/25/2015,america,aaa3,phone1904,ASD37014,15002
+4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003
+5,07/27/2015,england,aaa5,phone2441,ASD90633,15004
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv b/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv
new file mode 100644
index 0000000..7d7673e
--- /dev/null
+++ b/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv
@@ -0,0 +1,256 @@
+6,7/28/2015,newzealand,aaa6,phone294,ASD59961,15005
+7,7/29/2015,mexico,aaa7,phone610,ASD14875,15006
+8,7/30/2015,westindies,aaa8,phone1848,ASD57308,15007
+9,7/18/2015,burma,aaa9,phone706,ASD86717,15008
+10,7/19/2015,butan,aaa10,phone685,ASD30505,1500
+11,7/20/2015,burma,aaa11,phone664,ASD25707,1500
+12,7/21/2015,butan,aaa12,phone643,ASD81919,1500
+13,7/22/2015,burma,aaa13,phone622,ASD138131,1500
+14,7/23/2015,burma,aaa14,phone601,ASD194343,1500
+15,7/24/2015,butan,aaa15,phone580,ASD250555,1500
+16,7/25/2015,burma,aaa16,phone559,ASD306767,1500
+17,7/26/2015,butan,aaa17,phone538,ASD362979,1500
+18,7/27/2015,burma,aaa18,phone517,ASD419191,1500
+19,7/28/2015,burma,aaa19,phone496,ASD475403,1500
+20,7/29/2015,butan,aaa20,phone475,ASD531615,1500
+21,7/30/2015,burma,aaa21,phone454,ASD587827,1500
+22,7/31/2015,butan,aaa22,phone433,ASD644039,1500
+23,8/1/2015,burma,aaa23,phone412,ASD700251,1500
+24,8/2/2015,burma,aaa24,phone391,ASD756463,1500
+25,8/3/2015,butan,aaa25,phone370,ASD812675,1500
+26,8/4/2015,burma,aaa26,phone349,ASD868887,1500
+27,8/5/2015,butan,aaa27,phone328,ASD925099,1500
+28,8/6/2015,burma,aaa28,phone307,ASD981311,1500
+29,8/7/2015,burma,aaa29,phone286,ASD1037523,1500
+30,8/8/2015,butan,aaa30,phone265,ASD1093735,1500
+31,8/9/2015,burma,aaa31,phone244,ASD1149947,1500
+32,8/10/2015,butan,aaa32,phone223,ASD1206159,1500
+33,8/11/2015,burma,aaa33,phone202,ASD1262371,1500
+34,8/12/2015,burma,aaa34,phone181,ASD1318583,1500
+35,8/13/2015,butan,aaa35,phone160,ASD1374795,1500
+36,8/14/2015,burma,aaa36,phone139,ASD1431007,1500
+37,8/15/2015,butan,aaa37,phone118,ASD1487219,1500
+38,8/16/2015,burma,aaa38,phone97,ASD1543431,1500
+39,8/17/2015,burma,aaa39,phone76,ASD1599643,1500
+40,8/18/2015,butan,aaa40,phone55,ASD1655855,1500
+41,8/19/2015,burma,aaa41,phone34,ASD1712067,1500
+42,8/20/2015,butan,aaa42,phone13,ASD1768279,1500
+43,8/21/2015,burma,aaa43,phone8,ASD1824491,1500
+44,8/22/2015,burma,aaa44,phone29,ASD1880703,1500
+45,8/23/2015,butan,aaa45,phone50,ASD1936915,1500
+46,8/24/2015,burma,aaa46,phone71,ASD1993127,1500
+47,8/25/2015,butan,aaa47,phone92,ASD2049339,1500
+48,8/26/2015,burma,aaa48,phone113,ASD2105551,1500
+49,8/27/2015,burma,aaa49,phone134,ASD2161763,1500
+50,8/28/2015,butan,aaa50,phone155,ASD2217975,1500
+51,8/29/2015,burma,aaa51,phone176,ASD2274187,1500
+52,8/30/2015,butan,aaa52,phone197,ASD2330399,1500
+53,8/31/2015,burma,aaa53,phone218,ASD2386611,1500
+54,9/1/2015,burma,aaa54,phone239,ASD2442823,1500
+55,9/2/2015,butan,aaa55,phone260,ASD2499035,1500
+56,9/3/2015,burma,aaa56,phone281,ASD2555247,1500
+57,9/4/2015,butan,aaa57,phone302,ASD2611459,1500
+58,9/5/2015,burma,aaa58,phone323,ASD2667671,1500
+59,9/6/2015,burma,aaa59,phone344,ASD2723883,1500
+60,9/7/2015,butan,aaa60,phone365,ASD2780095,1500
+61,9/8/2015,burma,aaa61,phone386,ASD2836307,1500
+62,9/9/2015,butan,aaa62,phone407,ASD2892519,1500
+63,9/10/2015,burma,aaa63,phone428,ASD2948731,1500
+64,9/11/2015,burma,aaa64,phone449,ASD3004943,1500
+65,9/12/2015,butan,aaa65,phone470,ASD3061155,1500
+66,9/13/2015,burma,aaa66,phone491,ASD3117367,1500
+67,9/14/2015,butan,aaa67,phone512,ASD3173579,1500
+68,9/15/2015,burma,aaa68,phone533,ASD3229791,1500
+69,9/16/2015,burma,aaa69,phone554,ASD3286003,1500
+70,9/17/2015,butan,aaa70,phone575,ASD3342215,1500
+71,9/18/2015,burma,aaa71,phone596,ASD3398427,1500
+72,9/19/2015,butan,aaa72,phone617,ASD3454639,1500
+73,9/20/2015,burma,aaa73,phone638,ASD3510851,1500
+74,9/21/2015,burma,aaa74,phone659,ASD3567063,1500
+75,9/22/2015,butan,aaa75,phone680,ASD3623275,1500
+76,9/23/2015,burma,aaa76,phone701,ASD3679487,1500
+77,9/24/2015,butan,aaa77,phone722,ASD3735699,1500
+78,9/25/2015,burma,aaa78,phone743,ASD3791911,1500
+79,9/26/2015,burma,aaa79,phone764,ASD3848123,1500
+80,9/27/2015,butan,aaa80,phone785,ASD3904335,1500
+81,9/28/2015,burma,aaa81,phone806,ASD3960547,1500
+82,9/29/2015,butan,aaa82,phone827,ASD4016759,1500
+83,9/30/2015,burma,aaa83,phone848,ASD4072971,1500
+84,10/1/2015,burma,aaa84,phone869,ASD4129183,1500
+85,10/2/2015,butan,aaa85,phone890,ASD4185395,1500
+86,10/3/2015,burma,aaa86,phone911,ASD4241607,1500
+87,10/4/2015,butan,aaa87,phone932,ASD4297819,1500
+88,10/5/2015,burma,aaa88,phone953,ASD4354031,1500
+89,10/6/2015,burma,aaa89,phone974,ASD4410243,1500
+90,10/7/2015,butan,aaa90,phone995,ASD4466455,1500
+91,10/8/2015,burma,aaa91,phone1016,ASD4522667,1500
+92,10/9/2015,butan,aaa92,phone1037,ASD4578879,1500
+93,10/10/2015,burma,aaa93,phone1058,ASD4635091,1500
+94,10/11/2015,burma,aaa94,phone1079,ASD4691303,1500
+95,10/12/2015,butan,aaa95,phone1100,ASD4747515,1500
+96,10/13/2015,burma,aaa96,phone1121,ASD4803727,1500
+97,10/14/2015,butan,aaa97,phone1142,ASD4859939,1500
+98,10/15/2015,burma,aaa98,phone1163,ASD4916151,1500
+99,10/16/2015,burma,aaa99,phone1184,ASD4972363,1500
+100,10/17/2015,butan,aaa100,phone1205,ASD5028575,1500
+101,10/18/2015,burma,aaa101,phone1226,ASD5084787,1500
+102,10/19/2015,butan,aaa102,phone1247,ASD5140999,1500
+103,10/20/2015,burma,aaa103,phone1268,ASD5197211,1500
+104,10/21/2015,burma,aaa104,phone1289,ASD5253423,1500
+105,10/22/2015,butan,aaa105,phone1310,ASD5309635,1500
+106,10/23/2015,burma,aaa106,phone1331,ASD5365847,1500
+107,10/24/2015,butan,aaa107,phone1352,ASD5422059,1500
+108,10/25/2015,burma,aaa108,phone1373,ASD5478271,1500
+109,10/26/2015,burma,aaa109,phone1394,ASD5534483,1500
+110,10/27/2015,butan,aaa110,phone1415,ASD5590695,1500
+111,10/28/2015,burma,aaa111,phone1436,ASD5646907,1500
+112,10/29/2015,butan,aaa112,phone1457,ASD5703119,1500
+113,10/30/2015,burma,aaa113,phone1478,ASD5759331,1500
+114,10/31/2015,burma,aaa114,phone1499,ASD5815543,1500
+115,11/1/2015,butan,aaa115,phone1520,ASD5871755,1500
+116,11/2/2015,burma,aaa116,phone1541,ASD5927967,1500
+117,11/3/2015,butan,aaa117,phone1562,ASD5984179,1500
+118,11/4/2015,burma,aaa118,phone1583,ASD6040391,1500
+119,11/5/2015,burma,aaa119,phone1604,ASD6096603,1500
+120,11/6/2015,butan,aaa120,phone1625,ASD6152815,1500
+121,11/7/2015,burma,aaa121,phone1646,ASD6209027,1500
+122,11/8/2015,butan,aaa122,phone1667,ASD6265239,1500
+123,11/9/2015,burma,aaa123,phone1688,ASD6321451,1500
+124,11/10/2015,burma,aaa124,phone1709,ASD6377663,1500
+125,11/11/2015,butan,aaa125,phone1730,ASD6433875,1500
+126,11/12/2015,burma,aaa126,phone1751,ASD6490087,1500
+127,11/13/2015,butan,aaa127,phone1772,ASD6546299,1500
+128,11/14/2015,burma,aaa128,phone1793,ASD6602511,1500
+129,11/15/2015,burma,aaa129,phone1814,ASD6658723,1500
+130,11/16/2015,butan,aaa130,phone1835,ASD6714935,1500
+131,11/17/2015,burma,aaa131,phone1856,ASD6771147,1500
+132,11/18/2015,butan,aaa132,phone1877,ASD6827359,1500
+133,11/19/2015,burma,aaa133,phone1898,ASD6883571,1500
+134,11/20/2015,burma,aaa134,phone1919,ASD6939783,1500
+135,11/21/2015,butan,aaa135,phone1940,ASD6995995,1500
+136,11/22/2015,burma,aaa136,phone1961,ASD7052207,1500
+137,11/23/2015,butan,aaa137,phone1982,ASD7108419,1500
+138,11/24/2015,burma,aaa138,phone2003,ASD7164631,1500
+139,11/25/2015,burma,aaa139,phone2024,ASD7220843,1500
+140,11/26/2015,butan,aaa140,phone2045,ASD7277055,1500
+141,11/27/2015,burma,aaa141,phone2066,ASD7333267,1500
+142,11/28/2015,butan,aaa142,phone2087,ASD7389479,1500
+143,11/29/2015,burma,aaa143,phone2108,ASD7445691,1500
+144,11/30/2015,burma,aaa144,phone2129,ASD7501903,1500
+145,12/1/2015,butan,aaa145,phone2150,ASD7558115,1500
+146,12/2/2015,burma,aaa146,phone2171,ASD7614327,1500
+147,12/3/2015,butan,aaa147,phone2192,ASD7670539,1500
+148,12/4/2015,burma,aaa148,phone2213,ASD7726751,1500
+149,12/5/2015,burma,aaa149,phone2234,ASD7782963,1500
+150,12/6/2015,butan,aaa150,phone2255,ASD7839175,1500
+151,12/7/2015,burma,aaa151,phone2276,ASD7895387,1500
+152,12/8/2015,butan,aaa152,phone2297,ASD7951599,1500
+153,12/9/2015,burma,aaa153,phone2318,ASD8007811,1500
+154,12/10/2015,burma,aaa154,phone2339,ASD8064023,1500
+155,12/11/2015,butan,aaa155,phone2360,ASD8120235,1500
+156,12/12/2015,burma,aaa156,phone2381,ASD8176447,1500
+157,12/13/2015,butan,aaa157,phone2402,ASD8232659,1500
+158,12/14/2015,burma,aaa158,phone2423,ASD8288871,1500
+159,12/15/2015,burma,aaa159,phone2444,ASD8345083,1500
+160,12/16/2015,butan,aaa160,phone2465,ASD8401295,1500
+161,12/17/2015,burma,aaa161,phone2486,ASD8457507,1500
+162,12/18/2015,butan,aaa162,phone2507,ASD8513719,1500
+163,12/19/2015,burma,aaa163,phone2528,ASD8569931,1500
+164,12/20/2015,burma,aaa164,phone2549,ASD8626143,1500
+165,12/21/2015,butan,aaa165,phone2570,ASD8682355,1500
+166,12/22/2015,burma,aaa166,phone2591,ASD8738567,1500
+167,12/23/2015,butan,aaa167,phone2612,ASD8794779,1500
+168,12/24/2015,burma,aaa168,phone2633,ASD8850991,1500
+169,12/25/2015,burma,aaa169,phone2654,ASD8907203,1500
+170,12/26/2015,butan,aaa170,phone2675,ASD8963415,1500
+171,12/27/2015,burma,aaa171,phone2696,ASD9019627,1500
+172,12/28/2015,butan,aaa172,phone2717,ASD9075839,1500
+173,12/29/2015,burma,aaa173,phone2738,ASD9132051,1500
+174,12/30/2015,burma,aaa174,phone2759,ASD9188263,1500
+175,12/31/2015,butan,aaa175,phone2780,ASD9244475,1500
+176,1/1/2016,burma,aaa176,phone2801,ASD9300687,1500
+177,1/2/2016,butan,aaa177,phone2822,ASD9356899,1500
+178,1/3/2016,burma,aaa178,phone2843,ASD9413111,1500
+179,1/4/2016,burma,aaa179,phone2864,ASD9469323,1500
+180,1/5/2016,butan,aaa180,phone2885,ASD9525535,1500
+181,1/6/2016,burma,aaa181,phone2906,ASD9581747,1500
+182,1/7/2016,butan,aaa182,phone2927,ASD9637959,1500
+183,1/8/2016,burma,aaa183,phone2948,ASD9694171,1500
+184,1/9/2016,burma,aaa184,phone2969,ASD9750383,1500
+185,1/10/2016,butan,aaa185,phone2990,ASD9806595,1500
+186,1/11/2016,burma,aaa186,phone3011,ASD9862807,1500
+187,1/12/2016,butan,aaa187,phone3032,ASD9919019,1500
+188,1/13/2016,burma,aaa188,phone3053,ASD9975231,1500
+189,1/14/2016,burma,aaa189,phone3074,ASD10031443,1500
+190,1/15/2016,butan,aaa190,phone3095,ASD10087655,1500
+191,1/16/2016,burma,aaa191,phone3116,ASD10143867,1500
+192,1/17/2016,butan,aaa192,phone3137,ASD10200079,1500
+193,1/18/2016,burma,aaa193,phone3158,ASD10256291,1500
+194,1/19/2016,burma,aaa194,phone3179,ASD10312503,1500
+195,1/20/2016,butan,aaa195,phone3200,ASD10368715,1500
+196,1/21/2016,burma,aaa196,phone3221,ASD10424927,1500
+197,1/22/2016,butan,aaa197,phone3242,ASD10481139,1500
+198,1/23/2016,burma,aaa198,phone3263,ASD10537351,1500
+199,1/24/2016,burma,aaa199,phone3284,ASD10593563,1500
+200,1/25/2016,butan,aaa200,phone3305,ASD10649775,1500
+201,1/26/2016,burma,aaa201,phone3326,ASD10705987,1500
+202,1/27/2016,butan,aaa202,phone3347,ASD10762199,1500
+203,1/28/2016,burma,aaa203,phone3368,ASD10818411,1500
+204,1/29/2016,burma,aaa204,phone3389,ASD10874623,1500
+205,1/30/2016,butan,aaa205,phone3410,ASD10930835,1500
+206,1/31/2016,burma,aaa206,phone3431,ASD10987047,1500
+207,2/1/2016,butan,aaa207,phone3452,ASD11043259,1500
+208,2/2/2016,burma,aaa208,phone3473,ASD11099471,1500
+209,2/3/2016,burma,aaa209,phone3494,ASD11155683,1500
+210,2/4/2016,butan,aaa210,phone3515,ASD11211895,1500
+211,2/5/2016,burma,aaa211,phone3536,ASD11268107,1500
+212,2/6/2016,butan,aaa212,phone3557,ASD11324319,1500
+213,2/7/2016,burma,aaa213,phone3578,ASD11380531,1500
+214,2/8/2016,burma,aaa214,phone3599,ASD11436743,1500
+215,2/9/2016,butan,aaa215,phone3620,ASD11492955,1500
+216,2/10/2016,burma,aaa216,phone3641,ASD11549167,1500
+217,2/11/2016,butan,aaa217,phone3662,ASD11605379,1500
+218,2/12/2016,burma,aaa218,phone3683,ASD11661591,1500
+219,2/13/2016,burma,aaa219,phone3704,ASD11717803,1500
+220,2/14/2016,butan,aaa220,phone3725,ASD11774015,1500
+221,2/15/2016,burma,aaa221,phone3746,ASD11830227,1500
+222,2/16/2016,butan,aaa222,phone3767,ASD11886439,1500
+223,2/17/2016,burma,aaa223,phone3788,ASD11942651,1500
+224,2/18/2016,burma,aaa224,phone3809,ASD11998863,1500
+225,2/19/2016,butan,aaa225,phone3830,ASD12055075,1500
+226,2/20/2016,burma,aaa226,phone3851,ASD12111287,1500
+227,2/21/2016,butan,aaa227,phone3872,ASD12167499,1500
+228,2/22/2016,burma,aaa228,phone3893,ASD12223711,1500
+229,2/23/2016,burma,aaa229,phone3914,ASD12279923,1500
+230,2/24/2016,butan,aaa230,phone3935,ASD12336135,1500
+231,2/25/2016,burma,aaa231,phone3956,ASD12392347,1500
+232,2/26/2016,butan,aaa232,phone3977,ASD12448559,1500
+233,2/27/2016,burma,aaa233,phone3998,ASD12504771,1500
+234,2/28/2016,burma,aaa234,phone4019,ASD12560983,1500
+235,2/29/2016,butan,aaa235,phone4040,ASD12617195,1500
+236,3/1/2016,burma,aaa236,phone4061,ASD12673407,1500
+237,3/2/2016,butan,aaa237,phone4082,ASD12729619,1500
+238,3/3/2016,burma,aaa238,phone4103,ASD12785831,1500
+239,3/4/2016,burma,aaa239,phone4124,ASD12842043,1500
+240,3/5/2016,butan,aaa240,phone4145,ASD12898255,1500
+241,3/6/2016,burma,aaa241,phone4166,ASD12954467,1500
+242,3/7/2016,butan,aaa242,phone4187,ASD13010679,1500
+243,3/8/2016,burma,aaa243,phone4208,ASD13066891,1500
+244,3/9/2016,burma,aaa244,phone4229,ASD13123103,1500
+245,3/10/2016,butan,aaa245,phone4250,ASD13179315,1500
+246,3/11/2016,burma,aaa246,phone4271,ASD13235527,1500
+247,3/12/2016,butan,aaa247,phone4292,ASD13291739,1500
+248,3/13/2016,burma,aaa248,phone4313,ASD13347951,1500
+249,3/14/2016,burma,aaa249,phone4334,ASD13404163,1500
+250,3/15/2016,butan,aaa250,phone4355,ASD13460375,1500
+251,3/16/2016,burma,aaa251,phone4376,ASD13516587,1500
+252,3/17/2016,butan,aaa252,phone4397,ASD13572799,1500
+253,3/18/2016,burma,aaa253,phone4418,ASD13629011,1500
+254,3/19/2016,burma,aaa254,phone4439,ASD13685223,1500
+255,3/20/2016,butan,aaa255,phone4460,ASD13741435,1500
+256,3/21/2016,burma,aaa256,phone4481,ASD13797647,1500
+257,3/22/2016,butan,aaa257,phone4502,ASD13853859,1500
+258,3/23/2016,burma,aaa258,phone4523,ASD13910071,1500
+259,3/24/2016,burma,aaa259,phone4544,ASD13966283,1500
+260,3/25/2016,butan,aaa260,phone4565,ASD14022495,1500
+261,3/26/2016,burma,aaa261,phone4586,ASD14078707,1500
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
new file mode 100644
index 0000000..f7056a9
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datacompaction
+
+import java.io.File
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.lcm.status.SegmentStatusManager
+
+/**
+  * FT for data compaction scenario.
+  */
+class DataCompactionBlockletBoundryTest extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+    sql("drop table if exists  blocklettest")
+    sql("drop table if exists  Carbon_automation_hive")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+        "55")
+    sql(
+      "CREATE TABLE IF NOT EXISTS blocklettest (country String, ID String, date Timestamp,
name " +
+        "String, " +
+        "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata"
+
+        ".format'"
+    )
+
+
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+
+    // loading the rows greater than 256. so that the column cardinality crosses byte boundary.
+    val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compactioncard2.csv"
+
+
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE blocklettest OPTIONS" +
+      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE blocklettest  OPTIONS"
+
+      "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    // compaction will happen here.
+    sql("alter table blocklettest compact 'major'"
+    )
+
+    sql(
+      "create table Carbon_automation_hive (ID String, date " +
+      "Timestamp,country String, name String, phonetype String, serialname String, salary
Int ) row format " +
+      "delimited fields terminated by ',' TBLPROPERTIES ('skip.header.line.count'='1') "
+    )
+
+    sql("LOAD DATA LOCAL INPATH '" + currentDirectory +
+        "/src/test/resources/compaction/compaction1_forhive.csv" + "' INTO " +
+        "table Carbon_automation_hive ")
+    sql("LOAD DATA LOCAL INPATH '" + currentDirectory +
+        "/src/test/resources/compaction/compactioncard2_forhive.csv" + "' INTO " +
+        "table Carbon_automation_hive ")
+
+  }
+
+  test("select country,count(*) as a from blocklettest")({
+    sql("select country,count(*) as a from Carbon_automation_hive group by country").show()
+    sql("select *  from Carbon_automation_hive").show
+    checkAnswer(
+      sql("select country,count(*) as a from blocklettest group by country"),
+      sql("select country,count(*) as a from Carbon_automation_hive group by country")
+    )
+  }
+  )
+
+  override def afterAll {
+    /* sql("drop table blocklettest") */
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+        "" + CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
new file mode 100644
index 0000000..80a2320
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datacompaction
+
+import java.io.File
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.lcm.status.SegmentStatusManager
+
+/**
+ * FT for data compaction Boundary condition verification.
+ */
+class DataCompactionBoundaryConditionsTest extends QueryTest with BeforeAndAfterAll {
+  val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+    .getCanonicalPath
+  val resource = currentDirectory + "/src/test/resources/"
+
+  val storeLocation = new File(this.getClass.getResource("/").getPath + "/../test").getCanonicalPath
+  val carbonTableIdentifier: CarbonTableIdentifier =
+    new CarbonTableIdentifier("default", "boundarytest".toLowerCase(), "1")
+
+  val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+      AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier))
+
+  override def beforeAll {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2")
+    sql("drop table if exists  boundarytest")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+    sql(
+      "CREATE TABLE IF NOT EXISTS boundarytest (country String, ID Int, date " +
+      "Timestamp, name " +
+      "String, " +
+      "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata"
+
+      ".format'"
+    )
+
+  }
+
+  /**
+   * Compaction verificatoin in case of no loads.
+   */
+  test("check if compaction is completed correctly.") {
+
+    try {
+      sql("alter table boundarytest compact 'minor'")
+      sql("alter table boundarytest compact 'major'")
+    }
+    catch {
+      case e: Exception =>
+        assert(false)
+    }
+  }
+
+  /**
+   * Compaction verificatoin in case of one loads.
+   */
+  test("check if compaction is completed correctly for one load.") {
+
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+
+
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE boundarytest " +
+        "OPTIONS" +
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    sql("alter table boundarytest compact 'minor'")
+    sql("alter table boundarytest compact 'major'")
+
+  }
+
+
+  override def afterAll {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
new file mode 100644
index 0000000..fbb39d8
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datacompaction
+
+import java.io.File
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.carbon.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.lcm.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.lcm.status.SegmentStatusManager
+
+/**
+ * FT for data compaction Minor threshold verification.
+ */
+class DataCompactionMinorThresholdTest extends QueryTest with BeforeAndAfterAll {
+  val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+    .getCanonicalPath
+  val resource = currentDirectory + "/src/test/resources/"
+
+  val storeLocation = new File(this.getClass.getResource("/").getPath + "/../test").getCanonicalPath
+  val carbonTableIdentifier: CarbonTableIdentifier =
+    new CarbonTableIdentifier("default", "minorthreshold".toLowerCase(), "1")
+
+  val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+      AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier))
+
+  override def beforeAll {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2")
+    sql("drop table if exists  minorthreshold")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+    sql(
+      "CREATE TABLE IF NOT EXISTS minorthreshold (country String, ID Int, date " +
+      "Timestamp, name " +
+      "String, " +
+      "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata"
+
+      ".format'"
+    )
+
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+
+    var csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv"
+    var csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
+
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE minorthreshold " +
+        "OPTIONS" +
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE minorthreshold  " +
+        "OPTIONS" +
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold  " +
+        "OPTIONS" +
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold  " +
+        "OPTIONS" +
+        "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+    // compaction should happen here.
+    sql("alter table minorthreshold compact 'minor'")
+  }
+
+  /**
+   * Compaction should be completed correctly for minor compaction.
+   */
+  test("check if compaction is completed correctly for minor.") {
+
+    sql("clean files for table minorthreshold")
+
+    val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
+
+    assert(segments.contains("0.2"))
+    assert(!segments.contains("0.1"))
+    assert(!segments.contains("0"))
+    assert(!segments.contains("1"))
+    assert(!segments.contains("2"))
+    assert(!segments.contains("3"))
+  }
+
+
+  override def afterAll {
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
+        CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
+  }
+
+}


Mime
View raw message