carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "manoj mathpal (JIRA)" <j...@apache.org>
Subject [jira] [Created] (CARBONDATA-1479) Tree exception displays to user in spark 1.6
Date Wed, 13 Sep 2017 16:57:00 GMT
manoj mathpal created CARBONDATA-1479:
-----------------------------------------

             Summary: Tree exception displays to user in spark 1.6
                 Key: CARBONDATA-1479
                 URL: https://issues.apache.org/jira/browse/CARBONDATA-1479
             Project: CarbonData
          Issue Type: Bug
          Components: sql
    Affects Versions: 1.1.1
         Environment: spark 1.6
            Reporter: manoj mathpal
         Attachments: 2000_UniqData.csv, 3000_1_UniqData.csv, 3000_UniqData.csv, 4000_UniqData.csv,
5000_UniqData.csv, 6000_UniqData.csv, 7000_UniqData.csv

Step to reproduce:

1:Create a table in carbon data :

CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp,
DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),
DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1
int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ("TABLE_BLOCKSIZE"= "256 MB")

2: Load Data into the table:

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/2000_UniqData_tabdelm.csv' into table uniqdata
OPTIONS('DELIMITER'='/' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_UniqDatatdelm.csv' into table uniqdata
OPTIONS('DELIMITER'='\t' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/5000_UniqData!delm.csv' into table uniqdata
OPTIONS('DELIMITER'='!' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/6000_UniqData|delm.csv' into table uniqdata
OPTIONS('DELIMITER'='|' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/7000_UniqData;delm.csv' into table uniqdata
OPTIONS('DELIMITER'=';' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/uniqdata/3000_1_UniqData.csv' into table uniqdata
OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')

3: Select Query:


select cust_id,cust_name,to_date(doj),quarter(dob),month(doj),day(dob),hour(doj),minute(dob),second(doj),weekofyear(dob),datediff(doj,current_date),date_add(dob,4),date_sub(doj,1),to_utc_timestamp(doj,current_date),add_months(dob,5),last_day(doj),months_between(doj,current_date),date_format(dob,current_date)
from uniqdata where substr(cust_name,0,4)='CUST' and length(cust_name)in(15,14,13,16) and
ACTIVE_EMUI_VERSION rlike 'ACTIVE' and month(dob)=01  and minute(dob)=0  group by doj,cust_id,cust_name,ACTIVE_EMUI_VERSION,dob
having  max(cust_id)=10830 and count(distinct(cust_id))<=2001 and max(cust_name) not like
'%def%' order by cust_id,cust_name


Result:


execute, tree: CarbonDictionaryDecoder [CarbonDecoderRelation(Map(decimal_column1#17584 ->
decimal_column1#17584, dob#17580 -> dob#17580, double_column1#17586 -> double_column1#17586,
cust_name#17578 -> cust_name#17578, integer_column1#17588 -> integer_column1#17588,
bigint_column2#17583L -> bigint_column2#17583L, bigint_column1#17582L -> bigint_column1#17582L,
decimal_column2#17585 -> decimal_column2#17585, active_emui_version#17579 -> active_emui_version#17579,
double_column2#17587 -> double_column2#17587, cust_id#17577 -> cust_id#17577, doj#17581
-> doj#17581),CarbonDatasourceRelation(`default`.`uniqdata`,None))], ExcludeProfile(ArrayBuffer(ACTIVE_EMUI_VERSION#17579,
cust_name#17578, #19968, dob#17580)), CarbonAliasDecoderRelation() +- Sort [cust_id#17577
ASC,cust_name#17578 ASC], true, 0    +- ConvertToUnsafe       +- Exchange rangepartitioning(cust_id#17577
ASC,cust_name#17578 ASC,200), None          +- ConvertToSafe             +- Project [cust_id#17577,cust_name#17578,_c2#19946,_c3#19947,_c4#19948,_c5#19949,_c6#19943,_c7#19944,_c8#19945,_c9#19950,_c10#19951,_c11#19952,_c12#19953,_c13#19954,_c14#19955,_c15#19956,_c16#19957,_c17#19958]
               +- Filter havingCondition#19959: boolean                   +- CarbonDictionaryDecoder
[CarbonDecoderRelation(Map(decimal_column1#17584 -> decimal_column1#17584, dob#17580 ->
dob#17580, double_column1#17586 -> double_column1#17586, cust_name#17578 -> cust_name#17578,
integer_column1#17588 -> integer_column1#17588, bigint_column2#17583L -> bigint_column2#17583L,
bigint_column1#17582L -> bigint_column1#17582L, decimal_column2#17585 -> decimal_column2#17585,
active_emui_version#17579 -> active_emui_version#17579, double_column2#17587 -> double_column2#17587,
cust_id#17577 -> cust_id#17577, doj#17581 -> doj#17581),CarbonDatasourceRelation(`default`.`uniqdata`,None))],
IncludeProfile(ArrayBuffer(#19968)), CarbonAliasDecoderRelation()                      +-
TungstenAggregate(key=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580],
functions=[], output=[havingCondition#19959,cust_id#17577,cust_name#17578,_c2#19946,_c3#19947,_c4#19948,_c5#19949,_c6#19943,_c7#19944,_c8#19945,_c9#19950,_c10#19951,_c11#19952,_c12#19953,_c13#19954,_c14#19955,_c15#19956,_c16#19957,_c17#19958])
                        +- TungstenExchange hashpartitioning(doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,200),
None                            +- TungstenAggregate(key=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580],
functions=[], output=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580])
                              +- SortBasedAggregate(key=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962],
functions=[(max(cust_id#19964),mode=Final,isDistinct=false),(max(cust_name#19965),mode=Final,isDistinct=false)],
output=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962,max(cust_id)#19966,max(cust_name)#19967])
                                 +- ConvertToSafe                                     +- Sort
[doj#17581 ASC,cust_id#17577 ASC,cust_name#17578 ASC,ACTIVE_EMUI_VERSION#17579 ASC,dob#17580
ASC,cust_id#19963 ASC,gid#19962 ASC], false, 0                                        +- TungstenExchange
hashpartitioning(doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962,200),
None                                           +- ConvertToUnsafe                        
                     +- SortBasedAggregate(key=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962],
functions=[(max(cust_id#19964),mode=Partial,isDistinct=false),(max(cust_name#19965),mode=Partial,isDistinct=false)],
output=[doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962,max#19991,max#19992])
                                                +- ConvertToSafe                         
                          +- Sort [doj#17581 ASC,cust_id#17577 ASC,cust_name#17578 ASC,ACTIVE_EMUI_VERSION#17579
ASC,dob#17580 ASC,cust_id#19963 ASC,gid#19962 ASC], false, 0                             
                         +- Expand [List(doj#17581, cust_id#17577, cust_name#17578, ACTIVE_EMUI_VERSION#17579,
dob#17580, null, 0, cust_id#17577, cust_name#17578),List(doj#17581, cust_id#17577, cust_name#17578,
ACTIVE_EMUI_VERSION#17579, dob#17580, cust_id#17577, 1, null, null)], [doj#17581,cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,cust_id#19963,gid#19962,cust_id#19964,cust_name#19965]
                                                         +- Project [cust_name#17578,dob#17580,doj#17581,cust_id#17577,ACTIVE_EMUI_VERSION#17579]
                                                            +- Filter (((((substring(cust_name#17578,0,4)
= CUST) && length(cust_name#17578) IN (15,14,13,16)) && ACTIVE_EMUI_VERSION#17579
RLIKE ACTIVE) && (month(cast(dob#17580 as date)) = 1)) && (minute(dob#17580)
= 0))                                                                +- CarbonDictionaryDecoder
[CarbonDecoderRelation(Map(decimal_column1#17584 -> decimal_column1#17584, dob#17580 ->
dob#17580, double_column1#17586 -> double_column1#17586, cust_name#17578 -> cust_name#17578,
integer_column1#17588 -> integer_column1#17588, bigint_column2#17583L -> bigint_column2#17583L,
bigint_column1#17582L -> bigint_column1#17582L, decimal_column2#17585 -> decimal_column2#17585,
active_emui_version#17579 -> active_emui_version#17579, double_column2#17587 -> double_column2#17587,
cust_id#17577 -> cust_id#17577, doj#17581 -> doj#17581),CarbonDatasourceRelation(`default`.`uniqdata`,None))],
IncludeProfile(ArrayBuffer(cust_name#17578, ACTIVE_EMUI_VERSION#17579, dob#17580)), CarbonAliasDecoderRelation()
                                                                  +- CarbonScan [cust_id#17577,cust_name#17578,ACTIVE_EMUI_VERSION#17579,dob#17580,doj#17581],
CarbonRelation default, uniqdata, CarbonMetaData(ArrayBuffer(cust_name, active_emui_version,
dob, doj),ArrayBuffer(cust_id, bigint_column1, bigint_column2, decimal_column1, decimal_column2,
double_column1, double_column2, integer_column1),org.apache.carbondata.core.metadata.schema.table.CarbonTable@78bdd0c9,DictionaryMap(Map(cust_name
-> false, active_emui_version -> false, dob -> false, doj -> false))), org.apache.carbondata.processing.merger.TableMeta@474efbce,
None, [(substring(cust_name#17578,0,4) = CUST),length(cust_name#17578) IN (15,14,13,16),ACTIVE_EMUI_VERSION#17579
RLIKE ACTIVE,(month(cast(dob#17580 as date)) = 1),(minute(dob#17580) = 0)], true 
Stacktrace




--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message