carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [29/50] [abbrv] incubator-carbondata git commit: [Bug]fix bug in handle multiple quotes (#761)
Date Thu, 30 Jun 2016 17:42:16 GMT
[Bug]fix bug in handle multiple quotes (#761)

When handle data with quotes,we need to follow the rules:
1)the cell should be enclosed in quote character
2)inside the data eache quote is escaped using escape character.

for example:
"""abc""" -> "abc" (escape char and quote is ")
”\"i\"" ->"i" (escape char is\ ,quote is ")


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/cdfd08c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/cdfd08c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/cdfd08c8

Branch: refs/heads/master
Commit: cdfd08c81a7df47bd99f145e1119fcdb21a8df6c
Parents: 462eb39
Author: linyixin <linyixin@huawei.com>
Authored: Tue Jun 28 01:04:55 2016 +0800
Committer: Vimal-Das <vimaldas.kammath@gmail.com>
Committed: Mon Jun 27 10:04:55 2016 -0700

----------------------------------------------------------------------
 .../resources/datawithcomplexspecialchar.csv    | 151 +++++++++++++++++++
 .../test/resources/datawithspecialcharacter.csv |   1 +
 .../dataload/TestLoadDataWithHiveSyntax.scala   |  25 ++-
 .../csvreaderstep/BlockDataHandler.java         |  46 +++++-
 .../processing/csvreaderstep/CsvInput.java      |   1 +
 .../processing/csvreaderstep/CsvInputData.java  |   1 +
 6 files changed, 220 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/integration/spark/src/test/resources/datawithcomplexspecialchar.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datawithcomplexspecialchar.csv b/integration/spark/src/test/resources/datawithcomplexspecialchar.csv
new file mode 100644
index 0000000..b5d1beb
--- /dev/null
+++ b/integration/spark/src/test/resources/datawithcomplexspecialchar.csv
@@ -0,0 +1,151 @@
+customer_id,124_string_level_province,numeric_level,date_level,Time_level,account_num,lname,fname,mi,address1,address2,address3,address4,city,country,customer_region_id,phone1,phone2,marital_status,yearly_income,gender,total_children,num_children_at_home,education,member_card,occupation,houseowner,num_cars_owned,fullname
+1,Oaxaca,9535837227,26-08-1961,08:08:08,87462024688,Nowmer,Sheri,A.,2433 Bailey Road,,,,Tlaxiaco,Mexico,30,271-555-9715,119-555-1969,M,$30K - $50K,F,4,2,Partial High School,Bronze,Skilled Manual,Y,4,Sheri Nowmer
+2,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87470586299,Whelply,Derrick,I.,2219 Dewing Avenue,,,,Sooke,Canada,101,211-555-7669,807-555-9033,S,$70K - $90K,M,1,0,Partial High School,Bronze,Professional,N,3,Derrick Whelply
+3,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87475757600,Derry,Jeanne,,7640 First Ave.,,,,Issaquah,USA,21,656-555-2272,221-555-2493,M,$50K - $70K,F,1,1,Bachelors Degree,Bronze,Professional,Y,2,Jeanne Derry
+4,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87500482201,Spence,Michael,J.,337 Tosca Way,,,,Burnaby,Canada,92,929-555-7279,272-555-2844,M,$10K - $30K,M,4,4,Partial High School,Normal,Skilled Manual,N,2,Michael Spence
+5,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87514054179,Gutierrez,Maya,,8668 Via Neruda,,,,Novato,USA,42,387-555-7172,260-555-6936,S,$30K - $50K,F,3,0,Partial College,Silver,Manual,N,3,Maya Gutierrez
+6,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87517782449,Damstra,Robert,F.,1619 Stillman Court,,,,Lynnwood,USA,75,922-555-5465,333-555-5915,S,$70K - $90K,F,3,0,Bachelors Degree,Bronze,Professional,Y,3,Robert Damstra
+7,Oaxaca,9535837227,26-08-1961,08:08:08,87521172800,Kanagaki,Rebecca,,2860 D Mt. Hood Circle,,,,Tlaxiaco,Mexico,30,515-555-6247,934-555-9211,M,$30K - $50K,F,2,1,Partial High School,Bronze,Manual,Y,3,Rebecca Kanagaki
+8,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,87539744377,Brunner,Kim,H.,6064 Brodia Court,,,,San Andres,Mexico,106,411-555-6825,130-555-6818,M,$50K - $70K,M,2,2,Bachelors Degree,Bronze,Professional,Y,3,Kim Brunner
+9,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87544797658,Blumberg,Brenda,C.,7560 Trees Drive,,,,Richmond,Canada,90,815-555-3975,642-555-6483,M,$10K - $30K,M,5,3,Partial High School,Normal,Skilled Manual,Y,1,Brenda Blumberg
+10,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,87568712234,Stanz,Darren,M.,1019 Kenwal Rd.,,,,Lake Oswego,USA,64,847-555-5443,212-555-8635,S,$30K - $50K,M,4,0,Bachelors Degree,Golden,Management,N,4,Darren Stanz
+11,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87572821378,Murraiin,Jonathan,V.,5423 Camby Rd.,,,,La Mesa,USA,11,612-555-4878,747-555-6928,S,$50K - $70K,M,4,0,High School Degree,Bronze,Manual,N,2,Jonathan Murraiin
+12,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87579237222,Creek,Jewel,C.,1792 Belmont Rd.,,,,Chula Vista,USA,13,555-555-2714,228-555-5450,S,$30K - $50K,F,1,0,High School Degree,Bronze,Skilled Manual,N,3,Jewel Creek
+13,LonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmembernameLonglevelmember,0,10-05-1951,00:00:00,87587122917,Medina,Peggy,A.,3796 Keller Ridge,,,,Mexico City,Mexico,2,343-555-9778,785-555-2371,S,$30K - $50K,M,4,0,High School Degree,Bronze,Manual,N,4,Peggy Medina
+14,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87592626810,Rutledge,Bryan,K.,3074 Ardith Drive,,,,Lincoln Acres,USA,10,659-555-3160,640-555-5439,M,$50K - $70K,F,2,2,Bachelors Degree,Bronze,Management,Y,2,Bryan Rutledge
+15,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87597749829,Cavestany,Walter,G.,7987 Seawind Dr.,,,,Oak Bay,Canada,99,471-555-8853,560-555-4646,S,$90K - $110K,M,3,0,Graduate Degree,Bronze,Professional,Y,3,Walter Cavestany
+16,select * from scenario1_84gsinglecsv_updatedcolumns_test3;,1.00E+14,02-10-1969,20:20:20,87603285908,Planck,Peggy,M.,4864 San Carlos,,,,Camacho,Mexico,27,698-555-7603,986-555-9424,M,$50K - $70K,F,2,2,High School Degree,Bronze,Skilled Manual,Y,2,Peggy Planck
+17,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87625473141,Marshall,Brenda,S.,2687 Ridge Road,,,,Arcadia,USA,51,771-555-6581,929-555-7260,S,$90K - $110K,M,3,0,High School Degree,Bronze,Management,Y,3,Brenda Marshall
+18,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87637655735,Wolter,Daniel,P.,2473 Orchard Way,,,,Altadena,USA,50,121-555-3960,717-555-5324,M,$30K - $50K,M,3,2,Partial College,Bronze,Professional,Y,4,Daniel Wolter
+19,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87650814652,Collins,Dianne,,551 Rainier Dr,,,,Oakland,USA,37,217-555-1567,921-555-5446,S,$70K - $90K,F,1,0,High School Degree,Bronze,Professional,N,1,Dianne Collins
+20,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87653979700,Baker,Beverly,,591 Merriewood Drive,,,,Spring Valley,USA,15,617-555-7972,860-555-3536,S,$10K - $30K,F,2,0,Partial High School,Normal,Manual,Y,2,Beverly Baker
+21,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87663244009,Castillo,Pedro,,1579 Plaza Rosa,,,,Renton,USA,73,975-555-7865,187-555-2286,S,$30K - $50K,M,1,0,High School Degree,Normal,Manual,Y,2,Pedro Castillo
+22,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87675641200,Borges,Laurie,,1873 Lyon Circle,,,,Bellingham,USA,78,610-555-6158,921-555-6608,M,$30K - $50K,F,4,4,High School Degree,Golden,Skilled Manual,Y,4,Laurie Borges
+23,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87678398489,Wyro,Shauna,K.,3114 Notre Dame Ave.,,,,La Jolla,USA,33,444-555-1517,625-555-9824,S,$30K - $50K,F,2,0,High School Degree,Silver,Manual,N,3,Shauna Wyro
+24,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,87681713700,Wyllie,Jacqueline,,6318 Marclair Dr.,,,,Santa Fe,Mexico,109,102-555-7439,721-555-7431,S,$30K - $50K,F,0,0,High School Degree,Bronze,Manual,N,3,Jacqueline Wyllie
+25,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87686740159,Conley,Lin,N.,7814 Milburn Dr.,,,,Haney,Canada,93,506-555-4589,333-555-7096,S,$70K - $90K,F,3,0,High School Degree,Bronze,Professional,N,2,Lin Conley
+26,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87691326700,Bernard,Jose,,3928 San Francisco,,,,Burbank,USA,53,425-555-3796,452-555-3332,S,$10K - $30K,M,0,0,Partial High School,Normal,Skilled Manual,N,1,Jose Bernard
+27,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87700161933,Macaluso,Charles,B.,5678 McElroy Court,,,,Anacortes,USA,79,852-555-8574,986-555-1624,M,$30K - $50K,F,1,1,High School Degree,Bronze,Manual,Y,3,Charles Macaluso
+28,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87709978818,Wood,Lois,M.,2046 Las Palmas,,,,Everett,USA,81,794-555-6410,467-555-9146,S,$10K - $30K,F,1,0,Partial High School,Silver,Manual,Y,1,Lois Wood
+29,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87717585563,Armstrong,Kevin,A.,608 Lincoln Dr.,,,,Oak Bay,Canada,99,583-555-4474,125-555-6067,S,$10K - $30K,F,2,0,Partial High School,Normal,Manual,N,1,Kevin Armstrong
+30,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,87718932032,Goldey,Cody,A.,7230 Berrellesa Street,,,,Milwaukie,USA,65,548-555-9939,529-555-3217,S,$50K - $70K,F,4,0,Bachelors Degree,Bronze,Professional,Y,3,Cody Goldey
+31,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87729650139,Nickell,Paula,B.,2144 San Rafael,,,,Concord,USA,46,598-555-1289,448-555-2424,M,$50K - $70K,F,4,3,Graduate Degree,Golden,Professional,Y,3,Paula Nickell
+32,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87738383412,Long,Eric,J.,1676 Greer Ave.,,,,Ladner,Canada,96,937-555-2299,326-555-4120,S,$10K - $30K,F,2,0,Partial High School,Normal,Manual,Y,2,Eric Long
+33,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87742803692,Solimena,Mary,K.,6843 San Simeon Dr.,,,,Santa Cruz,USA,40,111-555-1278,268-555-1956,M,$30K - $50K,F,2,2,High School Degree,Bronze,Skilled Manual,N,2,Mary Solimena
+34,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87767067000,Muenich,Nathan,,6629 Polson Circle,,,,La Mesa,USA,11,361-555-7656,957-555-9020,M,$50K - $70K,F,3,1,Bachelors Degree,Silver,Management,Y,2,Nathan Muenich
+35,Yucatan,256,17-04-1920,12:00:00,87771052718,Young,Ed,,Rt. 470 Box A,,,,Merida,Mexico,26,457-555-5263,261-555-9142,M,$30K - $50K,M,5,1,Partial College,Bronze,Skilled Manual,Y,3,Ed Young
+36,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,87788449525,Arnold,Donna,K.,7403 N. Broadway,,,,Oregon City,USA,68,969-555-4928,972-555-5835,S,$10K - $30K,M,1,0,Partial High School,Normal,Skilled Manual,N,1,Donna Arnold
+37,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87797676324,Cooper,Jennifer,K.,5734 Ashford Court,,,,Seattle,USA,18,188-555-1163,299-555-4585,S,$50K - $70K,M,0,0,Bachelors Degree,Bronze,Professional,N,3,Jennifer Cooper
+38,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87801286578,Petty,Peggy,J.,8684 Military East,,,,Arcadia,USA,51,722-555-8456,134-555-8906,S,$30K - $50K,M,1,0,Partial College,Bronze,Management,N,1,Peggy Petty
+39,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,87808394432,Olguin,Jessica,A.,7270 Pepperidge Way,,,,Lebanon,USA,72,203-555-6978,384-555-6285,M,$90K - $110K,F,4,2,Graduate Degree,Bronze,Professional,Y,2,Jessica Olguin
+40,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87814209461,Burchett,Phyllis,,475 Santa Maria,,,,Santa Cruz,USA,40,761-555-3899,719-555-7549,S,$110K - $130K,M,1,0,Partial College,Bronze,Professional,Y,2,Phyllis Burchett
+41,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,87818307900,Bechard,Howard,,1970 Napa Ct.,,,,W. Linn,USA,66,616-555-6966,443-555-9474,S,$110K - $130K,M,1,0,High School Degree,Bronze,Professional,N,2,Howard Bechard
+42,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,87849287390,Carter,Doris,M.,8084 Cardiff Drive,,,,San Andres,Mexico,106,535-555-6174,562-555-5709,M,$70K - $90K,F,3,3,Bachelors Degree,Golden,Management,Y,3,Doris Carter
+43,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87855359911,Sharp,Juanita,D.,9833 Mt. Dias Blv.,,,,Burbank,USA,53,962-555-1952,196-555-4002,S,$50K - $70K,F,2,0,Bachelors Degree,Bronze,Professional,N,4,Juanita Sharp
+44,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87868926525,Brunner,Sandra,B.,6202 Seeno St.,,,,Bellflower,USA,55,904-555-8788,577-555-2523,S,$30K - $50K,M,1,0,High School Degree,Bronze,Skilled Manual,Y,2,Sandra Brunner
+45,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,87869548800,Staton,Ernest,,862 The Alameda,,,,Renton,USA,73,693-555-6852,235-555-8445,S,$50K - $70K,M,5,0,Bachelors Degree,Golden,Management,Y,2,Ernest Staton
+46,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87894418239,Sims,Rose,M.,7484 Roundtree Drive,,,,Mill Valley,USA,38,658-555-3316,639-555-5595,M,$70K - $90K,F,1,1,Graduate Degree,Silver,Professional,N,2,Rose Sims
+47,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87894944000,De Carlo,Lauretta,E.,5053 Loftus Road,,,,Oak Bay,Canada,99,708-555-3666,558-555-4802,M,$50K - $70K,M,0,0,Bachelors Degree,Bronze,Professional,N,1,Lauretta De Carlo
+48,select * from scenario1_84gsinglecsv_updatedcolumns_test3;,1.00E+14,02-10-1969,20:20:20,87901939153,Williams,Mary,R.,1930 Almondwood Drive,,,,Camacho,Mexico,27,696-555-7759,324-555-4238,M,$50K - $70K,M,0,0,Bachelors Degree,Bronze,Professional,N,3,Mary Williams
+49,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87902239561,Burke,Terri,L.,9752 Jeanne Circle,,,,Arcadia,USA,51,770-555-6738,266-555-2074,S,$30K - $50K,F,4,0,High School Degree,Bronze,Skilled Manual,N,2,Terri Burke
+50,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87910291552,Osborn,Audrey,H.,9539 Glenside Dr,,,,Altadena,USA,50,119-555-4116,715-555-5480,S,$110K - $130K,M,4,0,Partial High School,Bronze,Professional,Y,4,Audrey Osborn
+51,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87923139333,Binai,Brian,C.,7616 Honey Court,,,,Oakland,USA,37,216-555-1723,920-555-5602,M,$90K - $110K,F,4,2,Graduate Degree,Bronze,Professional,N,3,Brian Binai
+52,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,87939313100,Lozada,Concepcion,,7657 H St.,,,,Tixapan,Mexico,107,727-555-1388,731-555-2295,S,$70K - $90K,F,3,0,Bachelors Degree,Silver,Professional,N,2,Concepcion Lozada
+53,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87952010600,Tays,Paul,,5988 Red Leaf,,,,Oak Bay,Canada,99,298-555-3541,409-555-6962,M,$70K - $90K,M,4,2,Graduate Degree,Normal,Management,Y,4,Paul Tays
+54,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,87954998600,Thorton,Cheryl,,8938 West Hookston Road,,,,Metchosin,Canada,104,832-555-1834,244-555-2284,S,$30K - $50K,F,0,0,Partial College,Bronze,Professional,Y,3,Cheryl Thorton
+55,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87961350083,Dumin,Gary,,181 Gainsborough Drive,,,,Grossmont,USA,17,313-555-9355,733-555-3320,M,$10K - $30K,M,3,2,Partial High School,Normal,Skilled Manual,Y,1,Gary Dumin
+56,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87965223193,Chin,Pat,M.,3385 Crestview Drive,,,,Imperial Beach,USA,14,871-555-6277,829-555-9927,S,$10K - $30K,M,3,0,Partial High School,Normal,Manual,N,2,Pat Chin
+57,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,87978752300,Lovell,Zach,,6912 Laguna St.,,,,San Andres,Mexico,106,375-555-3427,201-555-5934,S,$10K - $30K,M,3,0,Partial High School,Golden,Manual,Y,0,Zach Lovell
+58,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,87986992001,Ratcliff,Dave,A.,371 Ahneita Dr.,,,,Oakland,USA,37,294-555-2634,560-555-5827,S,$30K - $50K,F,3,0,Partial High School,Bronze,Skilled Manual,N,3,Dave Ratcliff
+59,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88010123400,Moss,Elizabeth,,4776 Kentucky Drive,,,,Burnaby,Canada,92,721-555-7412,855-555-9462,M,$30K - $50K,M,5,5,High School Degree,Bronze,Manual,Y,3,Elizabeth Moss
+60,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88018041300,Anderson,Lori,M.,1145 Paraiso Ct.,,,,Langley,Canada,94,663-555-5248,336-555-7984,S,$30K - $50K,M,3,0,Partial College,Silver,Skilled Manual,Y,3,Lori Anderson
+61,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88049879074,Everson,Michael,A.,3148 Rose Street,,,,Tacoma,USA,84,451-555-3312,894-555-4905,M,$70K - $90K,M,1,0,Bachelors Degree,Bronze,Professional,N,1,Michael Everson
+62,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88067190100,Borden,Mary,A.,2427 Notre Dame Ave.,,,,Santa Monica,USA,49,656-555-3434,636-555-5712,M,$30K - $50K,M,1,1,High School Degree,Golden,Skilled Manual,Y,2,Mary Borden
+63,Oaxaca,9535837227,26-08-1961,08:08:08,88081979800,Willson,Sue,,7340 Green St.,,,,Tlaxiaco,Mexico,30,467-555-9127,317-555-1262,S,$70K - $90K,F,4,0,Bachelors Degree,Bronze,Professional,N,2,Sue Willson
+64,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88084045052,Harris,Clayton,J.,6872 Thornwood Dr.,,,,Beaverton,USA,67,814-555-9899,203-555-2721,S,$10K - $30K,M,4,0,Partial High School,Normal,Manual,Y,2,Clayton Harris
+65,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88096334934,Contreras,Whitney,L.,2040 Encino Drive,,,,Metchosin,Canada,104,648-555-5221,145-555-9556,M,$90K - $110K,F,0,0,Partial College,Normal,Management,Y,3,Whitney Contreras
+66,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88112406730,Pusedu,Boyd,A.,1826 Corte Del Sol,,,,Metchosin,Canada,104,898-555-2599,833-555-7621,M,$110K - $130K,F,4,4,Partial College,Silver,Professional,Y,2,Boyd Pusedu
+67,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88115072846,Stedman,Cian,F.,9903 East Leland,,,,Westminster,Canada,91,994-555-9206,799-555-4085,M,$30K - $50K,M,3,1,Partial College,Bronze,Skilled Manual,Y,3,Cian Stedman
+68,select * from scenario1_84gsinglecsv_updatedcolumns_test3;,1.00E+14,02-10-1969,20:20:20,88118441700,Zocchi,Michele,A.,9943 Stonehedge Dr.,,,,Hidalgo,Mexico,25,606-555-8871,610-555-9778,S,$30K - $50K,F,0,0,High School Degree,Silver,Manual,Y,1,Michele Zocchi
+69,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88137775063,Walsh,Jean,M.,931 Corte De Luna,,,,Los Angeles,USA,3,725-555-5106,837-555-8528,M,$10K - $30K,M,1,1,Partial High School,Normal,Skilled Manual,Y,1,Jean Walsh
+70,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88144207000,Renfro,Leopoldo,E.,1226 Shoenic,,,,Coronado,USA,8,360-555-3399,910-555-7506,M,$30K - $50K,M,0,0,High School Degree,Bronze,Manual,Y,1,Leopoldo Renfro
+71,Sinaloa,1.00E+78,08-04-1967,18:18:18,88161163926,Brockett,Donna,A.,2467 Clearland Circle,,,,La Cruz,Mexico,29,741-555-1921,260-555-4885,S,$10K - $30K,M,5,0,Partial High School,Normal,Manual,N,1,Donna Brockett
+72,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88161249756,Anderson,Laurie,A.,5671 Bel Air Dr.,,,,Shawnee,Canada,105,398-555-7842,356-555-2492,M,$10K - $30K,F,5,5,Partial High School,Normal,Skilled Manual,Y,2,Laurie Anderson
+73,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88168829679,Gomez,Louis,C.,7166 Brock Lane,,,,Yakima,USA,89,802-555-4992,629-555-7499,M,$30K - $50K,F,4,3,High School Degree,Golden,Skilled Manual,Y,3,Louis Gomez
+74,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88172026872,Walker,Julie,L.,3281 Hillview Dr.,,,,Ballard,USA,20,722-555-4199,987-555-7392,S,$50K - $70K,M,3,0,Bachelors Degree,Bronze,Professional,Y,3,Julie Walker
+75,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88182830810,Jones,Jay,W.,5030 Blue Ridge Dr.,,,,Lynnwood,USA,75,248-555-8978,383-555-2028,S,$30K - $50K,F,3,0,High School Degree,Bronze,Manual,N,3,Jay Jones
+76,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88185078501,Winfrey,Gayle,J.,1399 Firestone Drive,,,,Port Orchard,USA,77,191-555-6813,764-555-9549,S,$70K - $90K,F,1,0,High School Degree,Normal,Professional,N,1,Gayle Winfrey
+77,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88203880800,Brown,Yasmina,,6058 Hill Street,#4,,,Downey,USA,58,569-555-5452,351-555-1703,S,$50K - $70K,M,2,0,Bachelors Degree,Normal,Professional,Y,4,Yasmina Brown
+78,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88243577480,Touchstone,Joseph,A. C.,2681 Eagle Peak,,,,Salem,USA,23,774-555-5574,754-555-7853,M,$10K - $30K,M,1,1,Partial High School,Silver,Manual,Y,1,Joseph Touchstone
+79,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88249907493,Bakhtyari,Mary,J.,7594 Alexander Pl.,,,,Tacoma,USA,84,585-555-2267,435-555-3403,S,$70K - $90K,M,2,0,Bachelors Degree,Bronze,Management,Y,4,Mary Bakhtyari
+80,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88262524604,Glass,Melvin,W.,7126 Edie Ct.,,,,Lemon Grove,USA,12,812-555-1017,200-555-2838,S,$70K - $90K,F,1,0,Bachelors Degree,Bronze,Management,Y,4,Melvin Glass
+81,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88265968561,Cohen,Kristin,S.,2294 West 39th St.,,,,Milwaukie,USA,65,646-555-5338,143-555-9674,S,$10K - $30K,M,2,0,Partial High School,Normal,Skilled Manual,Y,2,Kristin Cohen
+82,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88275710344,Kharman,Susan,,2080 Sycamore Drive,,,,Richmond,USA,35,896-555-2717,831-555-7738,S,$10K - $30K,F,4,0,Partial High School,Normal,Skilled Manual,Y,1,Susan Kharman
+83,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88284432888,Kirschner,Gordon,R.,158 Walnut Ave,,,,Arcadia,USA,51,331-555-3981,796-555-4203,S,$110K - $130K,F,3,0,Partial High School,Golden,Management,Y,2,Gordon Kirschner
+84,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88304206132,Burger,William,W.,2854 Magda Way,,,,Haney,Canada,93,604-555-8988,608-555-9895,S,$50K - $70K,M,4,0,Bachelors Degree,Bronze,Professional,N,3,William Burger
+85,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88346847891,Blau,Lana,L.,1185 Dallis Drive,,,,Novato,USA,42,723-555-5224,835-555-8645,M,$30K - $50K,M,3,3,High School Degree,Golden,Skilled Manual,Y,3,Lana Blau
+86,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88352523400,Derby,Donna,,8154 Via Mexico,,,,Bremerton,USA,76,358-555-3517,908-555-7624,S,$30K - $50K,M,4,0,Partial High School,Normal,Skilled Manual,Y,4,Donna Derby
+87,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88355388045,Thomson,Kathleen,K.,6740 Jalisco,,,,N. Vancouver,Canada,97,739-555-2038,258-555-5003,S,$70K - $90K,F,4,0,High School Degree,Bronze,Management,N,3,Kathleen Thomson
+88,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88380010700,Walton,Gabriel,,2601 Cambridge Drive,,,,Milwaukie,USA,65,396-555-7960,466-555-4870,S,$30K - $50K,F,2,0,High School Degree,Bronze,Manual,N,4,Gabriel Walton
+89,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88386346101,Meastas,Bishop,,4096 San Remo,,,,Woodland Hills,USA,62,912-555-7369,739-555-9877,S,$70K - $90K,F,4,0,High School Degree,Bronze,Management,N,2,Bishop Meastas
+90,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88386882543,Kouba,Geneva,,7554 Lakeview Pl.,,,,Sooke,Canada,101,832-555-6577,197-555-9770,M,$70K - $90K,F,2,2,High School Degree,Bronze,Professional,Y,3,Geneva Kouba
+91,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88397842049,Clark,Tricia,F.,1960 Via Catanzaro,,,,Long Beach,USA,48,597-555-6013,732-555-8063,M,$50K - $70K,M,1,1,Bachelors Degree,Bronze,Professional,N,1,Tricia Clark
+92,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88398657441,Munn,Debi,,5672 Hale Dr.,,,,Arcadia,USA,51,540-555-3848,874-555-2927,M,$30K - $50K,M,4,2,High School Degree,Bronze,Skilled Manual,Y,4,Debi Munn
+93,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88416295647,Goldberg,Patricia,,332 Laguna Niguel,,,,Woodburn,USA,69,989-555-7255,770-555-3505,M,$130K - $150K,M,5,4,Bachelors Degree,Golden,Professional,Y,3,Patricia Goldberg
+94,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88416842818,Magenheim,Susan,L.,6954 Garcia Ranch Road,,,,Vancouver,Canada,5,293-555-7377,274-555-9655,S,$10K - $30K,M,1,0,Partial High School,Normal,Skilled Manual,Y,1,Susan Magenheim
+95,Oaxaca,9535837227,26-08-1961,08:08:08,88426868915,Hance,Nancy,D.,4523 Via Cortez,,,,Tlaxiaco,Mexico,30,471-555-9125,320-555-1260,S,$130K - $150K,F,2,0,Graduate Degree,Silver,Professional,Y,2,Nancy Hance
+96,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88445172309,Mehlert,Rhonda,L.,1400 Gibrix Drive,,,,Beaverton,USA,67,698-555-7875,986-555-9696,S,$50K - $70K,M,0,0,Bachelors Degree,Normal,Management,Y,2,Rhonda Mehlert
+97,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88470690800,Duran,Paula,,9222 Roseann Drive,#59,,,Richmond,Canada,90,532-555-3196,928-555-7532,M,$50K - $70K,M,4,4,High School Degree,Golden,Skilled Manual,Y,2,Paula Duran
+98,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88486376400,Earley,Margaret,,9009 W 46th St,,,,Richmond,Canada,90,121-555-4232,717-555-5596,M,$130K - $150K,F,1,1,High School Degree,Bronze,Management,Y,3,Margaret Earley
+99,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88506181836,Horne,Elizabeth,,7086 O St.,,,,Port Orchard,USA,77,217-555-1839,682-555-2060,S,$30K - $50K,F,3,0,Bachelors Degree,Bronze,Professional,N,2,Elizabeth Horne
+100,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88506991900,Hunt,Mary,,7127 Morengo Court,,,,San Jose,USA,39,490-555-6846,493-555-7753,M,$90K - $110K,F,2,2,Partial High School,Silver,Professional,Y,3,Mary Hunt
+101,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88507222533,Allen,Libby,,5458 Encino Drive,,,,N. Vancouver,Canada,97,848-555-6739,721-555-6503,S,$10K - $30K,F,1,0,Partial High School,Normal,Manual,N,1,Libby Allen
+102,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88530171513,Bruno,Bonnie,F.,8408 Via Del Sol,,,,Langford,Canada,102,244-555-1375,794-555-5482,M,$30K - $50K,M,2,0,Bachelors Degree,Bronze,Professional,N,4,Bonnie Bruno
+103,"""state province # 124""",4124,26-01-1950,16:16:16,88531920313,Bowers,Angela,W.,9650 Valley View Road,,,,Acapulco,Mexico,28,737-555-2156,256-555-5120,M,$50K - $70K,M,5,3,Bachelors Degree,Golden,Professional,Y,2,Angela Bowers
+104,select * from scenario1_84gsinglecsv_updatedcolumns_test3;,1.00E+14,02-10-1969,20:20:20,88537885546,Gonzales,Donald,C.,2855 Playa,,,,Hidalgo,Mexico,25,394-555-8077,352-555-2727,M,$30K - $50K,F,1,1,Partial College,Bronze,Clerical,N,2,Donald Gonzales
+105,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88539907932,Valdez,Sylvester,,4350 Mota Dr.,,,,National City,USA,9,137-555-8885,625-555-7734,M,$150K +,M,3,3,Bachelors Degree,Golden,Management,Y,3,Sylvester Valdez
+106,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88556258678,Stewart,John,C.,7808 Brown St.,,,,Long Beach,USA,48,717-555-4435,983-555-7627,S,$50K - $70K,M,5,0,Bachelors Degree,Bronze,Professional,Y,2,John Stewart
+107,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88589957952,Velasquez,Barney,C.,2214 Solano,,,,Cliffside,Canada,103,483-555-3870,618-555-5920,M,$30K - $50K,M,2,2,High School Degree,Bronze,Manual,Y,4,Barney Velasquez
+108,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88593557476,Dubois,Kenneth,R.,8582 San Jose Ave.,,,,Shawnee,Canada,105,425-555-1706,998-555-4442,M,$90K - $110K,M,1,1,Bachelors Degree,Bronze,Management,Y,3,Kenneth Dubois
+109,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88597859740,Maestas,Kate,A.,3241 Brock Lane,,,,Newton,Canada,95,214-555-8770,656-555-1363,S,$50K - $70K,M,2,0,Bachelors Degree,Bronze,Professional,N,2,Kate Maestas
+110,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88604978322,Underwood,Josie,B.,9863 Ridge Place,,,,West Covina,USA,60,179-555-5235,160-555-7513,S,$10K - $30K,M,3,0,Partial High School,Normal,Manual,N,1,Josie Underwood
+111,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88620250821,Strain,Ramon,,4777 Rockne Drive,,,,Palo Alto,USA,41,890-555-1927,979-555-6720,M,$70K - $90K,F,2,2,Bachelors Degree,Bronze,Management,Y,2,Ramon Strain
+112,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88622546792,Azari,Pat,C.,4310 Kenston Dr.,,,,Santa Monica,USA,49,217-555-9677,506-555-2499,M,$30K - $50K,F,4,1,High School Degree,Bronze,Skilled Manual,Y,3,Pat Azari
+113,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88624070286,Dabit,Bob,,9476 Hill Top Rd.,,,,Sooke,Canada,101,291-555-8656,448-555-9335,M,$110K - $130K,F,2,1,Bachelors Degree,Bronze,Professional,Y,3,Bob Dabit
+114,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88631725311,Carmona,Marty,T.,9263 Mt. McKinley Ct.,,,,Sooke,Canada,101,541-555-6035,237-555-7399,S,$150K +,F,3,0,High School Degree,Silver,Professional,Y,2,Marty Carmona
+115,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88638226985,Saxton,Gina,M.,620 Woodside Ct.,,,,Altadena,USA,50,330-555-4138,795-555-4359,S,$90K - $110K,F,3,0,Bachelors Degree,Silver,Professional,Y,2,Gina Saxton
+116,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88648338913,McLaughlin,Juan,R.,661 Miller Ave.,,,,Burnaby,Canada,92,602-555-9145,845-555-4709,S,$90K - $110K,M,2,0,High School Degree,Bronze,Management,N,3,Juan McLaughlin
+117,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88656943440,Eyster,Carol,,1648 Eastgate Lane,,,,Albany,USA,71,960-555-9038,833-555-8802,M,$90K - $110K,F,3,1,Graduate Degree,Bronze,Professional,Y,3,Carol Eyster
+118,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88662506341,Henry,Nancy,J.,4598 Manila Avenue,,,,Bremerton,USA,76,356-555-3673,906-555-7780,S,$50K - $70K,M,2,0,Bachelors Degree,Bronze,Professional,Y,2,Nancy Henry
+119,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88665478229,Barros,Chris,C.,3183 Trasher Road,,,,N. Vancouver,Canada,97,737-555-2195,256-555-5159,S,$10K - $30K,M,4,0,Partial High School,Normal,Manual,Y,1,Chris Barros
+120,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88673519492,Groome,Christopher,J.,6387 Scenic Avenue,,,,Milwaukie,USA,65,634-555-2773,353-555-2766,M,$30K - $50K,F,0,0,High School Degree,Bronze,Skilled Manual,Y,4,Christopher Groome
+121,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88683427572,Hill,Anna,L.,7883 Mitchell Canyon Court,,,,Santa Monica,USA,49,137-555-8923,864-555-2430,S,$30K - $50K,F,5,0,High School Degree,Silver,Manual,Y,2,Anna Hill
+122,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88690803647,Williams,Ramon,N.,3997 Via De Luna,,,,Walla Walla,USA,88,718-555-4473,983-555-7666,M,$30K - $50K,F,4,1,High School Degree,Bronze,Skilled Manual,N,3,Ramon Williams
+123,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88708876371,Masters,Scott,,5747 Shirley Drive,,,,Richmond,USA,35,483-555-3909,618-555-5959,M,$30K - $50K,F,2,0,High School Degree,Bronze,Manual,Y,1,Scott Masters
+124,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88719449639,"Brady, Jr.",Joseph,G.,2115 Pasado,,,,Oakland,USA,37,426-555-1745,999-555-4480,S,$70K - $90K,F,5,0,Bachelors Degree,Bronze,Professional,N,2,"Joseph Brady, Jr."
+125,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88734051585,Gray,Ellen,M.,6774 Bonanza,,,,Bellflower,USA,55,214-555-8809,656-555-1402,M,$70K - $90K,M,2,2,Bachelors Degree,Bronze,Professional,Y,3,Ellen Gray
+126,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88735752105,Fielder,Henry,T.,3397 Rancho View Drive,,,,Ballard,USA,20,179-555-5273,160-555-7552,S,$110K - $130K,F,2,0,Partial College,Normal,Professional,Y,2,Henry Fielder
+127,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88744179606,Finnell,Jeanine,L.,8310 Ridge Circle,,,,Everett,USA,81,103-555-4226,192-555-9019,S,$10K - $30K,M,1,0,Partial High School,Normal,Skilled Manual,Y,0,Jeanine Finnell
+128,1 2 3 4 5 6 7 8 9 0 ~!@#$%^&*() some thing long OR,0,10-05-1951,00:00:00,88750257492,Schuetz,Ian,T.,7842 Ygnacio Valley Road,,,,Salem,USA,23,330-555-2976,618-555-4797,S,$30K - $50K,F,1,0,High School Degree,Bronze,Skilled Manual,N,4,Ian Schuetz
+129,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88752945065,Runyon,Richard,,5666 Hazelnut Lane,,,,Concord,USA,46,403-555-1955,561-555-2633,M,$50K - $70K,F,0,0,Bachelors Degree,Bronze,Professional,Y,3,Richard Runyon
+130,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88771945834,Potter,Linda,M.,5452 Corte Gilberto,,,,Concord,USA,46,653-555-8333,349-555-9697,S,$110K - $130K,F,4,0,Graduate Degree,Bronze,Professional,N,2,Linda Potter
+131,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88772315979,Smith,Mary,E.,874 Olivera Road,,,,La Jolla,USA,33,749-555-5940,315-555-6162,S,$10K - $30K,M,3,0,Partial High School,Normal,Manual,Y,2,Mary Smith
+132,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88781231641,Brumagen,Barbara,A.,3570 Court Lane,,,,Cliffside,Canada,103,122-555-1947,365-555-6512,S,$30K - $50K,F,0,0,High School Degree,Silver,Skilled Manual,N,3,Barbara Brumagen
+133,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88788833022,Hamilton,Thelma,L.,1902 Santa Cruz,,,,Burbank,USA,53,480-555-1840,353-555-1604,M,$30K - $50K,M,1,0,High School Degree,Bronze,Skilled Manual,Y,3,Thelma Hamilton
+134,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88804722428,J Wilbert,Shirley,L.,4852 Chaparral Court,,,,Spokane,USA,87,115-555-9133,426-555-9583,S,$50K - $70K,M,4,0,Bachelors Degree,Bronze,Professional,Y,1,Shirley J Wilbert
+135,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,88820129036,Ingram,Jeremy,T.,6093 Midway Ct.,,,,Tixapan,Mexico,107,257-555-3997,676-555-6962,M,$110K - $130K,M,5,0,Bachelors Degree,Silver,Professional,Y,3,Jeremy Ingram
+136,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88825504183,Haddix,Sharon,A.,9297 Kenston Dr.,,,,Lynnwood,USA,75,153-555-4576,772-555-4569,M,$150K +,M,3,3,Bachelors Degree,Golden,Professional,Y,3,Sharon Haddix
+137,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88846119642,Thomas,Leland,P.,793 Crawford Street,,,,Newport Beach,USA,59,557-555-1726,384-555-4233,M,$130K - $150K,F,3,3,Partial College,Bronze,Professional,Y,3,Leland Thomas
+138,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,88880816698,Wilson,Don,R.,4251 San Onofre Court,,,,Oak Bay,Canada,99,238-555-6276,503-555-9469,S,$90K - $110K,F,4,0,Partial High School,Silver,Professional,Y,2,Don Wilson
+139,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88896459341,Stovall,Wayne,,8656 Lakespring Place,,,,Coronado,USA,8,593-555-6287,728-555-8337,M,$30K - $50K,M,1,1,Partial High School,Bronze,Manual,Y,1,Wayne Stovall
+140,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,88966604471,Russell,Douglas,D.,5025 Holiday Hills,,,,La Mesa,USA,11,536-555-4122,209-555-6858,M,$10K - $30K,M,2,1,Partial High School,Normal,Skilled Manual,Y,1,Douglas Russell
+141,"esc  !@~##%%&**(*&((*()()*  ""  some thing ""  ' DF ",0,10-05-1951,00:00:00,88974345326,Doose,Maureen,,7028 San Gabriel Dr.,,,,Santa Anita,Mexico,108,324-555-2187,767-555-3779,M,$10K - $30K,M,4,3,Partial High School,Normal,Skilled Manual,N,1,Maureen Doose
+142,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,88980621695,Morreale,Ann,,6307 Grenadine Way,,,,Renton,USA,73,289-555-7651,270-555-9929,M,$50K - $70K,F,2,1,Bachelors Degree,Golden,Professional,Y,4,Ann Morreale
+143,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,89028214812,Rupert,Sheila,M.,1220 Bradford Way,,,,Glendale,USA,54,101-555-4344,190-555-9137,M,$30K - $50K,F,5,5,Partial College,Golden,Manual,Y,3,Sheila Rupert
+144,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,89028284549,Pierson,Ashley,E.,1999 Orangewood Court,,,,Westminster,Canada,91,328-555-3094,616-555-4915,S,$10K - $30K,M,3,0,Partial High School,Bronze,Skilled Manual,Y,0,Ashley Pierson
+145,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,89029260873,Nickels,Ivan,,9821 Juniper Drive,,,,Victoria,Canada,6,401-555-2072,559-555-2751,M,$10K - $30K,M,3,3,Partial High School,Normal,Manual,Y,2,Ivan Nickels
+146,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,89035692811,Neuhauser,Chad,D.,9608 Grammercy Lane,,,,Victoria,Canada,6,651-555-8451,347-555-9815,S,$130K - $150K,M,2,0,Partial High School,Bronze,Professional,Y,4,Chad Neuhauser
+147,https://www.google.co.in/?gws_rd=cr&ei=BF7rUqOoEc6GrgeIooHQDQ#q=India+state BC,1.51E+09,15-08-1947,07:14:54,89059253334,Coke,Ann,,7685 Imperial Dr,,,,Sooke,Canada,101,747-555-6058,551-555-9937,S,$110K - $130K,M,4,0,Bachelors Degree,Bronze,Professional,Y,2,Ann Coke
+148,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,89061329364,Scarbrough,Jon,M.,7726 Driftwood Drive,,,,Kirkland,USA,63,120-555-2065,363-555-6629,S,$70K - $90K,M,2,0,High School Degree,Bronze,Management,N,2,Jon Scarbrough
+149,SpecialCharacter level ~!@#$%^&*()_+| CA,0,10-05-1951,00:00:00,89073216915,Martensen,Lillian,F.,8713 Yosemite Ct.,,,,Chula Vista,USA,13,478-555-1958,590-555-5380,S,$50K - $70K,M,1,0,Bachelors Degree,Bronze,Professional,Y,1,Lillian Martensen
+150,&&  1 2 3 4 5 special set to test &  & &  _ _ _  786,1,01-01-1900,23:59:59,89090994596,McCormick,Mary,E.,9008 Creekside Drive,,,,Seattle,USA,18,113-555-9251,424-555-9701,M,$30K - $50K,F,4,1,High School Degree,Bronze,Manual,Y,3,Mary McCormick
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/integration/spark/src/test/resources/datawithspecialcharacter.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datawithspecialcharacter.csv b/integration/spark/src/test/resources/datawithspecialcharacter.csv
index f807120..12cc8fa 100644
--- a/integration/spark/src/test/resources/datawithspecialcharacter.csv
+++ b/integration/spark/src/test/resources/datawithspecialcharacter.csv
@@ -35,3 +35,4 @@ imei,specialchar
 1AA33,fslash/hi
 1AA34,a@b
 1AA35,\\n
+1AA36,"\"i\""

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index a75f020..1a6259a 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -257,7 +257,8 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
     ("ayush@b.com"),Row("ayushb.com")))
     sql("DROP TABLE IF EXISTS t3")
   }
-  test("test carbon table data loading with special character") {
+
+  test("test carbon table data loading with special character 1") {
     sql("DROP TABLE IF EXISTS t3")
 
     sql("""
@@ -267,12 +268,30 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
 
     sql("""
        LOAD DATA LOCAL INPATH './src/test/resources/datawithspecialcharacter.csv' into table t3
-          options ('DELIMITER'=',', 'QUOTECHAR'='\"')
+          options ('DELIMITER'=',', 'QUOTECHAR'='"')
        """)
-    checkAnswer(sql("select count(*) from t3"), Seq(Row(36)))
+    checkAnswer(sql("select count(*) from t3"), Seq(Row(37)))
+    checkAnswer(sql("select specialchar from t3 where imei='1AA36'"),Seq(Row("\"i\"")))
     sql("DROP TABLE IF EXISTS t3")
   }
 
+  test("test carbon table data loading with special character 2") {
+    sql("DROP TABLE IF EXISTS t3")
+
+    sql("""
+        CREATE table t3(customer_id int, 124_string_level_province String, date_level String,
+        Time_level String, lname String, fname String, mi String, address1 String, address2 String, address3 String, address4 String, city String, country String, phone1 String, phone2 String, marital_status String, yearly_income String, gender String, education String, member_card String, occupation String, houseowner String, fullname String, numeric_level double, account_num double, customer_region_id int, total_children int, num_children_at_home int, num_cars_owned int)
+        STORED BY 'org.apache.carbondata.format'
+        """)
+
+    sql("""
+       LOAD DATA LOCAL INPATH './src/test/resources/datawithcomplexspecialchar.csv' into
+       table t3 options ('DELIMITER'=',', 'QUOTECHAR'='"','ESCAPECHAR'='"')
+        """)
+    checkAnswer(sql("select count(*) from t3"), Seq(Row(150)))
+    checkAnswer(sql("select 124_string_level_province from t3 where customer_id=103"),Seq(Row("\"state province # 124\"")))
+    sql("DROP TABLE IF EXISTS t3")
+  }
 
   override def afterAll {
     sql("drop table carbontable")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/processing/src/main/java/org/carbondata/processing/csvreaderstep/BlockDataHandler.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/BlockDataHandler.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/BlockDataHandler.java
index 4a56b85..e1201d7 100644
--- a/processing/src/main/java/org/carbondata/processing/csvreaderstep/BlockDataHandler.java
+++ b/processing/src/main/java/org/carbondata/processing/csvreaderstep/BlockDataHandler.java
@@ -136,6 +136,31 @@ public class BlockDataHandler {
     return false;
   }
 
+  /**
+   * <pre>
+   * [abcd "" defg] --> [abcd " defg]
+   * [""""] --> [""]
+   * [""] --> ["]
+   * </pre>
+   *
+   * @return the byte array with escaped enclosures escaped.
+   */
+  public byte[] removeEscapedEnclosures(byte[] field, int nrEnclosuresFound) {
+    byte[] result = new byte[field.length - nrEnclosuresFound];
+    int resultIndex = 0;
+    for (int i = 0; i < field.length; i++) {
+      if (field[i] == data.enclosure[0]) {
+        if (!(i + 1 < field.length && field[i + 1] == data.enclosure[0])) {
+          // Not an escaped enclosure...
+          result[resultIndex++] = field[i];
+        }
+      } else {
+        result[resultIndex++] = field[i];
+      }
+    }
+    return result;
+  }
+
   public byte[] removeEscapeChar(byte[] field, byte[] escapeChar) {
     byte[] result = new byte[field.length];
     int resultIndex = 0;
@@ -298,6 +323,8 @@ public class BlockDataHandler {
         boolean enclosureFound = false;
         boolean quoteAfterDelimiter = false;
         boolean quoteBeforeDelimiterOrCrLf = false;
+        int escapedEnclosureFound = 0;
+        boolean escapedEnclosureFlag = false;
         while (!delimiterFound) {
           // If we find the first char, we might find others as well ;-)
           // Single byte delimiters only for now.
@@ -406,6 +433,10 @@ public class BlockDataHandler {
                       data.enclosure);
               if (!keepGoing) {
                 outOfEnclosureFlag = !outOfEnclosureFlag;
+                if(this.endBuffer -1 >= 0 && data.escapeCharMatcher.matchesPattern(this
+                    .byteBuffer,this.endBuffer - 1,data.escapeCharacter)){
+                  escapedEnclosureFlag = true;
+                }
                 // We found an enclosure character.
                 // Read another byte...
                 if (this.increaseEndBuffer()) {
@@ -422,6 +453,9 @@ public class BlockDataHandler {
                         data.enclosure);
                 if (keepGoing) {
                   outOfEnclosureFlag = !outOfEnclosureFlag;
+                  if (!escapedEnclosureFlag || data.enclosure[0] == data.escapeCharacter[0]) {
+                    escapedEnclosureFound++;
+                  }
                 } else {
                   /**
                    * <pre>
@@ -445,7 +479,11 @@ public class BlockDataHandler {
                 }
 
               }
-              if (!keepGoing) {
+              if (this.endBuffer - 1 >= 0
+                  && data.delimiterMatcher.matchesPattern(this.byteBuffer, this.endBuffer,
+                  data.delimiter)
+                  || data.crLfMatcher.isReturn(this.byteBuffer, this.endBuffer)
+                  || data.crLfMatcher.isLineFeed(this.byteBuffer, this.endBuffer)) {
                 if (data.enclosureMatcher
                     .matchesPattern(this.byteBuffer, this.endBuffer - 1, data.enclosure)) {
                   quoteBeforeDelimiterOrCrLf = true;
@@ -507,7 +545,11 @@ public class BlockDataHandler {
         byte[] field = new byte[length];
         System.arraycopy(this.byteBuffer, this.startBuffer, field, 0, length);
 
-        if(quoteAfterDelimiter && quoteBeforeDelimiterOrCrLf){
+        if (escapedEnclosureFound > 0 && quoteAfterDelimiter && quoteBeforeDelimiterOrCrLf) {
+          field = this.removeEscapedEnclosures(field, escapedEnclosureFound);
+        }
+
+        if(doConversions && quoteAfterDelimiter && data.escapeCharacter[0] != data.enclosure[0]){
           field = removeEscapeChar(field,data.escapeCharacter);
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInput.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInput.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInput.java
index 42f20cc..a1a3618 100644
--- a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInput.java
+++ b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInput.java
@@ -569,6 +569,7 @@ public class CsvInput extends BaseStep implements StepInterface {
         }
       }
 
+      data.escapeCharMatcher = new SingleBytePatternMatcher();
       switch (data.encodingType) {
         case DOUBLE_BIG_ENDIAN:
           data.crLfMatcher = new MultiByteBigCrLfMatcher();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cdfd08c8/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInputData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInputData.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInputData.java
index f97afed..d28b77b 100644
--- a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInputData.java
+++ b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CsvInputData.java
@@ -45,6 +45,7 @@ public class CsvInputData extends BaseStepData implements StepDataInterface {
   public EncodingType encodingType;
   public PatternMatcherInterface delimiterMatcher;
   public PatternMatcherInterface enclosureMatcher;
+  public PatternMatcherInterface escapeCharMatcher;
   public CrLfMatcherInterface crLfMatcher;
 
   public CsvInputData() {


Mime
View raw message