asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject incubator-asterixdb git commit: Issue 867: Test cases for CSV with headers and various line separators.
Date Fri, 01 May 2015 02:37:34 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master 4a9a7d998 -> d6f782803


Issue 867: Test cases for CSV with headers and various line separators.

Change-Id: I046aa30824d14cbc894a28719c6c6560f46d0133
Reviewed-on: https://asterix-gerrit.ics.uci.edu/247
Reviewed-by: Ian Maxon <imaxon@uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Preston Carman <ecarm002@ucr.edu>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/d6f78280
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/d6f78280
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/d6f78280

Branch: refs/heads/master
Commit: d6f782803e5e883619eeaa69af1e16c0edff2bbc
Parents: 4a9a7d9
Author: Chris Hillery <chillery@lambda.nu>
Authored: Thu Apr 30 17:07:05 2015 -0700
Committer: Chris Hillery <ceej@lambda.nu>
Committed: Thu Apr 30 19:27:41 2015 -0700

----------------------------------------------------------------------
 .gitattributes                                  |  17 +
 asterix-app/data/csv/sample_08_header.csv       |   9 -
 asterix-app/data/csv/sample_08_header.csv.cr    |   1 +
 asterix-app/data/csv/sample_08_header.csv.crlf  |   9 +
 asterix-app/data/csv/sample_08_header.csv.lf    |   9 +
 .../queries/load/csv_08/csv_08.1.ddl.aql        |  22 -
 .../queries/load/csv_08/csv_08.2.update.aql     |  12 -
 .../queries/load/csv_08/csv_08.3.query.aql      |  18 -
 .../load/csv_08_header_cr/csv_08.1.ddl.aql      |  22 +
 .../load/csv_08_header_cr/csv_08.2.update.aql   |  12 +
 .../load/csv_08_header_cr/csv_08.3.query.aql    |  18 +
 .../load/csv_08_header_crlf/csv_08.1.ddl.aql    |  22 +
 .../load/csv_08_header_crlf/csv_08.2.update.aql |  12 +
 .../load/csv_08_header_crlf/csv_08.3.query.aql  |  18 +
 .../load/csv_08_header_lf/csv_08.1.ddl.aql      |  22 +
 .../load/csv_08_header_lf/csv_08.2.update.aql   |  12 +
 .../load/csv_08_header_lf/csv_08.3.query.aql    |  18 +
 .../src/test/resources/runtimets/testsuite.xml  |  12 +-
 .../operators/file/DelimitedDataParser.java     |  11 +-
 .../src/main/resources/Catalog.xsd              | 442 +++++++++----------
 20 files changed, 428 insertions(+), 290 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/.gitattributes
----------------------------------------------------------------------
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..803e30c
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,17 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Explicitly declare text files you want to always be normalized and converted
+# to native line endings on checkout.
+*.java text
+*.xml text
+*.xsd text
+
+# Declare files that will always have CRLF line endings on checkout.
+*.crlf text eol=crlf
+
+# Declare files that will always have CR line endings on checkout.
+*.cr text eol=cr
+
+# Declare files that will always have LF line endings on checkout.
+*.lf test eol=lf

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/data/csv/sample_08_header.csv
----------------------------------------------------------------------
diff --git a/asterix-app/data/csv/sample_08_header.csv b/asterix-app/data/csv/sample_08_header.csv
deleted file mode 100644
index 7444e77..0000000
--- a/asterix-app/data/csv/sample_08_header.csv
+++ /dev/null
@@ -1,9 +0,0 @@
-id,float,"double","date",time,datetime
-1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034
-2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002
-3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001
-4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001
-5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001
-6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001
-7,0.572733058,192674,-1923-03-24,19:33:34,
-8,,,,,
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/data/csv/sample_08_header.csv.cr
----------------------------------------------------------------------
diff --git a/asterix-app/data/csv/sample_08_header.csv.cr b/asterix-app/data/csv/sample_08_header.csv.cr
new file mode 100644
index 0000000..a539b24
--- /dev/null
+++ b/asterix-app/data/csv/sample_08_header.csv.cr
@@ -0,0 +1 @@
+id,float,"double","date",time,datetime
1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034
2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002
3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001
4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001
5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001
6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001
7,0.572733058,192674,-1923-03-24,19:33:34,
8,,,,,
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/data/csv/sample_08_header.csv.crlf
----------------------------------------------------------------------
diff --git a/asterix-app/data/csv/sample_08_header.csv.crlf b/asterix-app/data/csv/sample_08_header.csv.crlf
new file mode 100644
index 0000000..168340b
--- /dev/null
+++ b/asterix-app/data/csv/sample_08_header.csv.crlf
@@ -0,0 +1,9 @@
+id,float,"double","date",time,datetime
+1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034
+2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002
+3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001
+4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001
+5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001
+6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001
+7,0.572733058,192674,-1923-03-24,19:33:34,
+8,,,,,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/data/csv/sample_08_header.csv.lf
----------------------------------------------------------------------
diff --git a/asterix-app/data/csv/sample_08_header.csv.lf b/asterix-app/data/csv/sample_08_header.csv.lf
new file mode 100644
index 0000000..168340b
--- /dev/null
+++ b/asterix-app/data/csv/sample_08_header.csv.lf
@@ -0,0 +1,9 @@
+id,float,"double","date",time,datetime
+1,0.899682764,5.6256,2013-08-07,07:22:35,1979-02-25T23:48:27.034
+2,0.669052398,,-1923-03-29,19:33:34,-1979-02-25T23:48:27.002
+3,0.572733058,192674,-1923-03-28,19:33:34,-1979-02-25T23:48:27.001
+4,,192674,-1923-03-27,19:33:34,-1979-02-25T23:48:27.001
+5,0.572733058,192674,,19:33:34,-1979-02-25T23:48:27.001
+6,0.572733058,192674,-1923-03-25,,-1979-02-25T23:48:27.001
+7,0.572733058,192674,-1923-03-24,19:33:34,
+8,,,,,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.1.ddl.aql
deleted file mode 100644
index 2e441b2..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.1.ddl.aql
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- *
- * CSV file loading test
- * Expected result: success
- *
- */
-
-drop dataverse temp if exists;
-create dataverse temp
-use dataverse temp;
-
-create type test as closed {
-    id: int64,
-    float: float?,
-    double: double?,
-    date: string?,
-    time: string?,
-    datetime: string?
-};
-
-create dataset testds (test)
-primary key id;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.2.update.aql
deleted file mode 100644
index e8eecf1..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.2.update.aql
+++ /dev/null
@@ -1,12 +0,0 @@
-/**
- *
- * CSV file loading test
- * Expected result: success
- *
- */
-
-use dataverse temp;
-
-load dataset testds
-using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
-(("path"="nc1://data/csv/sample_08_header.csv"),("format"="delimited-text"),("header"="true"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.3.query.aql
deleted file mode 100644
index efa6dbc..0000000
--- a/asterix-app/src/test/resources/runtimets/queries/load/csv_08/csv_08.3.query.aql
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- *
- * CSV file loading test
- * Expected result: success
- *
- */
-
-use dataverse temp;
-
-for $i in dataset testds
-order by $i.id
-return { "id": $i.id, 
-         "float": $i.float, 
-         "double": $i.double, 
-         "date-before": $i.date, "date-after": date($i.date), 
-         "time-before": $i.time, "time-after": time($i.time),
-         "datetime-before": $i.datetime, "datetime-after": datetime($i.datetime)
-         }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.1.ddl.aql
new file mode 100644
index 0000000..2e441b2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.1.ddl.aql
@@ -0,0 +1,22 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+drop dataverse temp if exists;
+create dataverse temp
+use dataverse temp;
+
+create type test as closed {
+    id: int64,
+    float: float?,
+    double: double?,
+    date: string?,
+    time: string?,
+    datetime: string?
+};
+
+create dataset testds (test)
+primary key id;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.2.update.aql
new file mode 100644
index 0000000..f6a2017
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.2.update.aql
@@ -0,0 +1,12 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+load dataset testds
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/csv/sample_08_header.csv.cr"),("format"="delimited-text"),("header"="true"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.3.query.aql
new file mode 100644
index 0000000..efa6dbc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_cr/csv_08.3.query.aql
@@ -0,0 +1,18 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+for $i in dataset testds
+order by $i.id
+return { "id": $i.id, 
+         "float": $i.float, 
+         "double": $i.double, 
+         "date-before": $i.date, "date-after": date($i.date), 
+         "time-before": $i.time, "time-after": time($i.time),
+         "datetime-before": $i.datetime, "datetime-after": datetime($i.datetime)
+         }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.1.ddl.aql
new file mode 100644
index 0000000..2e441b2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.1.ddl.aql
@@ -0,0 +1,22 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+drop dataverse temp if exists;
+create dataverse temp
+use dataverse temp;
+
+create type test as closed {
+    id: int64,
+    float: float?,
+    double: double?,
+    date: string?,
+    time: string?,
+    datetime: string?
+};
+
+create dataset testds (test)
+primary key id;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.2.update.aql
new file mode 100644
index 0000000..7634ebb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.2.update.aql
@@ -0,0 +1,12 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+load dataset testds
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/csv/sample_08_header.csv.crlf"),("format"="delimited-text"),("header"="true"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.3.query.aql
new file mode 100644
index 0000000..efa6dbc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_crlf/csv_08.3.query.aql
@@ -0,0 +1,18 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+for $i in dataset testds
+order by $i.id
+return { "id": $i.id, 
+         "float": $i.float, 
+         "double": $i.double, 
+         "date-before": $i.date, "date-after": date($i.date), 
+         "time-before": $i.time, "time-after": time($i.time),
+         "datetime-before": $i.datetime, "datetime-after": datetime($i.datetime)
+         }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.1.ddl.aql
new file mode 100644
index 0000000..2e441b2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.1.ddl.aql
@@ -0,0 +1,22 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+drop dataverse temp if exists;
+create dataverse temp
+use dataverse temp;
+
+create type test as closed {
+    id: int64,
+    float: float?,
+    double: double?,
+    date: string?,
+    time: string?,
+    datetime: string?
+};
+
+create dataset testds (test)
+primary key id;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.2.update.aql
new file mode 100644
index 0000000..4a7c203
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.2.update.aql
@@ -0,0 +1,12 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+load dataset testds
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/csv/sample_08_header.csv.lf"),("format"="delimited-text"),("header"="true"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.3.query.aql
new file mode 100644
index 0000000..efa6dbc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/load/csv_08_header_lf/csv_08.3.query.aql
@@ -0,0 +1,18 @@
+/**
+ *
+ * CSV file loading test
+ * Expected result: success
+ *
+ */
+
+use dataverse temp;
+
+for $i in dataset testds
+order by $i.id
+return { "id": $i.id, 
+         "float": $i.float, 
+         "double": $i.double, 
+         "date-before": $i.date, "date-after": date($i.date), 
+         "time-before": $i.time, "time-after": time($i.time),
+         "datetime-before": $i.datetime, "datetime-after": datetime($i.datetime)
+         }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index b571514..9df6301 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -5943,7 +5943,17 @@
             </compilation-unit>
         </test-case>
         <test-case FilePath="load">
-            <compilation-unit name="csv_08">
+            <compilation-unit name="csv_08_header_cr">
+                <output-dir compare="Text">csv_08</output-dir>
+            </compilation-unit>
+        </test-case>
+        <test-case FilePath="load">
+            <compilation-unit name="csv_08_header_lf">
+                <output-dir compare="Text">csv_08</output-dir>
+            </compilation-unit>
+        </test-case>
+        <test-case FilePath="load">
+            <compilation-unit name="csv_08_header_crlf">
                 <output-dir compare="Text">csv_08</output-dir>
             </compilation-unit>
         </test-case>

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
index 0dd9014..fc38d37 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
@@ -51,7 +51,6 @@ public class DelimitedDataParser extends AbstractDataParser implements IDataPars
     private int[] fldIds;
     private ArrayBackedValueStorage[] nameBuffers;
     private boolean areAllNullFields;
-    private int fieldCount;
 
     public DelimitedDataParser(ARecordType recordType, IValueParserFactory[] valueParserFactories,
char fieldDelimter,
             char quote, boolean hasHeader) {
@@ -105,19 +104,18 @@ public class DelimitedDataParser extends AbstractDataParser implements
IDataPars
 
     @Override
     public boolean parse(DataOutput out) throws AsterixException, IOException {
-        if (hasHeader && cursor.lineCount == 1) {
+        if (hasHeader && cursor.recordCount == 0) {
             // Consume all fields of first record
             cursor.nextRecord();
-            while (cursor.nextField(fieldCount));
+            while (cursor.nextField());
         }
         while (cursor.nextRecord()) {
             recBuilder.reset(recordType);
             recBuilder.init();
             areAllNullFields = true;
             
-            fieldCount = 0;
             for (int i = 0; i < valueParsers.length; ++i) {
-                if (!cursor.nextField(fieldCount)) {
+                if (!cursor.nextField()) {
                     break;
                 }
                 fieldValueBuffer.reset();
@@ -129,7 +127,7 @@ public class DelimitedDataParser extends AbstractDataParser implements
IDataPars
                     // empty string
                     if (recordType.getFieldTypes()[i].getTypeTag() != ATypeTag.UNION
                             || !NonTaggedFormatUtil.isOptionalField((AUnionType) recordType.getFieldTypes()[i]))
{
-                        throw new AsterixException("At line: " + cursor.lineCount + " - Field
" + i
+                        throw new AsterixException("At record: " + cursor.recordCount + "
- Field " + cursor.fieldCount
                                 + " is not an optional type so it cannot accept null value.
");
                     }
                     fieldValueBufferOutput.writeByte(ATypeTag.NULL.serialize());
@@ -151,7 +149,6 @@ public class DelimitedDataParser extends AbstractDataParser implements
IDataPars
                 } else {
                     recBuilder.addField(fldIds[i], fieldValueBuffer);
                 }
-                fieldCount++;
             }
 
             if (!areAllNullFields) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d6f78280/asterix-test-framework/src/main/resources/Catalog.xsd
----------------------------------------------------------------------
diff --git a/asterix-test-framework/src/main/resources/Catalog.xsd b/asterix-test-framework/src/main/resources/Catalog.xsd
index e43acdc..65ba7a4 100644
--- a/asterix-test-framework/src/main/resources/Catalog.xsd
+++ b/asterix-test-framework/src/main/resources/Catalog.xsd
@@ -1,221 +1,221 @@
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
-   xmlns:test="urn:xml.testframework.asterix.ics.uci.edu"
-   targetNamespace="urn:xml.testframework.asterix.ics.uci.edu" elementFormDefault="qualified">
-
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-   <!-- test-suite - top level element                                       -->
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-
-   <xs:element name="test-suite">
-      <xs:annotation>
-         <xs:documentation>
-            This is the top level element for documents that use this schema.
-         </xs:documentation>
-      </xs:annotation>
-
-      <xs:complexType>
-         <xs:sequence>
-            <xs:element ref="test:test-group" maxOccurs="unbounded"/>
-         </xs:sequence>
-
-         <xs:attribute name="CatalogDesignDate" type="xs:date" use="required"/>
-
-         <xs:attribute name="ResultOffsetPath" type="test:SimplifiedRelativeFilePath"
use="required">
-            <xs:annotation>
-               <xs:documentation>
-                  offset from root to results
-               </xs:documentation>
-            </xs:annotation>
-         </xs:attribute>
-
-         <xs:attribute name="QueryOffsetPath" type="test:SimplifiedRelativeFilePath"
-            use="required">
-            <xs:annotation>
-               <xs:documentation>
-                  offset from root to Query expression files
-               </xs:documentation>
-            </xs:annotation>
-         </xs:attribute>
-
-      </xs:complexType>
-
-      <xs:unique name="unique-test-group">
-         <xs:selector xpath=".//test:test-group"/>
-         <xs:field xpath="@name"/>
-      </xs:unique>
-
-   </xs:element>
-
-
-   <!-- SimplifiedRelativeFilePath type                                      -->
-
-   <xs:simpleType name="SimplifiedRelativeFilePath">
-      <xs:restriction base="xs:anyURI">
-         <xs:pattern value="([a-zA-Z0-9\-\.]+/)+"/>
-      </xs:restriction>
-   </xs:simpleType>
-
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-   <!-- test-group                                                           -->
-   <!--                                                                      -->
-   <!-- Group of test cases and test groups.                                 -->
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-
-   <xs:element name="test-group">
-      <xs:annotation>
-         <xs:documentation>
-            Group of test cases and test groups.
-         </xs:documentation>
-      </xs:annotation>
-
-      <xs:complexType>
-         <xs:sequence>
-            <xs:element name="test-case" type="test:test-case" minOccurs="0" maxOccurs="unbounded">
-               <xs:unique name="unique-expected-error">
-                  <xs:selector xpath=".//test:expected-error"/>
-                  <xs:field xpath="."/>
-               </xs:unique>
-            </xs:element>
-
-            <xs:element ref="test:test-group" minOccurs="0" maxOccurs="unbounded"/>
-         </xs:sequence>
-         <xs:attribute name="name" type="xs:string" use="required"/>
-      </xs:complexType>
-   </xs:element>
-
-
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-   <!-- test-case                                                            -->
-   <!--                                                                      -->
-   <!-- A test case to be run.                                               -->
-   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
-
-   <xs:complexType name="test-case">
-      <xs:sequence>
-         <xs:element name="description" type="test:description"/>
-
-         <xs:element name="compilation-unit" minOccurs="1" maxOccurs="unbounded">
-            <xs:complexType>
-               <xs:sequence>
-                  <xs:element name="description" type="test:description" minOccurs="0"/>
-                  <xs:element name="output-dir" minOccurs="0">
-                     <xs:annotation>
-                        <xs:documentation>
-                           Zero or one file containing expected results for this query.
-                        </xs:documentation>
-                     </xs:annotation>
-                     <xs:complexType>
-                        <xs:simpleContent>
-                           <xs:extension base="xs:string">
-                              <xs:attribute name="compare" type="test:comparison-enum"
use="required"/>
-                           </xs:extension>
-                        </xs:simpleContent>
-                     </xs:complexType>
-                  </xs:element>
-
-
-                  <!-- Zero or more expected errors for this query -->
-
-                  <xs:element name="expected-error" minOccurs="0" maxOccurs="unbounded">
-                     <xs:annotation>
-                        <xs:documentation>
-                           Zero or more expected errors for this query.
-                        </xs:documentation>
-                     </xs:annotation>
-
-                     <xs:complexType>
-                        <xs:simpleContent>
-                           <xs:extension base="test:ErrorCode">
-                           </xs:extension>
-                        </xs:simpleContent>
-                     </xs:complexType>
-                  </xs:element>
-
-               </xs:sequence>
-
-               <!-- This name is always equal to the name of the test case -->
-               <xs:attribute name="name" type="xs:string" use="required"/>
-
-            </xs:complexType>
-         </xs:element>
-
-         <!-- Zero or more files containing expected results for this query -->
-
-      </xs:sequence>
-
-      <!-- The filename for this query can be constructed from:              -->
-      <!--    the QueryOffsetPath                                            -->
-      <!--    the FilePath                                                   -->
-      <!--    the name                                                       -->
-      <!--    the QueryFileExtension                                         -->
-
-      <xs:attribute name="FilePath" type="test:SimplifiedRelativeFilePath" use="required"/>
-      <xs:attribute name="date" type="xs:date" use="required"/>
-      <xs:attribute name="category" type="test:category-enum"/>
-   </xs:complexType>
-
-   <!-- category-enum type                                                   -->
-   <!--    Identify which category of test this is. Currently only "slow".   -->
-
-   <xs:simpleType name="category-enum">
-      <xs:annotation>
-         <xs:documentation>
-            Identify the category of test, for limiting when it is run.
-         </xs:documentation>
-      </xs:annotation>
-
-      <xs:restriction base="xs:string">
-         <xs:enumeration value="slow"/>
-      </xs:restriction>
-   </xs:simpleType>
-
-   <!-- comparison-enum type                                                 -->
-   <!--    Identify the type of comparison used to determine whether an      -->
-   <!--    expected result and an actual result match.                       -->
-
-   <xs:simpleType name="comparison-enum">
-      <xs:annotation>
-         <xs:documentation>
-            Identify the type of comparison used to determine whether an
-            expected result and an actual result match.
-         </xs:documentation>
-      </xs:annotation>
-
-      <xs:restriction base="xs:string">
-         <xs:enumeration value="XML"/>
-         <xs:enumeration value="Text"/>
-         <xs:enumeration value="Inspect"/>
-         <xs:enumeration value="Ignore"/>
-         <xs:enumeration value="JSON"/>
-         <xs:enumeration value="CSV"/>
-         <xs:enumeration value="CSV_Header"/>
-      </xs:restriction>
-   </xs:simpleType>
-
-   <!-- description type                                                     -->
-
-   <xs:complexType name="description">
-      <xs:simpleContent>
-         <xs:extension base="xs:string">
-            <xs:attribute name="last-mod" type="xs:date"/>
-         </xs:extension>
-      </xs:simpleContent>
-   </xs:complexType>
-
-
-   <!-- ErrorCode type                                                       -->
-   <!--   * is used to mean that any error code is acceptable                -->
-
-   <xs:simpleType name="ErrorCode">
-      <xs:annotation>
-         <xs:documentation>
-            * is used to mean that any error code is acceptable
-         </xs:documentation>
-      </xs:annotation>
-
-      <xs:restriction base="xs:string">
-         <xs:pattern value="\*|([A-Z]{4}[0-9]{4})"/>
-      </xs:restriction>
-   </xs:simpleType>
-   
-</xs:schema>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:test="urn:xml.testframework.asterix.ics.uci.edu"
+   targetNamespace="urn:xml.testframework.asterix.ics.uci.edu" elementFormDefault="qualified">
+
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+   <!-- test-suite - top level element                                       -->
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+
+   <xs:element name="test-suite">
+      <xs:annotation>
+         <xs:documentation>
+            This is the top level element for documents that use this schema.
+         </xs:documentation>
+      </xs:annotation>
+
+      <xs:complexType>
+         <xs:sequence>
+            <xs:element ref="test:test-group" maxOccurs="unbounded"/>
+         </xs:sequence>
+
+         <xs:attribute name="CatalogDesignDate" type="xs:date" use="required"/>
+
+         <xs:attribute name="ResultOffsetPath" type="test:SimplifiedRelativeFilePath"
use="required">
+            <xs:annotation>
+               <xs:documentation>
+                  offset from root to results
+               </xs:documentation>
+            </xs:annotation>
+         </xs:attribute>
+
+         <xs:attribute name="QueryOffsetPath" type="test:SimplifiedRelativeFilePath"
+            use="required">
+            <xs:annotation>
+               <xs:documentation>
+                  offset from root to Query expression files
+               </xs:documentation>
+            </xs:annotation>
+         </xs:attribute>
+
+      </xs:complexType>
+
+      <xs:unique name="unique-test-group">
+         <xs:selector xpath=".//test:test-group"/>
+         <xs:field xpath="@name"/>
+      </xs:unique>
+
+   </xs:element>
+
+
+   <!-- SimplifiedRelativeFilePath type                                      -->
+
+   <xs:simpleType name="SimplifiedRelativeFilePath">
+      <xs:restriction base="xs:anyURI">
+         <xs:pattern value="([a-zA-Z0-9\-\.]+/)+"/>
+      </xs:restriction>
+   </xs:simpleType>
+
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+   <!-- test-group                                                           -->
+   <!--                                                                      -->
+   <!-- Group of test cases and test groups.                                 -->
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+
+   <xs:element name="test-group">
+      <xs:annotation>
+         <xs:documentation>
+            Group of test cases and test groups.
+         </xs:documentation>
+      </xs:annotation>
+
+      <xs:complexType>
+         <xs:sequence>
+            <xs:element name="test-case" type="test:test-case" minOccurs="0" maxOccurs="unbounded">
+               <xs:unique name="unique-expected-error">
+                  <xs:selector xpath=".//test:expected-error"/>
+                  <xs:field xpath="."/>
+               </xs:unique>
+            </xs:element>
+
+            <xs:element ref="test:test-group" minOccurs="0" maxOccurs="unbounded"/>
+         </xs:sequence>
+         <xs:attribute name="name" type="xs:string" use="required"/>
+      </xs:complexType>
+   </xs:element>
+
+
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+   <!-- test-case                                                            -->
+   <!--                                                                      -->
+   <!-- A test case to be run.                                               -->
+   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
+
+   <xs:complexType name="test-case">
+      <xs:sequence>
+         <xs:element name="description" type="test:description"/>
+
+         <xs:element name="compilation-unit" minOccurs="1" maxOccurs="unbounded">
+            <xs:complexType>
+               <xs:sequence>
+                  <xs:element name="description" type="test:description" minOccurs="0"/>
+                  <xs:element name="output-dir" minOccurs="0">
+                     <xs:annotation>
+                        <xs:documentation>
+                           Zero or one file containing expected results for this query.
+                        </xs:documentation>
+                     </xs:annotation>
+                     <xs:complexType>
+                        <xs:simpleContent>
+                           <xs:extension base="xs:string">
+                              <xs:attribute name="compare" type="test:comparison-enum"
use="required"/>
+                           </xs:extension>
+                        </xs:simpleContent>
+                     </xs:complexType>
+                  </xs:element>
+
+
+                  <!-- Zero or more expected errors for this query -->
+
+                  <xs:element name="expected-error" minOccurs="0" maxOccurs="unbounded">
+                     <xs:annotation>
+                        <xs:documentation>
+                           Zero or more expected errors for this query.
+                        </xs:documentation>
+                     </xs:annotation>
+
+                     <xs:complexType>
+                        <xs:simpleContent>
+                           <xs:extension base="test:ErrorCode">
+                           </xs:extension>
+                        </xs:simpleContent>
+                     </xs:complexType>
+                  </xs:element>
+
+               </xs:sequence>
+
+               <!-- This name is always equal to the name of the test case -->
+               <xs:attribute name="name" type="xs:string" use="required"/>
+
+            </xs:complexType>
+         </xs:element>
+
+         <!-- Zero or more files containing expected results for this query -->
+
+      </xs:sequence>
+
+      <!-- The filename for this query can be constructed from:              -->
+      <!--    the QueryOffsetPath                                            -->
+      <!--    the FilePath                                                   -->
+      <!--    the name                                                       -->
+      <!--    the QueryFileExtension                                         -->
+
+      <xs:attribute name="FilePath" type="test:SimplifiedRelativeFilePath" use="required"/>
+      <xs:attribute name="date" type="xs:date" use="required"/>
+      <xs:attribute name="category" type="test:category-enum"/>
+   </xs:complexType>
+
+   <!-- category-enum type                                                   -->
+   <!--    Identify which category of test this is. Currently only "slow".   -->
+
+   <xs:simpleType name="category-enum">
+      <xs:annotation>
+         <xs:documentation>
+            Identify the category of test, for limiting when it is run.
+         </xs:documentation>
+      </xs:annotation>
+
+      <xs:restriction base="xs:string">
+         <xs:enumeration value="slow"/>
+      </xs:restriction>
+   </xs:simpleType>
+
+   <!-- comparison-enum type                                                 -->
+   <!--    Identify the type of comparison used to determine whether an      -->
+   <!--    expected result and an actual result match.                       -->
+
+   <xs:simpleType name="comparison-enum">
+      <xs:annotation>
+         <xs:documentation>
+            Identify the type of comparison used to determine whether an
+            expected result and an actual result match.
+         </xs:documentation>
+      </xs:annotation>
+
+      <xs:restriction base="xs:string">
+         <xs:enumeration value="XML"/>
+         <xs:enumeration value="Text"/>
+         <xs:enumeration value="Inspect"/>
+         <xs:enumeration value="Ignore"/>
+         <xs:enumeration value="JSON"/>
+         <xs:enumeration value="CSV"/>
+         <xs:enumeration value="CSV_Header"/>
+      </xs:restriction>
+   </xs:simpleType>
+
+   <!-- description type                                                     -->
+
+   <xs:complexType name="description">
+      <xs:simpleContent>
+         <xs:extension base="xs:string">
+            <xs:attribute name="last-mod" type="xs:date"/>
+         </xs:extension>
+      </xs:simpleContent>
+   </xs:complexType>
+
+
+   <!-- ErrorCode type                                                       -->
+   <!--   * is used to mean that any error code is acceptable                -->
+
+   <xs:simpleType name="ErrorCode">
+      <xs:annotation>
+         <xs:documentation>
+            * is used to mean that any error code is acceptable
+         </xs:documentation>
+      </xs:annotation>
+
+      <xs:restriction base="xs:string">
+         <xs:pattern value="\*|([A-Z]{4}[0-9]{4})"/>
+      </xs:restriction>
+   </xs:simpleType>
+   
+</xs:schema>


Mime
View raw message