asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xi...@apache.org
Subject [1/2] asterixdb git commit: Feed Adapter and RecordReader Clean
Date Tue, 23 May 2017 17:23:08 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master ae3b78107 -> b9f89adf9


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b9f89adf/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
index a782ad0..1584065 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
@@ -24,11 +24,13 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.asterix.external.api.IRawRecord;
 import org.apache.asterix.external.input.record.converter.CSVToRecordWithMetadataAndPKConverter;
-import org.apache.asterix.external.input.record.reader.stream.QuotedLineRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.LineRecordReader;
 import org.apache.asterix.external.input.stream.LocalFSInputStream;
 import org.apache.asterix.external.parser.ADMDataParser;
 import org.apache.asterix.external.parser.RecordWithMetadataParser;
@@ -84,8 +86,11 @@ public class RecordWithMetaTest {
             // create input stream
             LocalFSInputStream inputStream = new LocalFSInputStream(watcher);
             // create reader record reader
-            QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream,
-                    ExternalDataConstants.DEFAULT_QUOTE);
+            Map<String, String> config = new HashMap<>();
+            config.put(ExternalDataConstants.KEY_HEADER, "true");
+            config.put(ExternalDataConstants.KEY_QUOTE, ExternalDataConstants.DEFAULT_QUOTE);
+            LineRecordReader lineReader = new LineRecordReader();
+            lineReader.configure(inputStream, config);
             // create csv with json record reader
             CSVToRecordWithMetadataAndPKConverter recordConverter = new CSVToRecordWithMetadataAndPKConverter(
                     valueIndex, delimiter, metaType, recordType, pkIndicators, pkIndexes,
keyTypes);
@@ -105,7 +110,7 @@ public class RecordWithMetaTest {
             serdes[1] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType);
             printerFactories[0] = ADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(recordType);
             printerFactories[1] = ADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(metaType);
-            // create output descriptor 
+            // create output descriptor
             IPrinter[] printers = new IPrinter[printerFactories.length];
 
             for (int i = 0; i < printerFactories.length; i++) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b9f89adf/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java
new file mode 100644
index 0000000..71a5072
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/StreamRecordReaderProviderTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.parser.test;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.provider.StreamRecordReaderProvider;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class StreamRecordReaderProviderTest {
+
+    @Test
+    public void Test() throws AsterixException{
+        List<String> recordReaderFormats = Arrays.asList(
+                ExternalDataConstants.FORMAT_LINE_SEPARATED,
+                ExternalDataConstants.FORMAT_ADM,
+                ExternalDataConstants.FORMAT_JSON,
+                ExternalDataConstants.FORMAT_SEMISTRUCTURED,
+                ExternalDataConstants.FORMAT_DELIMITED_TEXT,
+                ExternalDataConstants.FORMAT_CSV);
+        Map<String, String> config = new HashMap<>();
+        for (String format : recordReaderFormats) {
+            config.clear();
+            config.put(ExternalDataConstants.KEY_FORMAT, format);
+            Class clazz = StreamRecordReaderProvider.getRecordReaderClazz(config);
+            Assert.assertTrue(clazz != null);
+        }
+        config.clear();
+        config.put(ExternalDataConstants.KEY_FORMAT, ExternalDataConstants.FORMAT_CSV);
+        config.put(ExternalDataConstants.KEY_QUOTE, "\u0000");
+        Assert.assertTrue(StreamRecordReaderProvider.getRecordReaderClazz(config) != null);
+    }
+}


Mime
View raw message