asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject [1/2] incubator-asterixdb git commit: ASTERIXDB-1360 Fix Error Message for Unknown Format
Date Fri, 01 Apr 2016 22:01:20 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master de6e0da24 -> 442e49b9f


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-external-data/src/main/java/org/apache/asterix/external/provider/StreamRecordReaderProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/provider/StreamRecordReaderProvider.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/StreamRecordReaderProvider.java
new file mode 100644
index 0000000..ea8bc98
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/provider/StreamRecordReaderProvider.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.provider;
+
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.input.record.reader.stream.EmptyLineSeparatedRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.LineRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.QuotedLineRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.SemiStructuredRecordReader;
+import org.apache.asterix.external.input.record.reader.stream.StreamRecordReader;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class StreamRecordReaderProvider {
+    public enum Format {
+        SEMISTRUCTURED,
+        CSV,
+        LINE_SEPARATED
+    }
+
+    public static Format getReaderFormat(Map<String, String> configuration) throws
AsterixException {
+        String format = configuration.get(ExternalDataConstants.KEY_FORMAT);
+        if (format != null) {
+            switch (format) {
+                case ExternalDataConstants.FORMAT_ADM:
+                case ExternalDataConstants.FORMAT_JSON:
+                case ExternalDataConstants.FORMAT_SEMISTRUCTURED:
+                    return Format.SEMISTRUCTURED;
+                case ExternalDataConstants.FORMAT_LINE_SEPARATED:
+                    return Format.LINE_SEPARATED;
+                case ExternalDataConstants.FORMAT_DELIMITED_TEXT:
+                case ExternalDataConstants.FORMAT_CSV:
+                    return Format.CSV;
+            }
+            throw new AsterixException("Unknown format: " + format);
+        }
+        throw new AsterixException("Unspecified paramter: " + ExternalDataConstants.KEY_FORMAT);
+    }
+
+    public static StreamRecordReader createRecordReader(Format format, AsterixInputStream
inputStream,
+            Map<String, String> configuration) throws HyracksDataException {
+        switch (format) {
+            case CSV:
+                String quoteString = configuration.get(ExternalDataConstants.KEY_QUOTE);
+                boolean hasHeader = ExternalDataUtils.hasHeader(configuration);
+                if (quoteString != null) {
+                    return new QuotedLineRecordReader(hasHeader, inputStream, quoteString);
+                } else {
+                    return new LineRecordReader(hasHeader, inputStream);
+                }
+            case LINE_SEPARATED:
+                return new EmptyLineSeparatedRecordReader(inputStream);
+            case SEMISTRUCTURED:
+                return new SemiStructuredRecordReader(inputStream,
+                        configuration.get(ExternalDataConstants.KEY_RECORD_START),
+                        configuration.get(ExternalDataConstants.KEY_RECORD_END));
+            default:
+                throw new HyracksDataException("Unknown format: " + format);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index a02152b..b5ec27a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -72,8 +72,6 @@ public class ExternalDataConstants {
     public static final String KEY_HIVE_SERDE = "hive-serde";
     public static final String KEY_RSS_URL = "url";
     public static final String KEY_INTERVAL = "interval";
-    public static final String KEY_PULL = "pull";
-    public static final String KEY_PUSH = "push";
     public static final String KEY_IS_FEED = "is-feed";
     public static final String KEY_WAIT_FOR_DATA = "wait-for-data";
     public static final String KEY_FEED_NAME = "feed";
@@ -123,8 +121,10 @@ public class ExternalDataConstants {
      */
     public static final String READER_HDFS = "hdfs";
     public static final String READER_KV = "key-value";
-    public static final String READER_TWITTER_PUSH = "twitter-push";
-    public static final String READER_TWITTER_PULL = "twitter-pull";
+    public static final String READER_TWITTER_PUSH = "twitter_push";
+    public static final String READER_PUSH_TWITTER = "push_twitter";
+    public static final String READER_TWITTER_PULL = "twitter_pull";
+    public static final String READER_PULL_TWITTER = "pull_twitter";
 
     public static final String CLUSTER_LOCATIONS = "cluster-locations";
     public static final String SCHEDULER = "hdfs-scheduler";
@@ -156,7 +156,7 @@ public class ExternalDataConstants {
      */
     public static final String STREAM_HDFS = "hdfs";
     public static final String STREAM_LOCAL_FILESYSTEM = "localfs";
-    public static final String STREAM_SOCKET = "socket";
+    public static final String SOCKET = "socket";
     public static final String STREAM_SOCKET_CLIENT = "socket-client";
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 42fe8bf..76898c2 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -84,9 +84,9 @@ public class ExternalDataUtils {
         if (reader == null) {
             throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER
+ " must be specified.");
         }
-        String parser = configuration.get(ExternalDataConstants.KEY_PARSER);
+        String parser = configuration.get(ExternalDataConstants.KEY_FORMAT);
         if (parser == null) {
-            throw new AsterixException("The parameter " + ExternalDataConstants.KEY_PARSER
+ " must be specified.");
+            throw new AsterixException("The parameter " + ExternalDataConstants.KEY_FORMAT
+ " must be specified.");
         }
     }
 
@@ -200,22 +200,6 @@ public class ExternalDataUtils {
         return false;
     }
 
-    public static boolean isPull(Map<String, String> configuration) {
-        String pull = configuration.get(ExternalDataConstants.KEY_PULL);
-        if (pull == null) {
-            return false;
-        }
-        return Boolean.parseBoolean(pull);
-    }
-
-    public static boolean isPush(Map<String, String> configuration) {
-        String push = configuration.get(ExternalDataConstants.KEY_PUSH);
-        if (push == null) {
-            return false;
-        }
-        return Boolean.parseBoolean(push);
-    }
-
     public static IRecordReaderFactory<?> createExternalRecordReaderFactory(Map<String,
String> configuration)
             throws AsterixException {
         String readerFactory = configuration.get(ExternalDataConstants.KEY_READER_FACTORY);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
index d822310..354aedb 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/ClassAdToADMTest.java
@@ -73,7 +73,7 @@ public class ClassAdToADMTest extends TestCase {
                         new FileSplit[] { new FileSplit("",
                                 new FileReference(Paths.get(getClass().getResource(path).toURI()).toFile()))
},
                         null, null, 0, null, false);
-                SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in,
null, "[", "]");
+                SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in,
"[", "]");
                 Value val = new Value(objectPool);
                 while (recordReader.hasNext()) {
                     val.reset();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
index 851a7e0..fc6e725 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java
@@ -86,7 +86,7 @@ public class RecordWithMetaTest {
                     null, null, 0, null, false);
 
             // create reader record reader
-            QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream,
null,
+            QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream,
                     ExternalDataConstants.DEFAULT_QUOTE);
             // create csv with json record reader
             CSVToRecordWithMetadataAndPKConverter recordConverter = new CSVToRecordWithMetadataAndPKConverter(

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/442e49b9/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql
b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql
index 7aa1129..e6b7e21 100644
--- a/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql
+++ b/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/external_index/external_index.2.ddl.aql
@@ -36,7 +36,11 @@ create type EmployeeType as closed {
 
 create external dataset EmployeeDataset(EmployeeType)
 using hdfs
-(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.txt"),("input-format"="text-input-format"),("format"="delimited-text"),("delimiter"="|"));
+(("hdfs"="hdfs://127.0.0.1:31888"),
+("path"="/asterix/external-indexing-test.txt"),
+("input-format"="text-input-format"),
+("format"="delimited-text"),
+("delimiter"="|"));
 
 create index EmployeeAgeIdx on EmployeeDataset(age);
 


Mime
View raw message