asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "abdullah alamoudi (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!
Date Sat, 04 Jun 2016 01:10:09 GMT
abdullah alamoudi has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/906

Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................

PLEASE EDIT to provide a meaningful commit message!

The following commits from your working branch will be included:

commit ebce45c396bebb79dcc1eb122af62e2b29026429
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Date:   Sat Jun 4 04:09:34 2016 +0300

    Improve External Data

Change-Id: Iae2560c73fa63e9454f731b8e893ae779a2ac7d9
---
M .gitignore
M asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
10 files changed, 45 insertions(+), 31 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/06/906/1

diff --git a/.gitignore b/.gitignore
index ced139e..636fb2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
 asterix_logs
 build
 bin
+asterixdb/asterix-opt/
 asterixdb/asterix-app/src/test/resources/externallib/
 asterixdb/asterix-app/rttest/
 asterixdb/asterix-app/mdtest/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql
index 31d6ea8..2c74ac7 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta/change-feed-with-meta-pk-in-meta.1.ddl.aql
@@ -31,11 +31,9 @@
 
 create type KVMetaType as open{
 "key":string,
-bucket:string,
 vbucket:int32,
 seq:int64,
 cas:int64,
-creationTime:int64,
 expiration:int32,
 flags:int32,
 revSeq:int64,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
index 0b19e22..ccaf86a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -6336,13 +6336,13 @@
     <test-case FilePath="load">
       <compilation-unit name="issue14_query">
         <output-dir compare="Text">issue14_query</output-dir>
-        <expected-error>The parameter format must be specified</expected-error>
+        <expected-error>The parameter format or parser-factory must be specified</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="load">
       <compilation-unit name="issue315_query">
         <output-dir compare="Text">none</output-dir>
-        <expected-error>The parameter format must be specified</expected-error>
+        <expected-error>The parameter format or parser-factory must be specified</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="load">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 3080027..3091284 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -6330,13 +6330,13 @@
     <test-case FilePath="load">
       <compilation-unit name="issue14_query">
         <output-dir compare="Text">issue14_query</output-dir>
-        <expected-error>The parameter format must be specified</expected-error>
+        <expected-error>The parameter format or parser-factory must be specified</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="load">
       <compilation-unit name="issue315_query">
         <output-dir compare="Text">none</output-dir>
-        <expected-error>The parameter format must be specified</expected-error>
+        <expected-error>The parameter format or parser-factory must be specified</expected-error>
       </compilation-unit>
     </test-case>
     <test-case FilePath="load">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index d3abd50..5274655 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -127,12 +127,14 @@
         this.recordType = outputType;
         this.metaType = metaType;
         this.configuration = configuration;
+        ExternalDataUtils.validateDataSourceParameters(configuration);
         dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(configuration);
-        dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration);
         if (dataSourceFactory.isIndexible() && (files != null)) {
             ((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
         }
         dataSourceFactory.configure(configuration);
+        ExternalDataUtils.validateDataParserParameters(configuration);
+        dataParserFactory = ParserFactoryProvider.getDataParserFactory(configuration);
         dataParserFactory.setRecordType(recordType);
         dataParserFactory.setMetaType(metaType);
         dataParserFactory.configure(configuration);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
index 724699c..4aa6dd9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPRequestToRecordWithMetadataAndPKConverter.java
@@ -47,10 +47,10 @@
     private final CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder();
     private final ByteBuffer bytes = ByteBuffer.allocateDirect(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
     private final CharBuffer chars = CharBuffer.allocate(ExternalDataConstants.DEFAULT_BUFFER_SIZE);
-    // metaTypes = {key(string), bucket(string), vbucket(int32), seq(long), cas(long),
-    // creationTime(long),expiration(int32),flags(int32),revSeqNumber(long),lockTime(int32)}
-    private static final IAType[] CB_META_TYPES = new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING,
-            BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT64, BuiltinType.AINT64,
BuiltinType.AINT32,
+    // metaTypes = {key(string), vbucket(int32), seq(long), cas(long),
+    //expiration(int32),flags(int32),revSeqNumber(long),lockTime(int32)}
+    private static final IAType[] CB_META_TYPES = new IAType[] { BuiltinType.ASTRING,
+            BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT64, BuiltinType.AINT32,
             BuiltinType.AINT32, BuiltinType.AINT64, BuiltinType.AINT32 };
     private static final int[] PK_INDICATOR = { 1 };
     private static final int[] PK_INDEXES = { 0 };
@@ -70,24 +70,21 @@
             final String key = message.key();
             final int vbucket = message.partition();
             final long seq = message.bySequenceNumber();
-            final String bucket = message.bucket();
             final long cas = message.cas();
-            final long creationTime = message.creationTime();
             final int expiration = message.expiration();
             final int flags = message.flags();
             final long revSeqNumber = message.revisionSequenceNumber();
             final int lockTime = message.lockTime();
+            int i = 0;
             recordWithMetadata.reset();
-            recordWithMetadata.setMetadata(0, key);
-            recordWithMetadata.setMetadata(1, bucket);
-            recordWithMetadata.setMetadata(2, vbucket);
-            recordWithMetadata.setMetadata(3, seq);
-            recordWithMetadata.setMetadata(4, cas);
-            recordWithMetadata.setMetadata(5, creationTime);
-            recordWithMetadata.setMetadata(6, expiration);
-            recordWithMetadata.setMetadata(7, flags);
-            recordWithMetadata.setMetadata(8, revSeqNumber);
-            recordWithMetadata.setMetadata(9, lockTime);
+            recordWithMetadata.setMetadata(i++, key);
+            recordWithMetadata.setMetadata(i++, vbucket);
+            recordWithMetadata.setMetadata(i++, seq);
+            recordWithMetadata.setMetadata(i++, cas);
+            recordWithMetadata.setMetadata(i++, expiration);
+            recordWithMetadata.setMetadata(i++, flags);
+            recordWithMetadata.setMetadata(i++, revSeqNumber);
+            recordWithMetadata.setMetadata(i++, lockTime);
             DCPRequestToRecordWithMetadataAndPKConverter.set(message.content(), decoder,
bytes, chars, value);
             ReferenceCountUtil.release(message.content());
         } else if (dcpRequest instanceof RemoveMessage) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
index e39e3bd..bf86ef3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/AdapterFactoryProvider.java
@@ -28,7 +28,6 @@
 import org.apache.asterix.external.api.IIndexingAdapterFactory;
 import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.external.util.ExternalDataCompatibilityUtils;
-import org.apache.asterix.external.util.ExternalDataUtils;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
 
@@ -37,11 +36,10 @@
  */
 public class AdapterFactoryProvider {
 
-    // Internal Adapters
+    // Adapters
     public static IAdapterFactory getAdapterFactory(String adapterName, Map<String, String>
configuration,
             ARecordType itemType, ARecordType metaType) throws AsterixException {
         ExternalDataCompatibilityUtils.prepare(adapterName, configuration);
-        ExternalDataUtils.validateParameters(configuration);
         GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
         adapterFactory.configure(configuration, itemType, metaType);
         return adapterFactory;
@@ -52,7 +50,6 @@
             Map<String, String> configuration, ARecordType itemType, List<ExternalFile>
snapshot, boolean indexingOp,
             ARecordType metaType) throws AsterixException {
         ExternalDataCompatibilityUtils.prepare(adapterName, configuration);
-        ExternalDataUtils.validateParameters(configuration);
         GenericAdapterFactory adapterFactory = new GenericAdapterFactory();
         adapterFactory.setSnapshot(snapshot, indexingOp);
         adapterFactory.configure(configuration, itemType, metaType);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
index 2e1a5a7..b37198a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/ParserFactoryProvider.java
@@ -45,8 +45,11 @@
             return ExternalDataUtils.createExternalParserFactory(ExternalDataUtils.getDataverse(configuration),
                     parserFactoryName);
         } else {
-            parserFactory = ParserFactoryProvider
-                    .getDataParserFactory(ExternalDataUtils.getRecordFormat(configuration));
+            String parserFactoryKey = ExternalDataUtils.getRecordFormat(configuration);
+            if (parserFactoryKey == null) {
+                parserFactoryKey = configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY);
+            }
+            parserFactory = ParserFactoryProvider.getDataParserFactory(parserFactoryKey);
         }
         return parserFactory;
     }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index c992723..55dee04 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -55,7 +55,7 @@
     public static final String KEY_HADOOP_BUFFER_SIZE = "io.file.buffer.size";
     public static final String KEY_SOURCE_DATATYPE = "type-name";
     public static final String KEY_DELIMITER = "delimiter";
-    public static final String KEY_PARSER_FACTORY = "tuple-parser";
+    public static final String KEY_PARSER_FACTORY = "parser-factory";
     public static final String KEY_DATA_PARSER = "parser";
     public static final String KEY_HEADER = "header";
     public static final String KEY_READER = "reader";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 1df4e42..cc66560 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -78,13 +78,29 @@
     }
 
     public static void validateParameters(Map<String, String> configuration) throws
AsterixException {
+        validateDataSourceParameters(configuration);
+        validateDataParserParameters(configuration);
+    }
+
+    public static void validateDataParserParameters(Map<String, String> configuration)
throws AsterixException {
         String reader = configuration.get(ExternalDataConstants.KEY_READER);
         if (reader == null) {
             throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER
+ " must be specified.");
         }
         String parser = configuration.get(ExternalDataConstants.KEY_FORMAT);
         if (parser == null) {
-            throw new AsterixException("The parameter " + ExternalDataConstants.KEY_FORMAT
+ " must be specified.");
+            String parserFactory = configuration.get(ExternalDataConstants.KEY_PARSER_FACTORY);
+            if (parserFactory == null) {
+                throw new AsterixException("The parameter " + ExternalDataConstants.KEY_FORMAT
+ " or "
+                        + ExternalDataConstants.KEY_PARSER_FACTORY + " must be specified.");
+            }
+        }
+    }
+
+    public static void validateDataSourceParameters(Map<String, String> configuration)
throws AsterixException {
+        String reader = configuration.get(ExternalDataConstants.KEY_READER);
+        if (reader == null) {
+            throw new AsterixException("The parameter " + ExternalDataConstants.KEY_READER
+ " must be specified.");
         }
     }
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/906
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iae2560c73fa63e9454f731b8e893ae779a2ac7d9
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message