Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 80048200B6F for ; Wed, 24 Aug 2016 17:34:53 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7E6DC160AB1; Wed, 24 Aug 2016 15:34:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7AC33160A91 for ; Wed, 24 Aug 2016 17:34:51 +0200 (CEST) Received: (qmail 463 invoked by uid 500); 24 Aug 2016 15:34:50 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 454 invoked by uid 99); 24 Aug 2016 15:34:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Aug 2016 15:34:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8C1AEE024E; Wed, 24 Aug 2016 15:34:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: buyingyi@apache.org To: commits@asterixdb.apache.org Message-Id: <8be91fdb1df14d92b2780f930abc8ecc@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: asterixdb git commit: TweetParser Extension Date: Wed, 24 Aug 2016 15:34:50 +0000 (UTC) archived-at: Wed, 24 Aug 2016 15:34:53 -0000 Repository: asterixdb Updated Branches: refs/heads/master de9ade641 -> 3b2acdc01 TweetParser Extension This patch includes following changes: 1. ExtendedTweetParser to parse more than fix attributes. 2. Changed the twitter feeds message unit from Status to String. Change-Id: I7021e7b779de05b9ec999a8d5f8464fb0ab413c0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1002 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/3b2acdc0 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/3b2acdc0 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/3b2acdc0 Branch: refs/heads/master Commit: 3b2acdc01d46a877d47fd13c2e9f4e30f7233275 Parents: de9ade6 Author: Xikui Wang Authored: Wed Aug 24 21:36:10 2016 +0800 Committer: Yingyi Bu Committed: Wed Aug 24 08:33:40 2016 -0700 ---------------------------------------------------------------------- .../revised-tweet-parser.1.ddl.aql | 55 ++++ .../revised-tweet-parser.2.update.aql | 22 ++ .../src/test/resources/runtimets/testsuite.xml | 7 + .../reader/twitter/TwitterPullRecordReader.java | 16 +- .../reader/twitter/TwitterPushRecordReader.java | 35 ++- .../twitter/TwitterRecordReaderFactory.java | 8 +- .../external/library/java/JObjectAccessors.java | 35 +-- .../asterix/external/parser/TweetParser.java | 291 +++++++++++++------ .../parser/factory/TweetParserFactory.java | 9 +- .../apache/asterix/external/util/Datatypes.java | 101 ++++++- .../external/util/ExternalDataConstants.java | 4 +- .../asterix/external/util/TwitterUtil.java | 33 ++- .../asterix/om/types/AOrderedListType.java | 3 + .../asterix/om/types/AUnorderedListType.java | 3 + .../apache/asterix/om/types/TypeTagUtil.java | 6 + 15 files changed, 478 insertions(+), 150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql new file mode 100644 index 0000000..ddffc43 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : This test query will check the revised + * parser from two perspective: open Tweet + * will automatically includes all fields + * from tweet status; closed TwitterUser + * type will only includes specified fields. + * consumer.secret is missing here to create + * exception. + * Expected Res : Failure + */ + +drop dataverse feeds if exists; +create dataverse feeds; +use dataverse feeds; + +create type TwitterUser as closed{ + screen_name: string, + lang: string, + friends_count: int32, + statuses_count: int32 + }; + +create type Tweet as open +{ + id: int64, + user: TwitterUser +} + +create dataset Tweets (Tweet) +primary key id; + +create feed TwitterFeed using push_twitter( +("type-name"="Tweet"), +("format"="twitter-status"),// +("consumer.key"="************"), +("access.token"="************"), +("access.token.secret"="************")); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql new file mode 100644 index 0000000..66ede3f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +use dataverse feeds; +connect feed TwitterFeed to dataset Tweets; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index 6b63ff0..e5710bc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -150,6 +150,13 @@ + + revised-tweet-parser + One or more parameters are missing from adapter configuration + Unknown source feed + + + feed-with-external-parser http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java index e31325a..5a7b4b9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java @@ -27,14 +27,14 @@ import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; import org.apache.asterix.external.input.record.GenericRecord; import org.apache.asterix.external.util.FeedLogManager; import org.apache.hyracks.api.exceptions.HyracksDataException; - import twitter4j.Query; import twitter4j.QueryResult; import twitter4j.Status; import twitter4j.Twitter; import twitter4j.TwitterException; +import twitter4j.TwitterObjectFactory; -public class TwitterPullRecordReader implements IRecordReader { +public class TwitterPullRecordReader implements IRecordReader { private Query query; private Twitter twitter; @@ -42,18 +42,19 @@ public class TwitterPullRecordReader implements IRecordReader { private QueryResult result; private int nextTweetIndex = 0; private long lastTweetIdReceived = 0; - private GenericRecord record; + private GenericRecord record; public TwitterPullRecordReader(Twitter twitter, String keywords, int requestInterval) { this.twitter = twitter; this.requestInterval = requestInterval; this.query = new Query(keywords); this.query.setCount(100); - this.record = new GenericRecord(); + this.record = new GenericRecord<>(); } @Override public void close() throws IOException { + // do nothing } @Override @@ -62,7 +63,7 @@ public class TwitterPullRecordReader implements IRecordReader { } @Override - public IRawRecord next() throws IOException, InterruptedException { + public IRawRecord next() throws IOException, InterruptedException { if (result == null || nextTweetIndex >= result.getTweets().size()) { Thread.sleep(1000 * requestInterval); query.setSinceId(lastTweetIdReceived); @@ -79,7 +80,8 @@ public class TwitterPullRecordReader implements IRecordReader { if (lastTweetIdReceived < tweet.getId()) { lastTweetIdReceived = tweet.getId(); } - record.set(tweet); + String jsonTweet = TwitterObjectFactory.getRawJSON(tweet); // transform tweet obj to json + record.set(jsonTweet); return record; } else { return null; @@ -93,10 +95,12 @@ public class TwitterPullRecordReader implements IRecordReader { @Override public void setFeedLogManager(FeedLogManager feedLogManager) { + // do nothing } @Override public void setController(AbstractFeedDataFlowController controller) { + // do nothing } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java index f04cdb9..9ead8a9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java @@ -26,31 +26,31 @@ import org.apache.asterix.external.api.IRecordReader; import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; import org.apache.asterix.external.input.record.GenericRecord; import org.apache.asterix.external.util.FeedLogManager; - import twitter4j.FilterQuery; import twitter4j.StallWarning; import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; +import twitter4j.TwitterObjectFactory; import twitter4j.TwitterStream; -public class TwitterPushRecordReader implements IRecordReader { - private LinkedBlockingQueue inputQ; +public class TwitterPushRecordReader implements IRecordReader { + private LinkedBlockingQueue inputQ; private TwitterStream twitterStream; - private GenericRecord record; + private GenericRecord record; private boolean closed = false; public TwitterPushRecordReader(TwitterStream twitterStream, FilterQuery query) { - record = new GenericRecord(); - inputQ = new LinkedBlockingQueue(); + record = new GenericRecord<>(); + inputQ = new LinkedBlockingQueue<>(); this.twitterStream = twitterStream;//TwitterUtil.getTwitterStream(configuration); this.twitterStream.addListener(new TweetListener(inputQ)); this.twitterStream.filter(query); } public TwitterPushRecordReader(TwitterStream twitterStream) { - record = new GenericRecord(); - inputQ = new LinkedBlockingQueue(); + record = new GenericRecord<>(); + inputQ = new LinkedBlockingQueue<>(); this.twitterStream = twitterStream;// this.twitterStream.addListener(new TweetListener(inputQ)); twitterStream.sample(); @@ -72,8 +72,8 @@ public class TwitterPushRecordReader implements IRecordReader { } @Override - public IRawRecord next() throws IOException, InterruptedException { - Status tweet = inputQ.poll(); + public IRawRecord next() throws IOException, InterruptedException { + String tweet = inputQ.poll(); if (tweet == null) { return null; } @@ -93,45 +93,52 @@ public class TwitterPushRecordReader implements IRecordReader { private class TweetListener implements StatusListener { - private LinkedBlockingQueue inputQ; + private LinkedBlockingQueue inputQ; - public TweetListener(LinkedBlockingQueue inputQ) { + public TweetListener(LinkedBlockingQueue inputQ) { this.inputQ = inputQ; } @Override public void onStatus(Status tweet) { - inputQ.add(tweet); + String jsonTweet = TwitterObjectFactory.getRawJSON(tweet); + inputQ.add(jsonTweet); } @Override public void onException(Exception arg0) { - + // do nothing } @Override public void onDeletionNotice(StatusDeletionNotice arg0) { + // do nothing } @Override public void onScrubGeo(long arg0, long arg1) { + // do nothing } @Override public void onStallWarning(StallWarning arg0) { + // do nothing } @Override public void onTrackLimitationNotice(int arg0) { + // do nothing } } @Override public void setFeedLogManager(FeedLogManager feedLogManager) { + // do nothing } @Override public void setController(AbstractFeedDataFlowController controller) { + // do nothing } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java index 541737a..172b22b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java @@ -37,7 +37,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import twitter4j.FilterQuery; import twitter4j.Status; -public class TwitterRecordReaderFactory implements IRecordReaderFactory { +public class TwitterRecordReaderFactory implements IRecordReaderFactory { private static final long serialVersionUID = 1L; private static final Logger LOGGER = Logger.getLogger(TwitterRecordReaderFactory.class.getName()); @@ -114,7 +114,7 @@ public class TwitterRecordReaderFactory implements IRecordReaderFactory } @Override - public IRecordReader createRecordReader(IHyracksTaskContext ctx, int partition) + public IRecordReader createRecordReader(IHyracksTaskContext ctx, int partition) throws HyracksDataException { if (pull) { return new TwitterPullRecordReader(TwitterUtil.getTwitterService(configuration), @@ -133,8 +133,8 @@ public class TwitterRecordReaderFactory implements IRecordReaderFactory } @Override - public Class getRecordClass() { - return Status.class; + public Class getRecordClass() { + return String.class; } private boolean validateConfiguration(Map configuration) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java index 5923354..ab908bf 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectAccessors.java @@ -18,12 +18,6 @@ */ package org.apache.asterix.external.library.java; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.List; - import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer; import org.apache.asterix.dataflow.data.nontagged.serde.ACircleSerializerDeserializer; @@ -83,14 +77,20 @@ import org.apache.asterix.om.pointables.ARecordVisitablePointable; import org.apache.asterix.om.pointables.base.IVisitablePointable; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.AbstractCollectionType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.types.TypeTagUtil; import org.apache.asterix.om.util.container.IObjectPool; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.util.string.UTF8StringReader; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; + public class JObjectAccessors { public static IJObjectAccessor createFlatJObjectAccessor(ATypeTag aTypeTag) { @@ -465,15 +465,18 @@ public class JObjectAccessors { List fieldTypeTags = recordPointable.getFieldTypeTags(); List fieldNames = recordPointable.getFieldNames(); int index = 0; - boolean closedPart = true; + boolean closedPart; try { IJObject fieldObject = null; for (IVisitablePointable fieldPointable : fieldPointables) { closedPart = index < recordType.getFieldTypes().length; IVisitablePointable tt = fieldTypeTags.get(index); - IAType fieldType = closedPart ? recordType.getFieldTypes()[index] : null; ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER .deserialize(tt.getByteArray()[tt.getStartOffset()]); + IAType fieldType; + fieldType = closedPart ? + recordType.getFieldTypes()[index] : + TypeTagUtil.getBuiltinTypeByTag(typeTag); IVisitablePointable fieldName = fieldNames.get(index); typeInfo.reset(fieldType, typeTag); switch (typeTag) { @@ -486,8 +489,8 @@ public class JObjectAccessors { // value is null fieldObject = null; } else { - fieldObject = pointableVisitor.visit((AListVisitablePointable) fieldPointable, - typeInfo); + fieldObject = pointableVisitor + .visit((AListVisitablePointable) fieldPointable, typeInfo); } break; case ANY: @@ -536,15 +539,16 @@ public class JObjectAccessors { List items = pointable.getItems(); List itemTags = pointable.getItemTags(); JList list = pointable.ordered() ? new JOrderedList(listType) : new JUnorderedList(listType); - IJObject listItem = null; + IJObject listItem; int index = 0; try { - for (IVisitablePointable itemPointable : items) { IVisitablePointable itemTagPointable = itemTags.get(index); ATypeTag itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER .deserialize(itemTagPointable.getByteArray()[itemTagPointable.getStartOffset()]); - typeInfo.reset(listType.getType(), listType.getTypeTag()); + IAType fieldType; + fieldType = TypeTagUtil.getBuiltinTypeByTag(itemTypeTag); + typeInfo.reset(fieldType, itemTypeTag); switch (itemTypeTag) { case RECORD: listItem = pointableVisitor.visit((ARecordVisitablePointable) itemPointable, typeInfo); @@ -557,10 +561,7 @@ public class JObjectAccessors { throw new IllegalArgumentException( "Cannot parse list item of type " + listType.getTypeTag()); default: - IAType itemType = ((AbstractCollectionType) listType).getItemType(); - typeInfo.reset(itemType, itemType.getTypeTag()); listItem = pointableVisitor.visit((AFlatValuePointable) itemPointable, typeInfo); - } list.add(listItem); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java index 522da06..8d483dc 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java @@ -18,111 +18,236 @@ */ package org.apache.asterix.external.parser; -import java.io.DataOutput; -import java.util.HashMap; -import java.util.Map; - -import org.apache.asterix.builders.RecordBuilder; -import org.apache.asterix.external.api.IDataParser; +import org.apache.asterix.builders.AbvsBuilderFactory; +import org.apache.asterix.builders.IARecordBuilder; +import org.apache.asterix.builders.IAsterixListBuilder; +import org.apache.asterix.builders.ListBuilderFactory; +import org.apache.asterix.builders.RecordBuilderFactory; +import org.apache.asterix.builders.UnorderedListBuilder; import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.api.IRecordDataParser; -import org.apache.asterix.external.library.java.JObjectUtil; -import org.apache.asterix.external.util.Datatypes.Tweet; -import org.apache.asterix.om.base.AMutableDouble; -import org.apache.asterix.om.base.AMutableInt32; -import org.apache.asterix.om.base.AMutableRecord; -import org.apache.asterix.om.base.AMutableString; -import org.apache.asterix.om.base.IAObject; +import org.apache.asterix.om.base.AMutablePoint; +import org.apache.asterix.om.base.ANull; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.util.container.IObjectPool; +import org.apache.asterix.om.util.container.ListObjectPool; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IMutableValueStorage; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.util.string.UTF8StringWriter; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; -import twitter4j.Status; -import twitter4j.User; - -public class TweetParser implements IRecordDataParser { +import java.io.DataOutput; +import java.io.IOException; - private IAObject[] mutableTweetFields; - private IAObject[] mutableUserFields; - private AMutableRecord mutableRecord; - private AMutableRecord mutableUser; - private final Map userFieldNameMap = new HashMap<>(); - private final Map tweetFieldNameMap = new HashMap<>(); - private RecordBuilder recordBuilder = new RecordBuilder(); +public class TweetParser extends AbstractDataParser implements IRecordDataParser { + private final IObjectPool recordBuilderPool = new ListObjectPool<>( + new RecordBuilderFactory()); + private final IObjectPool listBuilderPool = new ListObjectPool<>( + new ListBuilderFactory()); + private final IObjectPool abvsBuilderPool = new ListObjectPool<>( + new AbvsBuilderFactory()); + private ARecordType recordType; + private UTF8StringWriter utf8Writer = new UTF8StringWriter(); public TweetParser(ARecordType recordType) { - initFieldNames(recordType); - mutableUserFields = new IAObject[] { new AMutableString(null), new AMutableString(null), new AMutableInt32(0), - new AMutableInt32(0), new AMutableString(null), new AMutableInt32(0) }; - mutableUser = new AMutableRecord((ARecordType) recordType.getFieldTypes()[tweetFieldNameMap.get(Tweet.USER)], - mutableUserFields); - - mutableTweetFields = new IAObject[] { new AMutableString(null), mutableUser, new AMutableDouble(0), - new AMutableDouble(0), new AMutableString(null), new AMutableString(null) }; - mutableRecord = new AMutableRecord(recordType, mutableTweetFields); + this.recordType = recordType; + aPoint = new AMutablePoint(0, 0); } - // Initialize the hashmap values for the field names and positions - private void initFieldNames(ARecordType recordType) { - String tweetFields[] = recordType.getFieldNames(); - for (int i = 0; i < tweetFields.length; i++) { - tweetFieldNameMap.put(tweetFields[i], i); - if (tweetFields[i].equals(Tweet.USER)) { - IAType fieldType = recordType.getFieldTypes()[i]; - if (fieldType.getTypeTag() == ATypeTag.RECORD) { - String userFields[] = ((ARecordType) fieldType).getFieldNames(); - for (int j = 0; j < userFields.length; j++) { - userFieldNameMap.put(userFields[j], j); - } + private void parseUnorderedList(JSONArray jArray, DataOutput output) throws IOException, JSONException { + ArrayBackedValueStorage itemBuffer = getTempBuffer(); + UnorderedListBuilder unorderedListBuilder = (UnorderedListBuilder) getUnorderedListBuilder(); + + unorderedListBuilder.reset(null); + for (int iter1 = 0; iter1 < jArray.length(); iter1++) { + itemBuffer.reset(); + if (writeField(jArray.get(iter1), null, itemBuffer.getDataOutput())) { + unorderedListBuilder.addItem(itemBuffer); + } + } + unorderedListBuilder.write(output, true); + } + + private boolean writeField(Object fieldObj, IAType fieldType, DataOutput out) throws IOException, JSONException { + boolean writeResult = true; + if (fieldType != null) { + switch (fieldType.getTypeTag()) { + case STRING: + out.write(BuiltinType.ASTRING.getTypeTag().serialize()); + utf8Writer.writeUTF8(fieldObj.toString(), out); + break; + case INT64: + aInt64.setValue((long) fieldObj); + int64Serde.serialize(aInt64, out); + break; + case INT32: + out.write(BuiltinType.AINT32.getTypeTag().serialize()); + out.writeInt((Integer) fieldObj); + break; + case DOUBLE: + out.write(BuiltinType.ADOUBLE.getTypeTag().serialize()); + out.writeDouble((Double) fieldObj); + break; + case BOOLEAN: + out.write(BuiltinType.ABOOLEAN.getTypeTag().serialize()); + out.writeBoolean((Boolean) fieldObj); + break; + case RECORD: + writeRecord((JSONObject) fieldObj, out, (ARecordType) fieldType); + break; + default: + writeResult = false; + } + } else { + if (fieldObj == JSONObject.NULL) { + nullSerde.serialize(ANull.NULL, out); + } else if (fieldObj instanceof Integer) { + out.write(BuiltinType.AINT32.getTypeTag().serialize()); + out.writeInt((Integer) fieldObj); + } else if (fieldObj instanceof Boolean) { + out.write(BuiltinType.ABOOLEAN.getTypeTag().serialize()); + out.writeBoolean((Boolean) fieldObj); + } else if (fieldObj instanceof Double) { + out.write(BuiltinType.ADOUBLE.getTypeTag().serialize()); + out.writeDouble((Double) fieldObj); + } else if (fieldObj instanceof Long) { + out.write(BuiltinType.AINT64.getTypeTag().serialize()); + out.writeLong((Long) fieldObj); + } else if (fieldObj instanceof String) { + out.write(BuiltinType.ASTRING.getTypeTag().serialize()); + utf8Writer.writeUTF8((String) fieldObj, out); + } else if (fieldObj instanceof JSONArray) { + if (((JSONArray) fieldObj).length() != 0) { + parseUnorderedList((JSONArray) fieldObj, out); + } else { + writeResult = false; + } + } else if (fieldObj instanceof JSONObject) { + if (((JSONObject) fieldObj).length() != 0) { + writeRecord((JSONObject) fieldObj, out, null); + } else { + writeResult = false; } + } + } + return writeResult; + } + private int checkAttrNameIdx(String[] nameList, String name) { + int idx = 0; + if (nameList != null) { + for (String nln : nameList) { + if (name.equals(nln)) { + return idx; + } + idx++; } } + return -1; } - @Override - public void parse(IRawRecord record, DataOutput out) throws HyracksDataException { - Status tweet = record.get(); - User user = tweet.getUser(); - // Tweet user data - ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.SCREEN_NAME)]) - .setValue(JObjectUtil.getNormalizedString(user.getScreenName())); - ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.LANGUAGE)]) - .setValue(JObjectUtil.getNormalizedString(user.getLang())); - ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FRIENDS_COUNT)]).setValue(user.getFriendsCount()); - ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.STATUS_COUNT)]).setValue(user.getStatusesCount()); - ((AMutableString) mutableUserFields[userFieldNameMap.get(Tweet.NAME)]) - .setValue(JObjectUtil.getNormalizedString(user.getName())); - ((AMutableInt32) mutableUserFields[userFieldNameMap.get(Tweet.FOLLOWERS_COUNT)]) - .setValue(user.getFollowersCount()); - - // Tweet data - ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.ID)]).setValue(String.valueOf(tweet.getId())); - - int userPos = tweetFieldNameMap.get(Tweet.USER); - for (int i = 0; i < mutableUserFields.length; i++) { - ((AMutableRecord) mutableTweetFields[userPos]).setValueAtPos(i, mutableUserFields[i]); + public void writeRecord(JSONObject obj, DataOutput out, ARecordType curRecType) throws IOException, JSONException { + IAType[] curTypes = null; + String[] curFNames = null; + int fieldN; + int attrIdx; + + ArrayBackedValueStorage fieldValueBuffer = getTempBuffer(); + ArrayBackedValueStorage fieldNameBuffer = getTempBuffer(); + IARecordBuilder recBuilder = getRecordBuilder(); + + if (curRecType != null) { + curTypes = curRecType.getFieldTypes(); + curFNames = curRecType.getFieldNames(); } - if (tweet.getGeoLocation() != null) { - ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]) - .setValue(tweet.getGeoLocation().getLatitude()); - ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]) - .setValue(tweet.getGeoLocation().getLongitude()); + + recBuilder.reset(curRecType); + recBuilder.init(); + + if (curRecType != null && !curRecType.isOpen()) { + // closed record type + fieldN = curFNames.length; + for (int iter1 = 0; iter1 < fieldN; iter1++) { + fieldValueBuffer.reset(); + DataOutput fieldOutput = fieldValueBuffer.getDataOutput(); + if (obj.isNull(curFNames[iter1])) { + if (curRecType.isClosedField(curFNames[iter1])) { + throw new HyracksDataException("Closed field " + curFNames[iter1] + " has null value."); + } else { + continue; + } + } else { + if (writeField(obj.get(curFNames[iter1]), curTypes[iter1], fieldOutput)) { + recBuilder.addField(iter1, fieldValueBuffer); + } + } + } } else { - ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LATITUDE)]).setValue(0); - ((AMutableDouble) mutableTweetFields[tweetFieldNameMap.get(Tweet.LONGITUDE)]).setValue(0); + //open record type + int closedFieldCount = 0; + IAType curFieldType = null; + for (String attrName : JSONObject.getNames(obj)) { + if (obj.isNull(attrName) || obj.length() == 0) { + continue; + } + attrIdx = checkAttrNameIdx(curFNames, attrName); + if (curRecType != null) { + curFieldType = curRecType.getFieldType(attrName); + } + fieldValueBuffer.reset(); + fieldNameBuffer.reset(); + DataOutput fieldOutput = fieldValueBuffer.getDataOutput(); + if (writeField(obj.get(attrName), curFieldType, fieldOutput)) { + if (attrIdx == -1) { + aString.setValue(attrName); + stringSerde.serialize(aString, fieldNameBuffer.getDataOutput()); + recBuilder.addField(fieldNameBuffer, fieldValueBuffer); + } else { + recBuilder.addField(attrIdx, fieldValueBuffer); + closedFieldCount++; + } + } + } + if (curRecType != null && closedFieldCount < curFNames.length) { + throw new HyracksDataException("Non-null field is null"); + } } - ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.CREATED_AT)]) - .setValue(JObjectUtil.getNormalizedString(tweet.getCreatedAt().toString())); - ((AMutableString) mutableTweetFields[tweetFieldNameMap.get(Tweet.MESSAGE)]) - .setValue(JObjectUtil.getNormalizedString(tweet.getText())); + recBuilder.write(out, true); + } + + private IARecordBuilder getRecordBuilder() { + return recordBuilderPool.allocate(ATypeTag.RECORD); + } + + private IAsterixListBuilder getUnorderedListBuilder() { + return listBuilderPool.allocate(ATypeTag.UNORDEREDLIST); + } - for (int i = 0; i < mutableTweetFields.length; i++) { - mutableRecord.setValueAtPos(i, mutableTweetFields[i]); + private ArrayBackedValueStorage getTempBuffer() { + return (ArrayBackedValueStorage) abvsBuilderPool.allocate(ATypeTag.BINARY); + } + + @Override + public void parse(IRawRecord record, DataOutput out) throws HyracksDataException { + try { + //TODO get rid of this temporary json + resetPools(); + JSONObject jsObj = new JSONObject(record.get()); + writeRecord(jsObj, out, recordType); + } catch (JSONException | IOException e) { + throw new HyracksDataException(e); } - recordBuilder.reset(mutableRecord.getType()); - recordBuilder.init(); - IDataParser.writeRecord(mutableRecord, out, recordBuilder); + } + + private void resetPools() { + listBuilderPool.reset(); + recordBuilderPool.reset(); + abvsBuilderPool.reset(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java index d6e536d..3539f6e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/factory/TweetParserFactory.java @@ -28,7 +28,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; import twitter4j.Status; -public class TweetParserFactory implements IRecordDataParserFactory { +public class TweetParserFactory implements IRecordDataParserFactory { private static final long serialVersionUID = 1L; private ARecordType recordType; @@ -44,18 +44,19 @@ public class TweetParserFactory implements IRecordDataParserFactory { } @Override - public IRecordDataParser createRecordParser(IHyracksTaskContext ctx) { + public IRecordDataParser createRecordParser(IHyracksTaskContext ctx) { TweetParser dataParser = new TweetParser(recordType); return dataParser; } @Override - public Class getRecordClass() { - return Status.class; + public Class getRecordClass() { + return String.class; } @Override public void setMetaType(ARecordType metaType) { + // do nothing } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java index e1a7911..94d7b53 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.external.util; +import org.apache.asterix.external.parser.TweetParser; + public class Datatypes { /* @@ -42,16 +44,37 @@ public class Datatypes { public static class Tweet { public static final String ID = "id"; public static final String USER = "user"; - public static final String LATITUDE = "latitude"; - public static final String LONGITUDE = "longitude"; + public static final String GEOLOCATION = "geo"; public static final String CREATED_AT = "created_at"; - public static final String MESSAGE = "message_text"; - + public static final String TEXT = "text"; public static final String COUNTRY = "country"; + public static final String PLACE = "place"; + public static final String SOURCE = "source"; + public static final String TRUNCATED = "truncated"; + public static final String IN_REPLY_TO_STATUS_ID = "in_reply_to_status_id"; + public static final String IN_REPLY_TO_USER_ID = "in_reply_to_user_id"; + public static final String IN_REPLY_TO_SCREENNAME = "in_reply_to_screen_name"; + public static final String FAVORITED = "favorited"; + public static final String RETWEETED = "retweeted"; + public static final String FAVORITE_COUNT = "favorite_count"; + public static final String RETWEET_COUNT = "retweet_count"; + public static final String CONTRIBUTORS = "contributors"; + public static final String LANGUAGE = "lang"; + public static final String FILTER_LEVEL = "filter_level"; + public static final String TIMESTAMP_MS = "timestamp_ms"; + public static final String IS_QUOTE_STATUS = "is_quote_status"; + // in API but not int JSON + public static final String SENSITIVE = "sensitive"; + public static final String RETWEETED_BY_ME = "retweeted_by_me"; + public static final String CURRENT_USER_RETWEET_ID = "current_user_retweet_id"; + // consistency consider + public static final String MESSAGE = "message_text"; + public static final String LATITUDE = "latitude"; + public static final String LONGITUDE = "longitude"; // User fields (for the sub record "user") public static final String SCREEN_NAME = "screen_name"; - public static final String LANGUAGE = "language"; + public static final String USER_PREFERRED_LANGUAGE = "user_preferred_language"; public static final String FRIENDS_COUNT = "friends_count"; public static final String STATUS_COUNT = "status_count"; public static final String NAME = "name"; @@ -59,6 +82,73 @@ public class Datatypes { } + public static final class Tweet_Place { + public static final String ID = "id"; + public static final String URL = "url"; + public static final String PLACE_TYPE = "place_type"; + public static final String NAME = "name"; + public static final String FULL_NAME = "full_name"; + public static final String COUNTRY_CODE = "country_code"; + public static final String COUNTRY = "country"; + public static final String BOUNDING_BOX = "bounding_box"; + public static final String ATTRIBUTES = "attributes"; + + private Tweet_Place() { + } + } + + public static final class Tweet_User { + public static final String ID = "id"; + public static final String NAME = "name"; + public static final String SCREEN_NAME = "screen_name"; + public static final String LOCATION = "location"; + public static final String DESCRIPTION = "description"; + public static final String CONTRIBUTORS_ENABLED = "contributors_enabled"; + public static final String PROFILE_IMAGE_URL = "profile_image_url"; + public static final String PROFILE_IMAGE_URL_HTTPS = "profile_image_url_https"; + public static final String URL = "url"; + public static final String PROTECTED = "protected"; + public static final String FOLLOWERS_COUNT = "followers_count"; + public static final String PROFILE_BACKGROUND_COLOR = "profile_background_color"; + public static final String PROFILE_TEXT_COLOR = "profile_text_color"; + public static final String PROFILE_LINK_COLOR = "profile_link_color"; + public static final String PROFILE_SIDEBAR_FILL_COLOR = "profile_sidebar_fill_color"; + public static final String PROFILE_SIDEBAR_BORDER_COLOR = "profile_sidebar_border_color"; + public static final String PROFILE_USE_BACKGROUND_IMAGE = "profile_use_background_image"; + public static final String DEFAULT_PROFILE = "default_profile"; + public static final String DEFAULT_PROFILE_IMAGE = "default_profile_image"; + public static final String FRIENDS_COUNT = "friends_count"; + public static final String CREATED_AT = "CREATED_AT"; + public static final String FAVOURITES_COUNT = "favourites_count"; + public static final String UTC_OFFSET = "utc_offset"; + public static final String TIME_ZONE = "time_zone"; + public static final String PROFILE_BACKGROUND_IMAGE_URL = "profile_background_image_url"; + public static final String PROFILE_BACKGROUND_IMAGE_URL_HTTPS = "profile_background_image_url_https"; + public static final String PROFILE_BANNER_URL = "profile_banner_url"; + public static final String LANG = "lang"; + public static final String STATUSES_COUNT = "statuses_count"; + public static final String GEO_ENABLED = "geo_enabled"; + public static final String VERIFIED = "verified"; + public static final String IS_TRANSLATOR = "is_translator"; + public static final String LISTED_COUNT = "listed_count"; + public static final String FOLLOW_REQUEST_SENT = "follow_request_sent"; + // skip Entities, attrs in API but not in JSON is as below + public static final String WITHHELD_IN_COUNTRIES = "withheld_in_countries"; + public static final String BIGGER_PROFILE_IMAGE_URL = "bigger_profile_image_url"; + public static final String MINI_PROFILE_IMAGE_URL = "mini_profile_image_url"; + public static final String ORIGINAL_PROFILE_IMAGE_URL = "original_profile_image_url"; + public static final String SHOW_ALL_INLINE_MEDIA = "show_all_inline_media"; + public static final String PROFILE_BANNER_RETINA_URL = "profile_banner_retina_url"; + public static final String PROFILE_BANNER_IPAD_URL = "profile_banner_ipad_url"; + public static final String PROFILE_BANNER_IPAD_RETINA_URL = "profile_banner_ipad_retina_url"; + public static final String PROFILE_BANNER_MOBILE_URL = "profile_banner_mobile_url"; + public static final String PROFILE_BANNER_MOBILE_RETINA_URL = "profile_banner_mobile_retina_url"; + public static final String PROFILE_BACKGROUND_TILED = "profile_background_tiled"; + + private Tweet_User() { + } + } + /* The following assumes this DDL (but ignoring the field name orders): create type ProcessedTweet if not exists as open { @@ -76,5 +166,4 @@ public class Datatypes { public static final String LOCATION = "location"; public static final String TOPICS = "topics"; } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java index e251f32..b666487 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java @@ -170,7 +170,7 @@ public class ExternalDataConstants { public static final String ALIAS_SOCKET_ADAPTER = "socket_adapter"; public static final String ALIAS_TWITTER_FIREHOSE_ADAPTER = "twitter_firehose"; public static final String ALIAS_SOCKET_CLIENT_ADAPTER = "socket_client"; - public static final String ALIAS_RSS_ADAPTER = "rss_feed"; + public static final String ALIAS_RSS_ADAPTER = "rss"; public static final String ALIAS_FILE_FEED_ADAPTER = "file_feed"; public static final String ALIAS_TWITTER_PUSH_ADAPTER = "push_twitter"; public static final String ALIAS_TWITTER_PULL_ADAPTER = "pull_twitter"; @@ -223,7 +223,7 @@ public class ExternalDataConstants { public static final String KEY_STREAM_SOURCE = "stream-source"; public static final String EXTERNAL = "external"; public static final String KEY_READER_FACTORY = "reader-factory"; - public static final String READER_RSS = "rss"; + public static final String READER_RSS = "rss_feed"; public static final String FORMAT_CSV = "csv"; public static final String ERROR_PARSE_RECORD = "Parser failed to parse record"; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java index 4fb602b..70d31c0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java @@ -46,12 +46,12 @@ public class TwitterUtil { public static final String LOCATION_EU = "Europe"; public static final String LANGUAGES = "languages"; // languages to track public static final String TRACK = "keywords"; // terms to track - public static final String FILTER_LEVEL = "filter-level"; + public static final String FILTER_LEVEL = "filter-level"; } public static class GeoConstants { - public static final double[][] US = new double[][] { { -124.848974, 24.396308 }, { -66.885444, 49.384358 } }; - public static final double[][] EU = new double[][]{{-29.7,36.7},{79.2,72.0}}; + private static final double[][] US = new double[][] { { -124.848974, 24.396308 }, { -66.885444, 49.384358 } }; + private static final double[][] EU = new double[][] { { -29.7, 36.7 }, { 79.2, 72.0 } }; public static Map boundingBoxes = initializeBoundingBoxes(); } @@ -83,24 +83,25 @@ public class TwitterUtil { if (m.matches()) { String[] locationStrings = locationValue.trim().split(";\\s*"); - locations = new double[locationStrings.length*2][2]; + locations = new double[locationStrings.length * 2][2]; - for(int i=0; i configuration) { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true); + cb.setJSONStoreEnabled(true); String oAuthConsumerKey = configuration.get(AuthenticationConstants.OAUTH_CONSUMER_KEY); String oAuthConsumerSecret = configuration.get(AuthenticationConstants.OAUTH_CONSUMER_SECRET); String oAuthAccessToken = configuration.get(AuthenticationConstants.OAUTH_ACCESS_TOKEN); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AOrderedListType.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AOrderedListType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AOrderedListType.java index e16633e..a5af127 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AOrderedListType.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AOrderedListType.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.om.types; +import org.apache.asterix.om.base.AOrderedList; import org.apache.asterix.om.base.IAObject; import org.json.JSONException; import org.json.JSONObject; @@ -26,6 +27,8 @@ public class AOrderedListType extends AbstractCollectionType { private static final long serialVersionUID = 1L; + public static final AOrderedListType FULL_OPEN_ORDEREDLIST_TYPE = new AOrderedListType(null,""); + /** * @param itemType * if null, the list is untyped http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AUnorderedListType.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AUnorderedListType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AUnorderedListType.java index 80b13b5..febc6ad 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AUnorderedListType.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/AUnorderedListType.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.om.types; +import org.apache.asterix.om.base.AUnorderedList; import org.apache.asterix.om.base.IAObject; import org.json.JSONException; import org.json.JSONObject; @@ -26,6 +27,8 @@ public class AUnorderedListType extends AbstractCollectionType { private static final long serialVersionUID = 1L; + public static final AUnorderedListType FULLY_OPEN_UNORDEREDLIST_TYPE = new AUnorderedListType(null,""); + /** * @param itemType * if null, the collection is untyped http://git-wip-us.apache.org/repos/asf/asterixdb/blob/3b2acdc0/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/TypeTagUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/TypeTagUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/TypeTagUtil.java index 2a50506..b6c5712 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/TypeTagUtil.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/TypeTagUtil.java @@ -80,6 +80,12 @@ public class TypeTagUtil { return BuiltinType.ADAYTIMEDURATION; case UUID: return BuiltinType.AUUID; + case RECORD: + return ARecordType.FULLY_OPEN_RECORD_TYPE; + case UNORDEREDLIST: + return AUnorderedListType.FULLY_OPEN_UNORDEREDLIST_TYPE; + case ORDEREDLIST: + return AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE; default: throw new AsterixException("Typetag " + typeTag + " is not a built-in type"); }