asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Xikui Wang (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Fix ASTERIXDB-1609 and OrderedList bug in TweetParser
Date Thu, 03 Nov 2016 23:18:12 GMT
Xikui Wang has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1339

Change subject: Fix ASTERIXDB-1609 and OrderedList bug in TweetParser
......................................................................

Fix ASTERIXDB-1609 and OrderedList bug in TweetParser

1. For ASTERIXDB-1609, add UNION type check in writeField, and add one
more case for orderedList.
2. For OrderedList bug, change UnorderedListBuilder to
OrderedListBuilder.

Change-Id: Ia27148cb10206b93dabf7655aed68f3004f96dfd
---
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
1 file changed, 43 insertions(+), 23 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/39/1339/1

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
index 8d483dc..fc69d27 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java
@@ -22,14 +22,16 @@
 import org.apache.asterix.builders.IARecordBuilder;
 import org.apache.asterix.builders.IAsterixListBuilder;
 import org.apache.asterix.builders.ListBuilderFactory;
+import org.apache.asterix.builders.OrderedListBuilder;
 import org.apache.asterix.builders.RecordBuilderFactory;
-import org.apache.asterix.builders.UnorderedListBuilder;
 import org.apache.asterix.external.api.IRawRecord;
 import org.apache.asterix.external.api.IRecordDataParser;
 import org.apache.asterix.om.base.AMutablePoint;
 import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.types.AOrderedListType;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.util.container.IObjectPool;
@@ -60,46 +62,69 @@
         aPoint = new AMutablePoint(0, 0);
     }
 
-    private void parseUnorderedList(JSONArray jArray, DataOutput output) throws IOException,
JSONException {
+    private void parseJSONArray(JSONArray jArray, DataOutput output, AOrderedListType orderedListType)
+            throws IOException, JSONException {
         ArrayBackedValueStorage itemBuffer = getTempBuffer();
-        UnorderedListBuilder unorderedListBuilder = (UnorderedListBuilder) getUnorderedListBuilder();
+        OrderedListBuilder orderedList = (OrderedListBuilder) getOrderedListBuilder();
 
-        unorderedListBuilder.reset(null);
+        orderedList.reset(orderedListType);
         for (int iter1 = 0; iter1 < jArray.length(); iter1++) {
             itemBuffer.reset();
-            if (writeField(jArray.get(iter1), null, itemBuffer.getDataOutput())) {
-                unorderedListBuilder.addItem(itemBuffer);
+            if (writeField(jArray.get(iter1), orderedListType.getItemType(), itemBuffer.getDataOutput()))
{
+                orderedList.addItem(itemBuffer);
             }
         }
-        unorderedListBuilder.write(output, true);
+        orderedList.write(output, true);
     }
 
     private boolean writeField(Object fieldObj, IAType fieldType, DataOutput out) throws
IOException, JSONException {
         boolean writeResult = true;
-        if (fieldType != null) {
-            switch (fieldType.getTypeTag()) {
+        IAType chkFieldType;
+        if (fieldType instanceof AUnionType) {
+            chkFieldType = ((AUnionType) fieldType).getActualType();
+        } else {
+            chkFieldType = fieldType;
+        }
+        if (chkFieldType != null) {
+            switch (chkFieldType.getTypeTag()) {
                 case STRING:
-                    out.write(BuiltinType.ASTRING.getTypeTag().serialize());
+                    out.write(fieldType.getTypeTag().serialize());
                     utf8Writer.writeUTF8(fieldObj.toString(), out);
                     break;
                 case INT64:
-                    aInt64.setValue((long) fieldObj);
+                    out.write(fieldType.getTypeTag().serialize());
+                    if (fieldObj instanceof Integer) {
+                        out.writeLong(((Integer) fieldObj).longValue());
+                    } else {
+                        out.writeLong((Long) fieldObj);
+                    }
                     int64Serde.serialize(aInt64, out);
                     break;
                 case INT32:
-                    out.write(BuiltinType.AINT32.getTypeTag().serialize());
+                    out.write(fieldType.getTypeTag().serialize());
                     out.writeInt((Integer) fieldObj);
                     break;
                 case DOUBLE:
-                    out.write(BuiltinType.ADOUBLE.getTypeTag().serialize());
+                    out.write(fieldType.getTypeTag().serialize());
                     out.writeDouble((Double) fieldObj);
                     break;
                 case BOOLEAN:
-                    out.write(BuiltinType.ABOOLEAN.getTypeTag().serialize());
+                    out.write(fieldType.getTypeTag().serialize());
                     out.writeBoolean((Boolean) fieldObj);
                     break;
                 case RECORD:
-                    writeRecord((JSONObject) fieldObj, out, (ARecordType) fieldType);
+                    if (((JSONObject) fieldObj).length() != 0) {
+                        writeRecord((JSONObject) fieldObj, out, (ARecordType) chkFieldType);
+                    } else {
+                        writeResult = false;
+                    }
+                    break;
+                case ORDEREDLIST:
+                    if (((JSONArray) fieldObj).length() != 0) {
+                        parseJSONArray((JSONArray) fieldObj, out, ((AOrderedListType) chkFieldType));
+                    } else {
+                        writeResult = false;
+                    }
                     break;
                 default:
                     writeResult = false;
@@ -124,7 +149,7 @@
                 utf8Writer.writeUTF8((String) fieldObj, out);
             } else if (fieldObj instanceof JSONArray) {
                 if (((JSONArray) fieldObj).length() != 0) {
-                    parseUnorderedList((JSONArray) fieldObj, out);
+                    parseJSONArray((JSONArray) fieldObj, out, null);
                 } else {
                     writeResult = false;
                 }
@@ -190,7 +215,6 @@
             }
         } else {
             //open record type
-            int closedFieldCount = 0;
             IAType curFieldType = null;
             for (String attrName : JSONObject.getNames(obj)) {
                 if (obj.isNull(attrName) || obj.length() == 0) {
@@ -210,12 +234,8 @@
                         recBuilder.addField(fieldNameBuffer, fieldValueBuffer);
                     } else {
                         recBuilder.addField(attrIdx, fieldValueBuffer);
-                        closedFieldCount++;
                     }
                 }
-            }
-            if (curRecType != null && closedFieldCount < curFNames.length) {
-                throw new HyracksDataException("Non-null field is null");
             }
         }
         recBuilder.write(out, true);
@@ -225,8 +245,8 @@
         return recordBuilderPool.allocate(ATypeTag.RECORD);
     }
 
-    private IAsterixListBuilder getUnorderedListBuilder() {
-        return listBuilderPool.allocate(ATypeTag.UNORDEREDLIST);
+    private IAsterixListBuilder getOrderedListBuilder() {
+        return listBuilderPool.allocate(ATypeTag.ORDEREDLIST);
     }
 
     private ArrayBackedValueStorage getTempBuffer() {

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1339
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia27148cb10206b93dabf7655aed68f3004f96dfd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xkkwww@gmail.com>

Mime
View raw message