asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [12/13] incubator-asterixdb git commit: Add Support for Upsert Operation
Date Mon, 01 Feb 2016 08:31:31 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index e05c338..f8df183 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -21,32 +21,38 @@ package org.apache.asterix.optimizer.rules;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.lang3.mutable.Mutable;
-
 import org.apache.asterix.algebra.operators.CommitOperator;
 import org.apache.asterix.algebra.operators.physical.CommitPOperator;
 import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.lang.common.util.FunctionUtil;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
 import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 
 public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        // TODO Auto-generated method stub
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
@@ -63,20 +69,43 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
         List<Mutable<ILogicalExpression>> primaryKeyExprs = null;
         int datasetId = 0;
         AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) sinkOperator.getInputs().get(0).getValue();
+        LogicalVariable upsertVar = null;
+        AssignOperator upsertFlagAssign = null;
         while (descendantOp != null) {
-            if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE) {
-                IndexInsertDeleteOperator indexInsertDeleteOperator = (IndexInsertDeleteOperator) descendantOp;
-                if (!indexInsertDeleteOperator.isBulkload()) {
-                    primaryKeyExprs = indexInsertDeleteOperator.getPrimaryKeyExpressions();
-                    datasetId = ((DatasetDataSource) indexInsertDeleteOperator.getDataSourceIndex().getDataSource())
-                            .getDataset().getDatasetId();
+            if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) {
+                IndexInsertDeleteUpsertOperator indexInsertDeleteUpsertOperator = (IndexInsertDeleteUpsertOperator) descendantOp;
+                if (!indexInsertDeleteUpsertOperator.isBulkload()
+                        && indexInsertDeleteUpsertOperator.getPrevSecondaryKeyExprs() == null) {
+                    primaryKeyExprs = indexInsertDeleteUpsertOperator.getPrimaryKeyExpressions();
+                    datasetId = ((DatasetDataSource) indexInsertDeleteUpsertOperator.getDataSourceIndex()
+                            .getDataSource()).getDataset().getDatasetId();
                     break;
                 }
-            } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE) {
-                InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) descendantOp;
-                if (!insertDeleteOperator.isBulkload()) {
-                    primaryKeyExprs = insertDeleteOperator.getPrimaryKeyExpressions();
-                    datasetId = ((DatasetDataSource) insertDeleteOperator.getDataSource()).getDataset().getDatasetId();
+            } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
+                InsertDeleteUpsertOperator insertDeleteUpsertOperator = (InsertDeleteUpsertOperator) descendantOp;
+                if (!insertDeleteUpsertOperator.isBulkload()) {
+                    primaryKeyExprs = insertDeleteUpsertOperator.getPrimaryKeyExpressions();
+                    datasetId = ((DatasetDataSource) insertDeleteUpsertOperator.getDataSource()).getDataset()
+                            .getDatasetId();
+                    if (insertDeleteUpsertOperator.getOperation() == Kind.UPSERT) {
+                        //we need to add a function that checks if previous record was found
+                        upsertVar = context.newVar();
+                        //introduce casting to enforced type
+                        AbstractFunctionCallExpression isNullFunc = new ScalarFunctionCallExpression(
+                                FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL));
+                        // The first argument is the record
+                        isNullFunc.getArguments().add(new MutableObject<ILogicalExpression>(
+                                new VariableReferenceExpression(insertDeleteUpsertOperator.getPrevRecordVar())));
+                        // AssignOperator puts in the cast var the casted record
+                        upsertFlagAssign = new AssignOperator(upsertVar,
+                                new MutableObject<ILogicalExpression>(isNullFunc));
+                        // Connect the current top of the plan to the cast operator
+                        upsertFlagAssign.getInputs()
+                                .add(new MutableObject<ILogicalOperator>(sinkOperator.getInputs().get(0).getValue()));
+                        sinkOperator.getInputs().clear();
+                        sinkOperator.getInputs().add(new MutableObject<ILogicalOperator>(upsertFlagAssign));
+                        context.computeAndSetTypeEnvironmentForOperator(upsertFlagAssign);
+                    }
                     break;
                 }
             }
@@ -102,8 +131,8 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
         JobId jobId = mp.getJobId();
 
         //create the logical and physical operator
-        CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars);
-        CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars);
+        CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars, upsertVar);
+        CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars, upsertVar);
         commitOperator.setPhysicalOperator(commitPOperator);
 
         //create ExtensionOperator and put the commitOperator in it.

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 85c7478..9a82309 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -39,9 +39,9 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOper
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -282,12 +282,12 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule
         }
 
         @Override
-        public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void tag) throws AlgebricksException {
+        public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void tag) throws AlgebricksException {
             return null;
         }
 
         @Override
-        public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void tag) throws AlgebricksException {
+        public Void visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, Void tag) throws AlgebricksException {
             return null;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 7221972..5dc9f18 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -327,6 +327,18 @@ public class CompiledStatements {
         }
     }
 
+    public static class CompiledUpsertStatement extends CompiledInsertStatement {
+
+        public CompiledUpsertStatement(String dataverseName, String datasetName, Query query, int varCounter) {
+            super(dataverseName, datasetName, query, varCounter);
+        }
+
+        @Override
+        public Kind getKind() {
+            return Kind.UPSERT;
+        }
+    }
+
     public static class CompiledConnectFeedStatement implements ICompiledDmlStatement {
         private String dataverseName;
         private String feedName;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index cd0d21a..b8fe0e1 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -80,6 +80,7 @@ import org.apache.asterix.om.base.AString;
 import org.apache.asterix.om.constants.AsterixConstantValue;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
 import org.apache.asterix.om.functions.AsterixFunctionInfo;
+import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.asterix.runtime.formats.FormatUtils;
@@ -117,7 +118,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceSc
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
@@ -238,8 +239,8 @@ class LangExpressionToPlanTranslator
                     additionalFilteringAssignExpressions);
         }
 
-        InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, payloadRef, varRefsForLoading,
-                InsertDeleteOperator.Kind.INSERT, true);
+        InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef,
+                varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true);
         insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
 
         if (additionalFilteringAssign != null) {
@@ -344,17 +345,38 @@ class LangExpressionToPlanTranslator
 
             switch (stmt.getKind()) {
                 case INSERT: {
-                    InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef,
-                            varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false);
+                    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+                            varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
                     insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     leafOperator = new SinkOperator();
                     leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));
                     break;
                 }
+                case UPSERT: {
+                    InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+                            varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
+                    upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+                    upsertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+                    // Create and add a new variable used for representing the original record
+                    ARecordType recordType = (ARecordType) targetDatasource.getItemType();
+                    upsertOp.setPrevRecordVar(context.newVar());
+                    upsertOp.setPrevRecordType(recordType);
+                    if (additionalFilteringField != null) {
+                        upsertOp.setPrevFilterVar(context.newVar());
+                        try {
+                            upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+                        } catch (IOException e) {
+                            throw new AlgebricksException("unable to get the type of filter field");
+                        }
+                    }
+                    leafOperator = new SinkOperator();
+                    leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(upsertOp));
+                    break;
+                }
                 case DELETE: {
-                    InsertDeleteOperator deleteOp = new InsertDeleteOperator(targetDatasource, varRef,
-                            varRefsForLoading, InsertDeleteOperator.Kind.DELETE, false);
+                    InsertDeleteUpsertOperator deleteOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+                            varRefsForLoading, InsertDeleteUpsertOperator.Kind.DELETE, false);
                     deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     deleteOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     leafOperator = new SinkOperator();
@@ -362,8 +384,8 @@ class LangExpressionToPlanTranslator
                     break;
                 }
                 case CONNECT_FEED: {
-                    InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef,
-                            varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false);
+                    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+                            varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
                     insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     leafOperator = new SinkOperator();
@@ -371,8 +393,8 @@ class LangExpressionToPlanTranslator
                     break;
                 }
                 case SUBSCRIBE_FEED: {
-                    ILogicalOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, varRefsForLoading,
-                            InsertDeleteOperator.Kind.INSERT, false);
+                    ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+                            varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
                     insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     leafOperator = new SinkOperator();
                     leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/dblp-small/more-dblp-small-id.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/dblp-small/more-dblp-small-id.txt b/asterix-app/data/dblp-small/more-dblp-small-id.txt
new file mode 100644
index 0000000..c5cfad6
--- /dev/null
+++ b/asterix-app/data/dblp-small/more-dblp-small-id.txt
@@ -0,0 +1,13 @@
+1:books/acm/kim95/AnnevelinkACFHK95:aaaaaaaaaaaaaaaaa:Jurgen Annevelink Rafiul Ahad Amelia Carlson Daniel H. Fishman Michael L. Heytens William Kent:2002-01-03 42-68 1995 Modern Database Systems db/books/collections/kim95.html#AnnevelinkACFHK95
+2:books/acm/kim95/Blakeley95:OQL[C++]  Extending C++ with an Object Query Capability.:José A. Blakeley:2002-01-03 69-88 Modern Database Systems db/books/collections/kim95.html#Blakeley95 1995
+3:books/acm/kim95/BreitbartGS95:SQL bbbbbbbbbbbbbbbbbb:Yuri Breitbart Hector Garcia-Molina Abraham Silberschatz:2004-03-08 573-591 Modern Database Systems books/acm/Kim95 db/books/collections/kim95.html#BreitbartGS95 1995
+4:books/acm/kim95/ChristodoulakisK95:ccccccc:Stavros Christodoulakis Leonidas Koveos:2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95
+5:books/acm/kim95/DayalHW95:ddddddddd:Umeshwar Dayal Eric N. Hanson Jennifer Widom:2002-01-03 434-456 1995 Modern Database Systems db/books/collections/kim95.html#DayalHW95
+666:books/acm/kim95/DittrichD95:hhhhhhhhhhhhBMSs SQL Should Do Better  A Critique Based on Early Experiences.:Angelika Kotz Dittrich Klaus R. Dittrich:2002-01-03 238-254 1995 Modern Database Systems db/books/collections/kim95.html#DittrichD95
+7:books/acm/kim95/Garcia-MolinaH95:Distributed Databases.:Hector Garcia-Molina Meichun Hsu:2002-01-03 477-493 1995 Modern Database Systems db/books/collections/kim95.html#Garcia-MolinaH95
+101:journals/jacm/GalilHLSW87:An O(n³log n) deterministic and an O(n³) Las Vegs isomorphism test for trivalent graphs.:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2003-11-20 513-531 1987 34 J. ACM 3 http //doi.acm.org/10.1145/28869.28870 db/journals/jacm/jacm34.html#GalilHLSW87
+102:conf/focs/GalilHLSW82:An O(n^3 log n) Deterministic and an O(n^3) Probabilistic Isomorphism Test for Trivalent Graphs:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2006-04-25 118-125 conf/focs/FOCS23 1982 FOCS db/conf/focs/focs82.html#GalilHLSW82
+103:journals/jacm/GalilT88:An O(n²(m + n log n)log n) min-cost flow algorithm.:Zvi Galil Éva Tardos:2003-11-20 374-386 1988 35 J. ACM 2 http //doi.acm.org/10.1145/42282.214090 db/journals/jacm/jacm35.html#GalilT88
+104:conf/focs/GalilT86:SQL An O(n^2 (m + n log n) log n) Min-Cost Flow Algorithm:Zvi Galil Éva Tardos:2006-04-25 1-9 conf/focs/FOCS27 1986 FOCS db/conf/focs/focs86.html#GalilT86
+105:series/synthesis/2009Weintraub:Jordan Canonical Form  Theory and Practice:Steven H. Weintraub:2009-09-06 Jordan Canonical Form  Theory and Practice http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers
+111:series/synthesis/2009Brozos:The Geometry of Walker Manifolds:Miguel Brozos-Vázquez Eduardo García-Río Peter Gilkey Stana Nikcevic Rámon Vázquez-Lorenzo:2009-09-06 The Geometry of Walker Manifolds http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/more-fbm-with-send-time.adm
----------------------------------------------------------------------
diff --git a/asterix-app/data/more-fbm-with-send-time.adm b/asterix-app/data/more-fbm-with-send-time.adm
new file mode 100644
index 0000000..faf8ff0
--- /dev/null
+++ b/asterix-app/data/more-fbm-with-send-time.adm
@@ -0,0 +1,10 @@
+{"message-id":1,"author-id":34,"in-response-to":2,"sender-location":point("47.16,77.75"),"message":" love sprint its shortcut-menu is awesome:)","send-time":datetime("2012-01-20T10:10:00")}
+{"message-id":3,"author-id":645,"in-response-to":4,"sender-location":point("48.09,81.01"),"message":" like samsung the plan is amazing","send-time":datetime("2012-03-20T10:10:00")}
+{"message-id":5,"author-id":23,"in-response-to":2,"sender-location":point("34.7,90.76"),"message":" love sprint the customization is mind-blowing","send-time":datetime("2012-05-20T10:10:00")}
+{"message-id":7,"author-id":897,"in-response-to":15,"sender-location":point("32.91,85.05"),"message":" dislike sprint the speed is horrible","send-time":datetime("2012-07-20T10:10:00")}
+{"message-id":9,"author-id":65,"in-response-to":12,"sender-location":point("34.45,96.48"),"message":" love verizon its wireless is good","send-time":datetime("2012-09-20T10:10:00")}
+{"message-id":11,"author-id":232,"in-response-to":1,"sender-location":point("38.97,77.49"),"message":" can't stand at&t its plan is terrible","send-time":datetime("2012-11-20T10:10:00")}
+{"message-id":17,"author-id":10,"in-response-to":6,"sender-location":point("42.26,77.76"),"message":" can't stand t-mobile its voicemail-service is OMG:(","send-time":datetime("2012-12-20T10:10:00")}
+{"message-id":13,"author-id":10,"in-response-to":4,"sender-location":point("42.77,78.92"),"message":" dislike iphone the voice-command is bad:(","send-time":datetime("2013-08-20T10:10:00")}
+{"message-id":19,"author-id":9,"in-response-to":12,"sender-location":point("41.33,85.28"),"message":" love at&t its 3G is good:)","send-time":datetime("2013-09-20T10:10:00")}
+{"message-id":15,"author-id":7,"in-response-to":11,"sender-location":point("44.47,67.11"),"message":" like iphone the voicemail-service is awesome","send-time":datetime("2014-01-20T10:10:00")}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/semistructured/tiny01/more-customer.adm
----------------------------------------------------------------------
diff --git a/asterix-app/data/semistructured/tiny01/more-customer.adm b/asterix-app/data/semistructured/tiny01/more-customer.adm
new file mode 100644
index 0000000..7e7770e
--- /dev/null
+++ b/asterix-app/data/semistructured/tiny01/more-customer.adm
@@ -0,0 +1,4 @@
+{  "cid": 112,  "name": "Dorie Love",  "age": 12,  "address": {  "number": 2286,  "street": "Lake St.",  "city": "Los Angeles" },  "interests": {{ "Coffee" }},  "children": [ {  "name": "Grady Lave" }, {  "name": "Daysi Lave" } ] }
+{  "cid": 94,  "name": "Edgardo Dunnegan",  "age": 19,"interests": {{  }},  "children": [ {  "name": "Lyndia Dunnegan" } ] }
+{  "cid": 619,  "name": "Jacques Gaskill",  "interests": {{ "Cigars", "Coffee", "Computers", "Wine" }},  "children": [ {  "name": "Angelyn Gaskill" }, {  "name": "Jeanett Gaskill",  "age": 40 }, {  "name": "Emelda Gaskill",  "age": 34 } ] }
+{  "cid": 92,  "name": "Kenny Laychock", "address": {  "number": 4790,  "street": "Washington St.",  "city": "Portland" },  "interests": {{ "Video Games", "Basketball" }},  "children": [  ] }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/spatial/moreSpatialData.json
----------------------------------------------------------------------
diff --git a/asterix-app/data/spatial/moreSpatialData.json b/asterix-app/data/spatial/moreSpatialData.json
new file mode 100644
index 0000000..3795511
--- /dev/null
+++ b/asterix-app/data/spatial/moreSpatialData.json
@@ -0,0 +1,9 @@
+{"id": 1, "kwds": "aaaaaaa", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.2,7.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("4.1,7.0")}
+{"id": 2, "kwds": "factory hosedan", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")}
+{"id": 3, "kwds": "bbbbbbb", "line1": line("3.0,0.0 0.0,4.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("5.5,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 4, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0 2.0,1.0 1.0,0.0"), "poly2": polygon("2.0,1.0 2.0,2.0 3.0,2.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("77.0,4.0 30.0"), "point": point("43.5083,-79.3007")}
+{"id": 5, "kwds": "cccccccccc", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("100.0,100.0 100.0,400.0 300.0,400.0 300.0,100.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("88.0,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 67, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 1.0,7.0"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("3.1,1.0 2.9,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 77, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 4.0,7.1"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("3.0,1.0 3.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("3.0,6.0 5.0,7.0"), "circle" : circle("13.0,75.0 1.0"), "point": point("43.5083,-79.3007")}
+{"id": 87, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("-3.0,-3.0 -1.0,-3.0 -3.0,-5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("76.0,87.0 50.0"), "point": point("43.5083,-79.3007")}
+{"id": 12, "kwds": "zzzz", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/tpch0.001/other-orders.tbl
----------------------------------------------------------------------
diff --git a/asterix-app/data/tpch0.001/other-orders.tbl b/asterix-app/data/tpch0.001/other-orders.tbl
new file mode 100644
index 0000000..933e96a
--- /dev/null
+++ b/asterix-app/data/tpch0.001/other-orders.tbl
@@ -0,0 +1,14 @@
+5954|22|F|167262.34|1992-12-03|1-URGENT|Clerk#000000968|0|requests along the blith|
+5955|99|P|67944.38|1995-03-27|5-LOW|Clerk#000000340|0|deas integrate. fluffily regular pa|
+5956|22|O|118036.54|1998-05-18|1-URGENT|Clerk#000000587|0|le even, express platelets.|
+5957|88|F|230949.45|1993-12-27|2-HIGH|Clerk#000000020|0| dependencies are slyly. bold accounts according to the carefully regular r|
+5958|115|O|145060.41|1995-09-16|3-MEDIUM|Clerk#000000787|0|e final requests detect alongside of the qu|
+5959|23|F|195515.26|1992-05-15|3-MEDIUM|Clerk#000000913|0|into beans use ironic, unusual foxes. carefully regular excuses boost caref|
+5984|70|F|83413.30|1994-06-18|5-LOW|Clerk#000000023|0|ickly final pains haggle along the furiously ironic pinto bea|
+5985|143|F|3942.73|1995-01-12|3-MEDIUM|Clerk#000000417|0|as nag fluffily slyly permanent accounts. regular depo|
+10986|115|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+5986|55|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+10987|64|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+5987|44|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+10988|31|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|
+5988|11|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/more-data.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/more-data.txt b/asterix-app/data/upsert/raw-data/more-data.txt
new file mode 100644
index 0000000..d77c9c9
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/more-data.txt
@@ -0,0 +1,4 @@
+2,2,Stephen,155
+4,4,Angela,333
+6,6,Tracy,123
+8,8,George,555

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/overlapping.data
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/overlapping.data b/asterix-app/data/upsert/raw-data/overlapping.data
new file mode 100644
index 0000000..a013ac1
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/overlapping.data
@@ -0,0 +1,5 @@
+2,10,Nadia,155
+4,10,Igor,333
+6,10,Silvester,123
+1,11,Cloud,777
+12,44,Smith,987

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/test-data.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/test-data.txt b/asterix-app/data/upsert/raw-data/test-data.txt
new file mode 100644
index 0000000..e1e4c47
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/test-data.txt
@@ -0,0 +1,4 @@
+1,2,Mohammed,155
+3,4,Kate,333
+5,6,William,123
+7,8,Stanly,555

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
index 3852020..5c76c40 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -41,8 +41,8 @@ public class UpdateAPIServlet extends RESTAPIServlet {
 
     @Override
     protected List<Statement.Kind> getAllowedStatements() {
-        Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST,
-                Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT,
+        Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPSERT, Kind.UPDATE,
+                Kind.DML_CMD_LIST, Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT,
                 Kind.EXTERNAL_DATASET_REFRESH, Kind.RUN };
         return Arrays.asList(statementsArray);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 3b6af6d..4adadf9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -162,6 +162,7 @@ import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStateme
 import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
 import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
 import org.apache.asterix.translator.CompiledStatements.CompiledSubscribeFeedStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
 import org.apache.asterix.translator.TypeTranslator;
 import org.apache.asterix.translator.util.ValidateUtil;
@@ -330,8 +331,9 @@ public class QueryTranslator extends AbstractLangTranslator {
                     handleLoadStatement(metadataProvider, stmt, hcc);
                     break;
                 }
-                case INSERT: {
-                    handleInsertStatement(metadataProvider, stmt, hcc);
+                case INSERT:
+                case UPSERT: {
+                    handleInsertUpsertStatement(metadataProvider, stmt, hcc);
                     break;
                 }
                 case DELETE: {
@@ -853,7 +855,8 @@ public class QueryTranslator extends AbstractLangTranslator {
                     fieldType = typeMap.get(typeSignature);
                 }
                 if (fieldType == null) {
-                    throw new AlgebricksException("Unknown type " + fieldExpr.second);
+                    throw new AlgebricksException(
+                            "Unknown type " + (fieldExpr.second == null ? fieldExpr.first : fieldExpr.second));
                 }
 
                 indexFields.add(fieldExpr.first);
@@ -1817,22 +1820,33 @@ public class QueryTranslator extends AbstractLangTranslator {
         }
     }
 
-    private void handleInsertStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+    private void handleInsertUpsertStatement(AqlMetadataProvider metadataProvider, Statement stmt,
             IHyracksClientConnection hcc) throws Exception {
 
-        InsertStatement stmtInsert = (InsertStatement) stmt;
-        String dataverseName = getActiveDataverse(stmtInsert.getDataverseName());
-        Query query = stmtInsert.getQuery();
+        InsertStatement stmtInsertUpsert = (InsertStatement) stmt;
+        String dataverseName = getActiveDataverse(stmtInsertUpsert.getDataverseName());
+        Query query = stmtInsertUpsert.getQuery();
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
-        MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtInsert.getDatasetName(),
-                query.getDataverses(), query.getDatasets());
+        MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName,
+                dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(), query.getDatasets());
 
         try {
             metadataProvider.setWriteTransaction(true);
-            CompiledInsertStatement clfrqs = new CompiledInsertStatement(dataverseName,
-                    stmtInsert.getDatasetName().getValue(), query, stmtInsert.getVarCounter());
+            CompiledInsertStatement clfrqs = null;
+            switch (stmtInsertUpsert.getKind()) {
+                case INSERT:
+                    clfrqs = new CompiledInsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(),
+                            query, stmtInsertUpsert.getVarCounter());
+                    break;
+                case UPSERT:
+                    clfrqs = new CompiledUpsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(),
+                            query, stmtInsertUpsert.getVarCounter());
+                    break;
+                default:
+                    throw new AlgebricksException("Unsupported statement type " + stmtInsertUpsert.getKind());
+            }
             JobSpecification compiled = rewriteCompileQuery(metadataProvider, query, clfrqs);
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1848,8 +1862,9 @@ public class QueryTranslator extends AbstractLangTranslator {
             }
             throw e;
         } finally {
-            MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName,
-                    dataverseName + "." + stmtInsert.getDatasetName(), query.getDataverses(), query.getDatasets());
+            MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName,
+                    dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(),
+                    query.getDatasets());
         }
     }
 
@@ -1861,8 +1876,9 @@ public class QueryTranslator extends AbstractLangTranslator {
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
-        MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtDelete.getDatasetName(),
-                stmtDelete.getDataverses(), stmtDelete.getDatasets());
+        MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName,
+                dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(),
+                stmtDelete.getDatasets());
 
         try {
             metadataProvider.setWriteTransaction(true);
@@ -1884,7 +1900,7 @@ public class QueryTranslator extends AbstractLangTranslator {
             }
             throw e;
         } finally {
-            MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName,
+            MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName,
                     dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(),
                     stmtDelete.getDatasets());
         }
@@ -2822,7 +2838,8 @@ public class QueryTranslator extends AbstractLangTranslator {
         readDataverses.add(dataverseNameFrom);
         List<String> readDatasets = new ArrayList<String>();
         readDatasets.add(datasetNameFrom);
-        MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseNameTo, datasetNameTo, readDataverses, readDatasets);
+        MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseNameTo, datasetNameTo, readDataverses,
+                readDatasets);
         try {
             prepareRunExternalRuntime(metadataProvider, hcc, pregelixStmt, dataverseNameFrom, dataverseNameTo,
                     datasetNameFrom, datasetNameTo, mdTxnCtx);
@@ -2864,7 +2881,8 @@ public class QueryTranslator extends AbstractLangTranslator {
             }
             throw e;
         } finally {
-            MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseNameTo, datasetNameTo, readDataverses, readDatasets);
+            MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseNameTo, datasetNameTo, readDataverses,
+                    readDatasets);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 199ae39..4c79965 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -56,9 +56,14 @@ public class ExecutionTest {
 
     @BeforeClass
     public static void setUp() throws Exception {
-        File outdir = new File(PATH_ACTUAL);
-        outdir.mkdirs();
-        ExecutionTestUtil.setUp();
+        try {
+            File outdir = new File(PATH_ACTUAL);
+            outdir.mkdirs();
+            ExecutionTestUtil.setUp();
+        } catch (Throwable th) {
+            th.printStackTrace();
+            throw th;
+        }
     }
 
     @AfterClass

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
index eefd40e..4c10eb2 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 
 /**
  * Manages a Mini (local VM) HDFS cluster with a configured number of datanodes.
- *
  * @author ramangrover29
  */
 public class HDFSCluster {
@@ -68,8 +69,7 @@ public class HDFSCluster {
         conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
         conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
         cleanupLocal();
-        //this constructor is deprecated in hadoop 2x
-        //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
+        setLoggingLevel(Level.WARN);
         MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
         build.nameNodePort(nameNodePort);
         build.numDataNodes(numDataNodes);
@@ -79,6 +79,11 @@ public class HDFSCluster {
         loadData(basePath);
     }
 
+    private void setLoggingLevel(Level level) {
+        Logger rootLogger = Logger.getRootLogger();
+        rootLogger.setLevel(level);
+    }
+
     private void loadData(String localDataRoot) throws IOException {
         Path destDir = new Path(HDFS_PATH);
         dfs.mkdirs(destDir);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
new file mode 100644
index 0000000..5eea00b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test filters with upsert pipeline
+ * Expected Res : Success
+ * Date         : 13th Jan 2016
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type FacebookMessageType as closed {
+        message-id: int64,
+        author-id: int64,
+        in-response-to: int64?,
+        sender-location: point?,
+        message: string,
+        send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id;
+
+create dataset FilteredFacebookMessages(FacebookMessageType)
+primary key message-id with filter on send-time;
+
+create index AutherIdx on FilteredFacebookMessages(author-id);
+create index MessageIdx on FilteredFacebookMessages(message);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
new file mode 100644
index 0000000..eecf79d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset FilteredFacebookMessages using localfs
+(("path"="asterix_nc1://data/fbm-with-send-time.adm"),("format"="adm"));
+
+load dataset FacebookMessages using localfs
+(("path"="asterix_nc1://data/more-fbm-with-send-time.adm"),("format"="adm"));
+
+upsert into dataset FilteredFacebookMessages(
+ for $x in dataset FacebookMessages
+ return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
new file mode 100644
index 0000000..d1b3925
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $m in dataset('FilteredFacebookMessages')
+where $m.send-time > datetime("2012-08-20T10:10:00")
+return $m;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
new file mode 100644
index 0000000..70960ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+  id: int64,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+
+create dataset UpsertTo(MyRecord)
+  primary key id;
+
+ create dataset UpsertFrom(MyRecord)
+ primary key id;
+
+create index btree_index on UpsertTo(kwds);
+create index rtree_index on UpsertTo(point) type rtree;
+create index inverted_index on UpsertTo(kwds) type keyword;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
new file mode 100644
index 0000000..9e6086d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset UpsertTo
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/spatial/spatialData.json"),("format"="adm"));
+
+load dataset UpsertFrom
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/spatial/moreSpatialData.json"),("format"="adm"));
+
+
+upsert into dataset UpsertTo(
+for $x in dataset UpsertFrom
+return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
new file mode 100644
index 0000000..a785237
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('UpsertTo')
+where spatial-intersect($o.point, create-polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
new file mode 100644
index 0000000..41cbd5a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type OrderTypetmp as closed {
+  o_orderkey: int64,
+  o_custkey: int64,
+  o_orderstatus: string,
+  o_totalprice: double,
+  o_orderdate: string,
+  o_orderpriority: string,
+  o_clerk: string,
+  o_shippriority: int64,
+  o_comment: string
+}
+
+create type OrderType as closed {
+nested : OrderTypetmp
+}
+
+create dataset UpsertTo(OrderTypetmp)
+  primary key o_orderkey;
+
+create dataset UpsertFrom(OrderTypetmp)
+  primary key o_orderkey;
+
+  create dataset Orders(OrderType)
+  primary key nested.o_orderkey;
+
+create index idx_Orders_Custkey on Orders(nested.o_custkey);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
new file mode 100644
index 0000000..7a46315
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset UpsertTo
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+load dataset UpsertFrom
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+insert into dataset Orders
+(
+ for $c in dataset('UpsertTo')
+ return {
+   "nested" : $c
+ }
+);
+
+upsert into dataset Orders
+(
+ for $c in dataset('UpsertFrom')
+ return {
+  "nested" : $c
+ }
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
new file mode 100644
index 0000000..6af352c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('Orders')
+where
+  $o.nested.o_custkey < 60
+order by $o.nested.o_orderkey
+return {
+  "o_orderkey": $o.nested.o_orderkey,
+  "o_custkey": $o.nested.o_custkey
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
new file mode 100644
index 0000000..c281fe1
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type AddressType as open {
+  number: int64,
+  street: string,
+  city: string
+}
+
+create type CustomerType as open {
+  cid: int64,
+  name: string,
+  age: int64?,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset MoreCustomers(CustomerType) primary key cid;
+create index age_index on Customers(age);
+

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
new file mode 100644
index 0000000..589eb3f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset Customers
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/semistructured/tiny01/customer.adm"),("format"="adm"));
+
+load dataset MoreCustomers
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/semistructured/tiny01/more-customer.adm"),("format"="adm"));
+
+upsert into dataset Customers(
+for $x in dataset MoreCustomers
+return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
new file mode 100644
index 0000000..747e1ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $c in dataset('Customers')
+where $c.age < 20
+order by $c.cid
+return $c

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
new file mode 100644
index 0000000..a1755ce
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type OrderType as closed {
+  o_orderkey: int64,
+  o_custkey: int64,
+  o_orderstatus: string,
+  o_totalprice: double,
+  o_orderdate: string,
+  o_orderpriority: string,
+  o_clerk: string,
+  o_shippriority: int64,
+  o_comment: string
+}
+
+create type OrderOpenType as open {
+  o_orderkey: int64,
+  o_orderstatus: string,
+  o_totalprice: double,
+  o_orderdate: string,
+  o_orderpriority: string,
+  o_clerk: string,
+  o_shippriority: int64,
+  o_comment: string
+}
+
+create dataset Orders(OrderType)
+  primary key o_orderkey;
+
+  create dataset OtherOrders(OrderType)
+  primary key o_orderkey;
+
+create dataset OrdersOpen(OrderOpenType)
+primary key o_orderkey;
+
+create index idx_Orders_Custkey on OrdersOpen(o_custkey:int32) enforced;
+

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
new file mode 100644
index 0000000..5951e05
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset Orders
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+load dataset OtherOrders
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+insert into dataset OrdersOpen (
+ for $x in dataset OtherOrders
+  return $x
+);
+
+upsert into dataset OrdersOpen (
+ for $x in dataset Orders
+  return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
new file mode 100644
index 0000000..3b12c24
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('OrdersOpen')
+where
+  $o.o_custkey > 40
+order by $o.o_orderkey
+return {
+  "o_orderkey": $o.o_orderkey,
+  "o_custkey": $o.o_custkey
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
new file mode 100644
index 0000000..79c6fa7
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id;
+create dataset UpsertFrom("TestType") primary key id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
new file mode 100644
index 0000000..4446525
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+  ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+  ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
new file mode 100644
index 0000000..d614ed9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+use dataverse test;
+for $x in dataset UpsertTo
+return $x;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..258230b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id;
+create index ageindex on UpsertTo('age');
+create dataset UpsertFrom("TestType") primary key id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
new file mode 100644
index 0000000..9207acb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+  ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+  ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
new file mode 100644
index 0000000..d52feb8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date         : Sep 15th 2015
+ */
+
+ // So far this one doesn't use the btree index, need another query
+use dataverse test;
+for $x in dataset UpsertTo
+where $x.age >5
+return $x;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
new file mode 100644
index 0000000..143511e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type DBLPType as closed {
+  id: int64,
+  dblpid: string,
+  title: string,
+  authors: string,
+  misc: string
+}
+
+create dataset UpsertToDBLP(DBLPType)
+  primary key id;
+
+create dataset UpsertFromDBLP(DBLPType)
+  primary key id;
+
+create index keyword_index on UpsertToDBLP(title) type keyword;
\ No newline at end of file



Mime
View raw message