Return-Path: X-Original-To: apmail-asterixdb-commits-archive@minotaur.apache.org Delivered-To: apmail-asterixdb-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7B5EC18467 for ; Mon, 1 Feb 2016 08:31:53 +0000 (UTC) Received: (qmail 79871 invoked by uid 500); 1 Feb 2016 08:31:44 -0000 Delivered-To: apmail-asterixdb-commits-archive@asterixdb.apache.org Received: (qmail 79787 invoked by uid 500); 1 Feb 2016 08:31:43 -0000 Mailing-List: contact commits-help@asterixdb.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.incubator.apache.org Delivered-To: mailing list commits@asterixdb.incubator.apache.org Received: (qmail 79771 invoked by uid 99); 1 Feb 2016 08:31:43 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2016 08:31:43 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 3D367C0742 for ; Mon, 1 Feb 2016 08:31:43 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.256 X-Spam-Level: * X-Spam-Status: No, score=1.256 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=-0.545, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id OUBdjLa17kak for ; Mon, 1 Feb 2016 08:31:31 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 4704E43F64 for ; Mon, 1 Feb 2016 08:31:22 +0000 (UTC) Received: (qmail 78758 invoked by uid 99); 1 Feb 2016 08:31:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 01 Feb 2016 08:31:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9CE11E3830; Mon, 1 Feb 2016 08:31:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: amoudi@apache.org To: commits@asterixdb.incubator.apache.org Date: Mon, 01 Feb 2016 08:31:31 -0000 Message-Id: <13651db066fb4641a4960b7609a146bb@git.apache.org> In-Reply-To: <000912130a4c4f908332adea2727855c@git.apache.org> References: <000912130a4c4f908332adea2727855c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [12/13] incubator-asterixdb git commit: Add Support for Upsert Operation http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java ---------------------------------------------------------------------- diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java index e05c338..f8df183 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java @@ -21,32 +21,38 @@ package org.apache.asterix.optimizer.rules; import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang3.mutable.Mutable; - import org.apache.asterix.algebra.operators.CommitOperator; import org.apache.asterix.algebra.operators.physical.CommitPOperator; import org.apache.asterix.common.transactions.JobId; +import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.metadata.declared.AqlMetadataProvider; import org.apache.asterix.metadata.declared.DatasetDataSource; +import org.apache.asterix.om.functions.AsterixBuiltinFunctions; +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule { @Override - public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { - // TODO Auto-generated method stub + public boolean rewritePre(Mutable opRef, IOptimizationContext context) + throws AlgebricksException { return false; } @@ -63,20 +69,43 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule { List> primaryKeyExprs = null; int datasetId = 0; AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) sinkOperator.getInputs().get(0).getValue(); + LogicalVariable upsertVar = null; + AssignOperator upsertFlagAssign = null; while (descendantOp != null) { - if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE) { - IndexInsertDeleteOperator indexInsertDeleteOperator = (IndexInsertDeleteOperator) descendantOp; - if (!indexInsertDeleteOperator.isBulkload()) { - primaryKeyExprs = indexInsertDeleteOperator.getPrimaryKeyExpressions(); - datasetId = ((DatasetDataSource) indexInsertDeleteOperator.getDataSourceIndex().getDataSource()) - .getDataset().getDatasetId(); + if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) { + IndexInsertDeleteUpsertOperator indexInsertDeleteUpsertOperator = (IndexInsertDeleteUpsertOperator) descendantOp; + if (!indexInsertDeleteUpsertOperator.isBulkload() + && indexInsertDeleteUpsertOperator.getPrevSecondaryKeyExprs() == null) { + primaryKeyExprs = indexInsertDeleteUpsertOperator.getPrimaryKeyExpressions(); + datasetId = ((DatasetDataSource) indexInsertDeleteUpsertOperator.getDataSourceIndex() + .getDataSource()).getDataset().getDatasetId(); break; } - } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE) { - InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) descendantOp; - if (!insertDeleteOperator.isBulkload()) { - primaryKeyExprs = insertDeleteOperator.getPrimaryKeyExpressions(); - datasetId = ((DatasetDataSource) insertDeleteOperator.getDataSource()).getDataset().getDatasetId(); + } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) { + InsertDeleteUpsertOperator insertDeleteUpsertOperator = (InsertDeleteUpsertOperator) descendantOp; + if (!insertDeleteUpsertOperator.isBulkload()) { + primaryKeyExprs = insertDeleteUpsertOperator.getPrimaryKeyExpressions(); + datasetId = ((DatasetDataSource) insertDeleteUpsertOperator.getDataSource()).getDataset() + .getDatasetId(); + if (insertDeleteUpsertOperator.getOperation() == Kind.UPSERT) { + //we need to add a function that checks if previous record was found + upsertVar = context.newVar(); + //introduce casting to enforced type + AbstractFunctionCallExpression isNullFunc = new ScalarFunctionCallExpression( + FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL)); + // The first argument is the record + isNullFunc.getArguments().add(new MutableObject( + new VariableReferenceExpression(insertDeleteUpsertOperator.getPrevRecordVar()))); + // AssignOperator puts in the cast var the casted record + upsertFlagAssign = new AssignOperator(upsertVar, + new MutableObject(isNullFunc)); + // Connect the current top of the plan to the cast operator + upsertFlagAssign.getInputs() + .add(new MutableObject(sinkOperator.getInputs().get(0).getValue())); + sinkOperator.getInputs().clear(); + sinkOperator.getInputs().add(new MutableObject(upsertFlagAssign)); + context.computeAndSetTypeEnvironmentForOperator(upsertFlagAssign); + } break; } } @@ -102,8 +131,8 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule { JobId jobId = mp.getJobId(); //create the logical and physical operator - CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars); - CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars); + CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars, upsertVar); + CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars, upsertVar); commitOperator.setPhysicalOperator(commitPOperator); //create ExtensionOperator and put the commitOperator in it. http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java ---------------------------------------------------------------------- diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java index 85c7478..9a82309 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java @@ -39,9 +39,9 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOper import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator; @@ -282,12 +282,12 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule } @Override - public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void tag) throws AlgebricksException { + public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void tag) throws AlgebricksException { return null; } @Override - public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void tag) throws AlgebricksException { + public Void visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, Void tag) throws AlgebricksException { return null; } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java ---------------------------------------------------------------------- diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java index 7221972..5dc9f18 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java @@ -327,6 +327,18 @@ public class CompiledStatements { } } + public static class CompiledUpsertStatement extends CompiledInsertStatement { + + public CompiledUpsertStatement(String dataverseName, String datasetName, Query query, int varCounter) { + super(dataverseName, datasetName, query, varCounter); + } + + @Override + public Kind getKind() { + return Kind.UPSERT; + } + } + public static class CompiledConnectFeedStatement implements ICompiledDmlStatement { private String dataverseName; private String feedName; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index cd0d21a..b8fe0e1 100644 --- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -80,6 +80,7 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; import org.apache.asterix.om.functions.AsterixFunctionInfo; +import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.util.AsterixAppContextInfo; import org.apache.asterix.runtime.formats.FormatUtils; @@ -117,7 +118,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceSc import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; @@ -238,8 +239,8 @@ class LangExpressionToPlanTranslator additionalFilteringAssignExpressions); } - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, payloadRef, varRefsForLoading, - InsertDeleteOperator.Kind.INSERT, true); + InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true); insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); if (additionalFilteringAssign != null) { @@ -344,17 +345,38 @@ class LangExpressionToPlanTranslator switch (stmt.getKind()) { case INSERT: { - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false); + InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); insertOp.getInputs().add(new MutableObject(assign)); leafOperator = new SinkOperator(); leafOperator.getInputs().add(new MutableObject(insertOp)); break; } + case UPSERT: { + InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false); + upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); + upsertOp.getInputs().add(new MutableObject(assign)); + // Create and add a new variable used for representing the original record + ARecordType recordType = (ARecordType) targetDatasource.getItemType(); + upsertOp.setPrevRecordVar(context.newVar()); + upsertOp.setPrevRecordType(recordType); + if (additionalFilteringField != null) { + upsertOp.setPrevFilterVar(context.newVar()); + try { + upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0))); + } catch (IOException e) { + throw new AlgebricksException("unable to get the type of filter field"); + } + } + leafOperator = new SinkOperator(); + leafOperator.getInputs().add(new MutableObject(upsertOp)); + break; + } case DELETE: { - InsertDeleteOperator deleteOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.DELETE, false); + InsertDeleteUpsertOperator deleteOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.DELETE, false); deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); deleteOp.getInputs().add(new MutableObject(assign)); leafOperator = new SinkOperator(); @@ -362,8 +384,8 @@ class LangExpressionToPlanTranslator break; } case CONNECT_FEED: { - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false); + InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); insertOp.getInputs().add(new MutableObject(assign)); leafOperator = new SinkOperator(); @@ -371,8 +393,8 @@ class LangExpressionToPlanTranslator break; } case SUBSCRIBE_FEED: { - ILogicalOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, varRefsForLoading, - InsertDeleteOperator.Kind.INSERT, false); + ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, + varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); insertOp.getInputs().add(new MutableObject(assign)); leafOperator = new SinkOperator(); leafOperator.getInputs().add(new MutableObject(insertOp)); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/dblp-small/more-dblp-small-id.txt ---------------------------------------------------------------------- diff --git a/asterix-app/data/dblp-small/more-dblp-small-id.txt b/asterix-app/data/dblp-small/more-dblp-small-id.txt new file mode 100644 index 0000000..c5cfad6 --- /dev/null +++ b/asterix-app/data/dblp-small/more-dblp-small-id.txt @@ -0,0 +1,13 @@ +1:books/acm/kim95/AnnevelinkACFHK95:aaaaaaaaaaaaaaaaa:Jurgen Annevelink Rafiul Ahad Amelia Carlson Daniel H. Fishman Michael L. Heytens William Kent:2002-01-03 42-68 1995 Modern Database Systems db/books/collections/kim95.html#AnnevelinkACFHK95 +2:books/acm/kim95/Blakeley95:OQL[C++] Extending C++ with an Object Query Capability.:José A. Blakeley:2002-01-03 69-88 Modern Database Systems db/books/collections/kim95.html#Blakeley95 1995 +3:books/acm/kim95/BreitbartGS95:SQL bbbbbbbbbbbbbbbbbb:Yuri Breitbart Hector Garcia-Molina Abraham Silberschatz:2004-03-08 573-591 Modern Database Systems books/acm/Kim95 db/books/collections/kim95.html#BreitbartGS95 1995 +4:books/acm/kim95/ChristodoulakisK95:ccccccc:Stavros Christodoulakis Leonidas Koveos:2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95 +5:books/acm/kim95/DayalHW95:ddddddddd:Umeshwar Dayal Eric N. Hanson Jennifer Widom:2002-01-03 434-456 1995 Modern Database Systems db/books/collections/kim95.html#DayalHW95 +666:books/acm/kim95/DittrichD95:hhhhhhhhhhhhBMSs SQL Should Do Better A Critique Based on Early Experiences.:Angelika Kotz Dittrich Klaus R. Dittrich:2002-01-03 238-254 1995 Modern Database Systems db/books/collections/kim95.html#DittrichD95 +7:books/acm/kim95/Garcia-MolinaH95:Distributed Databases.:Hector Garcia-Molina Meichun Hsu:2002-01-03 477-493 1995 Modern Database Systems db/books/collections/kim95.html#Garcia-MolinaH95 +101:journals/jacm/GalilHLSW87:An O(n³log n) deterministic and an O(n³) Las Vegs isomorphism test for trivalent graphs.:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2003-11-20 513-531 1987 34 J. ACM 3 http //doi.acm.org/10.1145/28869.28870 db/journals/jacm/jacm34.html#GalilHLSW87 +102:conf/focs/GalilHLSW82:An O(n^3 log n) Deterministic and an O(n^3) Probabilistic Isomorphism Test for Trivalent Graphs:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2006-04-25 118-125 conf/focs/FOCS23 1982 FOCS db/conf/focs/focs82.html#GalilHLSW82 +103:journals/jacm/GalilT88:An O(n²(m + n log n)log n) min-cost flow algorithm.:Zvi Galil Éva Tardos:2003-11-20 374-386 1988 35 J. ACM 2 http //doi.acm.org/10.1145/42282.214090 db/journals/jacm/jacm35.html#GalilT88 +104:conf/focs/GalilT86:SQL An O(n^2 (m + n log n) log n) Min-Cost Flow Algorithm:Zvi Galil Éva Tardos:2006-04-25 1-9 conf/focs/FOCS27 1986 FOCS db/conf/focs/focs86.html#GalilT86 +105:series/synthesis/2009Weintraub:Jordan Canonical Form Theory and Practice:Steven H. Weintraub:2009-09-06 Jordan Canonical Form Theory and Practice http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers +111:series/synthesis/2009Brozos:The Geometry of Walker Manifolds:Miguel Brozos-Vázquez Eduardo García-Río Peter Gilkey Stana Nikcevic Rámon Vázquez-Lorenzo:2009-09-06 The Geometry of Walker Manifolds http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/more-fbm-with-send-time.adm ---------------------------------------------------------------------- diff --git a/asterix-app/data/more-fbm-with-send-time.adm b/asterix-app/data/more-fbm-with-send-time.adm new file mode 100644 index 0000000..faf8ff0 --- /dev/null +++ b/asterix-app/data/more-fbm-with-send-time.adm @@ -0,0 +1,10 @@ +{"message-id":1,"author-id":34,"in-response-to":2,"sender-location":point("47.16,77.75"),"message":" love sprint its shortcut-menu is awesome:)","send-time":datetime("2012-01-20T10:10:00")} +{"message-id":3,"author-id":645,"in-response-to":4,"sender-location":point("48.09,81.01"),"message":" like samsung the plan is amazing","send-time":datetime("2012-03-20T10:10:00")} +{"message-id":5,"author-id":23,"in-response-to":2,"sender-location":point("34.7,90.76"),"message":" love sprint the customization is mind-blowing","send-time":datetime("2012-05-20T10:10:00")} +{"message-id":7,"author-id":897,"in-response-to":15,"sender-location":point("32.91,85.05"),"message":" dislike sprint the speed is horrible","send-time":datetime("2012-07-20T10:10:00")} +{"message-id":9,"author-id":65,"in-response-to":12,"sender-location":point("34.45,96.48"),"message":" love verizon its wireless is good","send-time":datetime("2012-09-20T10:10:00")} +{"message-id":11,"author-id":232,"in-response-to":1,"sender-location":point("38.97,77.49"),"message":" can't stand at&t its plan is terrible","send-time":datetime("2012-11-20T10:10:00")} +{"message-id":17,"author-id":10,"in-response-to":6,"sender-location":point("42.26,77.76"),"message":" can't stand t-mobile its voicemail-service is OMG:(","send-time":datetime("2012-12-20T10:10:00")} +{"message-id":13,"author-id":10,"in-response-to":4,"sender-location":point("42.77,78.92"),"message":" dislike iphone the voice-command is bad:(","send-time":datetime("2013-08-20T10:10:00")} +{"message-id":19,"author-id":9,"in-response-to":12,"sender-location":point("41.33,85.28"),"message":" love at&t its 3G is good:)","send-time":datetime("2013-09-20T10:10:00")} +{"message-id":15,"author-id":7,"in-response-to":11,"sender-location":point("44.47,67.11"),"message":" like iphone the voicemail-service is awesome","send-time":datetime("2014-01-20T10:10:00")} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/semistructured/tiny01/more-customer.adm ---------------------------------------------------------------------- diff --git a/asterix-app/data/semistructured/tiny01/more-customer.adm b/asterix-app/data/semistructured/tiny01/more-customer.adm new file mode 100644 index 0000000..7e7770e --- /dev/null +++ b/asterix-app/data/semistructured/tiny01/more-customer.adm @@ -0,0 +1,4 @@ +{ "cid": 112, "name": "Dorie Love", "age": 12, "address": { "number": 2286, "street": "Lake St.", "city": "Los Angeles" }, "interests": {{ "Coffee" }}, "children": [ { "name": "Grady Lave" }, { "name": "Daysi Lave" } ] } +{ "cid": 94, "name": "Edgardo Dunnegan", "age": 19,"interests": {{ }}, "children": [ { "name": "Lyndia Dunnegan" } ] } +{ "cid": 619, "name": "Jacques Gaskill", "interests": {{ "Cigars", "Coffee", "Computers", "Wine" }}, "children": [ { "name": "Angelyn Gaskill" }, { "name": "Jeanett Gaskill", "age": 40 }, { "name": "Emelda Gaskill", "age": 34 } ] } +{ "cid": 92, "name": "Kenny Laychock", "address": { "number": 4790, "street": "Washington St.", "city": "Portland" }, "interests": {{ "Video Games", "Basketball" }}, "children": [ ] } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/spatial/moreSpatialData.json ---------------------------------------------------------------------- diff --git a/asterix-app/data/spatial/moreSpatialData.json b/asterix-app/data/spatial/moreSpatialData.json new file mode 100644 index 0000000..3795511 --- /dev/null +++ b/asterix-app/data/spatial/moreSpatialData.json @@ -0,0 +1,9 @@ +{"id": 1, "kwds": "aaaaaaa", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.2,7.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("4.1,7.0")} +{"id": 2, "kwds": "factory hosedan", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")} +{"id": 3, "kwds": "bbbbbbb", "line1": line("3.0,0.0 0.0,4.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("5.5,1.0 10.0"), "point": point("43.5083,-79.3007")} +{"id": 4, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0 2.0,1.0 1.0,0.0"), "poly2": polygon("2.0,1.0 2.0,2.0 3.0,2.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("77.0,4.0 30.0"), "point": point("43.5083,-79.3007")} +{"id": 5, "kwds": "cccccccccc", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("100.0,100.0 100.0,400.0 300.0,400.0 300.0,100.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("88.0,1.0 10.0"), "point": point("43.5083,-79.3007")} +{"id": 67, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 1.0,7.0"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("3.1,1.0 2.9,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("43.5083,-79.3007")} +{"id": 77, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 4.0,7.1"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("3.0,1.0 3.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("3.0,6.0 5.0,7.0"), "circle" : circle("13.0,75.0 1.0"), "point": point("43.5083,-79.3007")} +{"id": 87, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("-3.0,-3.0 -1.0,-3.0 -3.0,-5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("76.0,87.0 50.0"), "point": point("43.5083,-79.3007")} +{"id": 12, "kwds": "zzzz", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/tpch0.001/other-orders.tbl ---------------------------------------------------------------------- diff --git a/asterix-app/data/tpch0.001/other-orders.tbl b/asterix-app/data/tpch0.001/other-orders.tbl new file mode 100644 index 0000000..933e96a --- /dev/null +++ b/asterix-app/data/tpch0.001/other-orders.tbl @@ -0,0 +1,14 @@ +5954|22|F|167262.34|1992-12-03|1-URGENT|Clerk#000000968|0|requests along the blith| +5955|99|P|67944.38|1995-03-27|5-LOW|Clerk#000000340|0|deas integrate. fluffily regular pa| +5956|22|O|118036.54|1998-05-18|1-URGENT|Clerk#000000587|0|le even, express platelets.| +5957|88|F|230949.45|1993-12-27|2-HIGH|Clerk#000000020|0| dependencies are slyly. bold accounts according to the carefully regular r| +5958|115|O|145060.41|1995-09-16|3-MEDIUM|Clerk#000000787|0|e final requests detect alongside of the qu| +5959|23|F|195515.26|1992-05-15|3-MEDIUM|Clerk#000000913|0|into beans use ironic, unusual foxes. carefully regular excuses boost caref| +5984|70|F|83413.30|1994-06-18|5-LOW|Clerk#000000023|0|ickly final pains haggle along the furiously ironic pinto bea| +5985|143|F|3942.73|1995-01-12|3-MEDIUM|Clerk#000000417|0|as nag fluffily slyly permanent accounts. regular depo| +10986|115|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are | +5986|55|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are | +10987|64|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade| +5987|44|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade| +10988|31|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi| +5988|11|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi| http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/more-data.txt ---------------------------------------------------------------------- diff --git a/asterix-app/data/upsert/raw-data/more-data.txt b/asterix-app/data/upsert/raw-data/more-data.txt new file mode 100644 index 0000000..d77c9c9 --- /dev/null +++ b/asterix-app/data/upsert/raw-data/more-data.txt @@ -0,0 +1,4 @@ +2,2,Stephen,155 +4,4,Angela,333 +6,6,Tracy,123 +8,8,George,555 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/overlapping.data ---------------------------------------------------------------------- diff --git a/asterix-app/data/upsert/raw-data/overlapping.data b/asterix-app/data/upsert/raw-data/overlapping.data new file mode 100644 index 0000000..a013ac1 --- /dev/null +++ b/asterix-app/data/upsert/raw-data/overlapping.data @@ -0,0 +1,5 @@ +2,10,Nadia,155 +4,10,Igor,333 +6,10,Silvester,123 +1,11,Cloud,777 +12,44,Smith,987 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/test-data.txt ---------------------------------------------------------------------- diff --git a/asterix-app/data/upsert/raw-data/test-data.txt b/asterix-app/data/upsert/raw-data/test-data.txt new file mode 100644 index 0000000..e1e4c47 --- /dev/null +++ b/asterix-app/data/upsert/raw-data/test-data.txt @@ -0,0 +1,4 @@ +1,2,Mohammed,155 +3,4,Kate,333 +5,6,William,123 +7,8,Stanly,555 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java ---------------------------------------------------------------------- diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java index 3852020..5c76c40 100644 --- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java +++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java @@ -41,8 +41,8 @@ public class UpdateAPIServlet extends RESTAPIServlet { @Override protected List getAllowedStatements() { - Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST, - Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT, + Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPSERT, Kind.UPDATE, + Kind.DML_CMD_LIST, Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT, Kind.EXTERNAL_DATASET_REFRESH, Kind.RUN }; return Arrays.asList(statementsArray); } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 3b6af6d..4adadf9 100644 --- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -162,6 +162,7 @@ import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStateme import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement; import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement; import org.apache.asterix.translator.CompiledStatements.CompiledSubscribeFeedStatement; +import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; import org.apache.asterix.translator.TypeTranslator; import org.apache.asterix.translator.util.ValidateUtil; @@ -330,8 +331,9 @@ public class QueryTranslator extends AbstractLangTranslator { handleLoadStatement(metadataProvider, stmt, hcc); break; } - case INSERT: { - handleInsertStatement(metadataProvider, stmt, hcc); + case INSERT: + case UPSERT: { + handleInsertUpsertStatement(metadataProvider, stmt, hcc); break; } case DELETE: { @@ -853,7 +855,8 @@ public class QueryTranslator extends AbstractLangTranslator { fieldType = typeMap.get(typeSignature); } if (fieldType == null) { - throw new AlgebricksException("Unknown type " + fieldExpr.second); + throw new AlgebricksException( + "Unknown type " + (fieldExpr.second == null ? fieldExpr.first : fieldExpr.second)); } indexFields.add(fieldExpr.first); @@ -1817,22 +1820,33 @@ public class QueryTranslator extends AbstractLangTranslator { } } - private void handleInsertStatement(AqlMetadataProvider metadataProvider, Statement stmt, + private void handleInsertUpsertStatement(AqlMetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc) throws Exception { - InsertStatement stmtInsert = (InsertStatement) stmt; - String dataverseName = getActiveDataverse(stmtInsert.getDataverseName()); - Query query = stmtInsert.getQuery(); + InsertStatement stmtInsertUpsert = (InsertStatement) stmt; + String dataverseName = getActiveDataverse(stmtInsertUpsert.getDataverseName()); + Query query = stmtInsertUpsert.getQuery(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); - MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtInsert.getDatasetName(), - query.getDataverses(), query.getDatasets()); + MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName, + dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(), query.getDatasets()); try { metadataProvider.setWriteTransaction(true); - CompiledInsertStatement clfrqs = new CompiledInsertStatement(dataverseName, - stmtInsert.getDatasetName().getValue(), query, stmtInsert.getVarCounter()); + CompiledInsertStatement clfrqs = null; + switch (stmtInsertUpsert.getKind()) { + case INSERT: + clfrqs = new CompiledInsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(), + query, stmtInsertUpsert.getVarCounter()); + break; + case UPSERT: + clfrqs = new CompiledUpsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(), + query, stmtInsertUpsert.getVarCounter()); + break; + default: + throw new AlgebricksException("Unsupported statement type " + stmtInsertUpsert.getKind()); + } JobSpecification compiled = rewriteCompileQuery(metadataProvider, query, clfrqs); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); @@ -1848,8 +1862,9 @@ public class QueryTranslator extends AbstractLangTranslator { } throw e; } finally { - MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName, - dataverseName + "." + stmtInsert.getDatasetName(), query.getDataverses(), query.getDatasets()); + MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName, + dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(), + query.getDatasets()); } } @@ -1861,8 +1876,9 @@ public class QueryTranslator extends AbstractLangTranslator { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); - MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtDelete.getDatasetName(), - stmtDelete.getDataverses(), stmtDelete.getDatasets()); + MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName, + dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(), + stmtDelete.getDatasets()); try { metadataProvider.setWriteTransaction(true); @@ -1884,7 +1900,7 @@ public class QueryTranslator extends AbstractLangTranslator { } throw e; } finally { - MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName, + MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName, dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(), stmtDelete.getDatasets()); } @@ -2822,7 +2838,8 @@ public class QueryTranslator extends AbstractLangTranslator { readDataverses.add(dataverseNameFrom); List readDatasets = new ArrayList(); readDatasets.add(datasetNameFrom); - MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseNameTo, datasetNameTo, readDataverses, readDatasets); + MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseNameTo, datasetNameTo, readDataverses, + readDatasets); try { prepareRunExternalRuntime(metadataProvider, hcc, pregelixStmt, dataverseNameFrom, dataverseNameTo, datasetNameFrom, datasetNameTo, mdTxnCtx); @@ -2864,7 +2881,8 @@ public class QueryTranslator extends AbstractLangTranslator { } throw e; } finally { - MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseNameTo, datasetNameTo, readDataverses, readDatasets); + MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseNameTo, datasetNameTo, readDataverses, + readDatasets); } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java index 199ae39..4c79965 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java @@ -56,9 +56,14 @@ public class ExecutionTest { @BeforeClass public static void setUp() throws Exception { - File outdir = new File(PATH_ACTUAL); - outdir.mkdirs(); - ExecutionTestUtil.setUp(); + try { + File outdir = new File(PATH_ACTUAL); + outdir.mkdirs(); + ExecutionTestUtil.setUp(); + } catch (Throwable th) { + th.printStackTrace(); + throw th; + } } @AfterClass http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java index eefd40e..4c10eb2 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java @@ -27,10 +27,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.mapred.JobConf; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; /** * Manages a Mini (local VM) HDFS cluster with a configured number of datanodes. - * * @author ramangrover29 */ public class HDFSCluster { @@ -68,8 +69,7 @@ public class HDFSCluster { conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml")); conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); cleanupLocal(); - //this constructor is deprecated in hadoop 2x - //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null); + setLoggingLevel(Level.WARN); MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); build.nameNodePort(nameNodePort); build.numDataNodes(numDataNodes); @@ -79,6 +79,11 @@ public class HDFSCluster { loadData(basePath); } + private void setLoggingLevel(Level level) { + Logger rootLogger = Logger.getRootLogger(); + rootLogger.setLevel(level); + } + private void loadData(String localDataRoot) throws IOException { Path destDir = new Path(HDFS_PATH); dfs.mkdirs(destDir); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql new file mode 100644 index 0000000..5eea00b --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Test filters with upsert pipeline + * Expected Res : Success + * Date : 13th Jan 2016 + */ + +drop dataverse test if exists; +create dataverse test; + +use dataverse test; + +create type FacebookMessageType as closed { + message-id: int64, + author-id: int64, + in-response-to: int64?, + sender-location: point?, + message: string, + send-time: datetime +} + +create dataset FacebookMessages(FacebookMessageType) +primary key message-id; + +create dataset FilteredFacebookMessages(FacebookMessageType) +primary key message-id with filter on send-time; + +create index AutherIdx on FilteredFacebookMessages(author-id); +create index MessageIdx on FilteredFacebookMessages(message); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql new file mode 100644 index 0000000..eecf79d --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +load dataset FilteredFacebookMessages using localfs +(("path"="asterix_nc1://data/fbm-with-send-time.adm"),("format"="adm")); + +load dataset FacebookMessages using localfs +(("path"="asterix_nc1://data/more-fbm-with-send-time.adm"),("format"="adm")); + +upsert into dataset FilteredFacebookMessages( + for $x in dataset FacebookMessages + return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql new file mode 100644 index 0000000..d1b3925 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +for $m in dataset('FilteredFacebookMessages') +where $m.send-time > datetime("2012-08-20T10:10:00") +return $m; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql new file mode 100644 index 0000000..70960ed --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type MyRecord as closed { + id: int64, + point: point, + kwds: string, + line1: line, + line2: line, + poly1: polygon, + poly2: polygon, + rec: rectangle, + circle: circle +} + + +create dataset UpsertTo(MyRecord) + primary key id; + + create dataset UpsertFrom(MyRecord) + primary key id; + +create index btree_index on UpsertTo(kwds); +create index rtree_index on UpsertTo(point) type rtree; +create index inverted_index on UpsertTo(kwds) type keyword; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql new file mode 100644 index 0000000..9e6086d --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +load dataset UpsertTo +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/spatial/spatialData.json"),("format"="adm")); + +load dataset UpsertFrom +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/spatial/moreSpatialData.json"),("format"="adm")); + + +upsert into dataset UpsertTo( +for $x in dataset UpsertFrom +return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql new file mode 100644 index 0000000..a785237 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +for $o in dataset('UpsertTo') +where spatial-intersect($o.point, create-polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0])) +order by $o.id +return {"id":$o.id} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql new file mode 100644 index 0000000..41cbd5a --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type OrderTypetmp as closed { + o_orderkey: int64, + o_custkey: int64, + o_orderstatus: string, + o_totalprice: double, + o_orderdate: string, + o_orderpriority: string, + o_clerk: string, + o_shippriority: int64, + o_comment: string +} + +create type OrderType as closed { +nested : OrderTypetmp +} + +create dataset UpsertTo(OrderTypetmp) + primary key o_orderkey; + +create dataset UpsertFrom(OrderTypetmp) + primary key o_orderkey; + + create dataset Orders(OrderType) + primary key nested.o_orderkey; + +create index idx_Orders_Custkey on Orders(nested.o_custkey); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql new file mode 100644 index 0000000..7a46315 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +load dataset UpsertTo +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted; + +load dataset UpsertFrom +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted; + +insert into dataset Orders +( + for $c in dataset('UpsertTo') + return { + "nested" : $c + } +); + +upsert into dataset Orders +( + for $c in dataset('UpsertFrom') + return { + "nested" : $c + } +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql new file mode 100644 index 0000000..6af352c --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +for $o in dataset('Orders') +where + $o.nested.o_custkey < 60 +order by $o.nested.o_orderkey +return { + "o_orderkey": $o.nested.o_orderkey, + "o_custkey": $o.nested.o_custkey +} http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql new file mode 100644 index 0000000..c281fe1 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type AddressType as open { + number: int64, + street: string, + city: string +} + +create type CustomerType as open { + cid: int64, + name: string, + age: int64?, + address: AddressType?, + interests: {{string}}, + children: [ { name: string, age: int64? } ] +} + +create dataset Customers(CustomerType) primary key cid; +create dataset MoreCustomers(CustomerType) primary key cid; +create index age_index on Customers(age); + http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql new file mode 100644 index 0000000..589eb3f --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +load dataset Customers +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/semistructured/tiny01/customer.adm"),("format"="adm")); + +load dataset MoreCustomers +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/semistructured/tiny01/more-customer.adm"),("format"="adm")); + +upsert into dataset Customers( +for $x in dataset MoreCustomers +return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql new file mode 100644 index 0000000..747e1ed --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +for $c in dataset('Customers') +where $c.age < 20 +order by $c.cid +return $c http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql new file mode 100644 index 0000000..a1755ce --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type OrderType as closed { + o_orderkey: int64, + o_custkey: int64, + o_orderstatus: string, + o_totalprice: double, + o_orderdate: string, + o_orderpriority: string, + o_clerk: string, + o_shippriority: int64, + o_comment: string +} + +create type OrderOpenType as open { + o_orderkey: int64, + o_orderstatus: string, + o_totalprice: double, + o_orderdate: string, + o_orderpriority: string, + o_clerk: string, + o_shippriority: int64, + o_comment: string +} + +create dataset Orders(OrderType) + primary key o_orderkey; + + create dataset OtherOrders(OrderType) + primary key o_orderkey; + +create dataset OrdersOpen(OrderOpenType) +primary key o_orderkey; + +create index idx_Orders_Custkey on OrdersOpen(o_custkey:int32) enforced; + http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql new file mode 100644 index 0000000..5951e05 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +load dataset Orders +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted; + +load dataset OtherOrders +using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter" +(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted; + +insert into dataset OrdersOpen ( + for $x in dataset OtherOrders + return $x +); + +upsert into dataset OrdersOpen ( + for $x in dataset Orders + return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql new file mode 100644 index 0000000..3b12c24 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse test; + +for $o in dataset('OrdersOpen') +where + $o.o_custkey > 40 +order by $o.o_orderkey +return { + "o_orderkey": $o.o_orderkey, + "o_custkey": $o.o_custkey +} http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql new file mode 100644 index 0000000..79c6fa7 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which doesn't have any secondary indexes + * Expected Res : Success + * Date : Sep 15th 2015 + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type TestType as closed{ +id:int32, +age:int32, +name:string, +salary:double +}; + +create dataset UpsertTo("TestType") primary key id; +create dataset UpsertFrom("TestType") primary key id; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql new file mode 100644 index 0000000..4446525 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which doesn't have any secondary indexes + * Expected Res : Success + * Date : Sep 15th 2015 + */ + +use dataverse test; +// load first dataset +load dataset UpsertTo using +localfs(("format"="delimited-text"), + ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"), + ("delimiter"=",")); +// load second dataset +load dataset UpsertFrom using +localfs(("format"="delimited-text"), + ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"), + ("delimiter"=",")); + +// upsert UpsertFrom into UpsertTo +use dataverse test; +upsert into dataset UpsertTo( + for $x in dataset UpsertFrom + return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql new file mode 100644 index 0000000..d614ed9 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which doesn't have any secondary indexes + * Expected Res : Success + * Date : Sep 15th 2015 + */ + +use dataverse test; +for $x in dataset UpsertTo +return $x; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql new file mode 100644 index 0000000..258230b --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which has a b-tree secondary index + * Expected Res : Success + * Date : Sep 15th 2015 + */ + +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type TestType as closed{ +id:int32, +age:int32, +name:string, +salary:double +}; + +create dataset UpsertTo("TestType") primary key id; +create index ageindex on UpsertTo('age'); +create dataset UpsertFrom("TestType") primary key id; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql new file mode 100644 index 0000000..9207acb --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which has a b-tree secondary index + * Expected Res : Success + * Date : Sep 15th 2015 + */ + +use dataverse test; +// load first dataset +load dataset UpsertTo using +localfs(("format"="delimited-text"), + ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"), + ("delimiter"=",")); +// load second dataset +load dataset UpsertFrom using +localfs(("format"="delimited-text"), + ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"), + ("delimiter"=",")); + +// upsert UpsertFrom into UpsertTo +use dataverse test; +upsert into dataset UpsertTo( + for $x in dataset UpsertFrom + return $x +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql new file mode 100644 index 0000000..d52feb8 --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Upsert into a dataset which has a b-tree secondary index + * Expected Res : Success + * Date : Sep 15th 2015 + */ + + // So far this one doesn't use the btree index, need another query +use dataverse test; +for $x in dataset UpsertTo +where $x.age >5 +return $x; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql new file mode 100644 index 0000000..143511e --- /dev/null +++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse test if exists; +create dataverse test; +use dataverse test; + +create type DBLPType as closed { + id: int64, + dblpid: string, + title: string, + authors: string, + misc: string +} + +create dataset UpsertToDBLP(DBLPType) + primary key id; + +create dataset UpsertFromDBLP(DBLPType) + primary key id; + +create index keyword_index on UpsertToDBLP(title) type keyword; \ No newline at end of file