Return-Path: X-Original-To: apmail-asterixdb-commits-archive@minotaur.apache.org Delivered-To: apmail-asterixdb-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A3BDE18CFE for ; Tue, 25 Aug 2015 16:50:49 +0000 (UTC) Received: (qmail 99119 invoked by uid 500); 25 Aug 2015 16:50:47 -0000 Delivered-To: apmail-asterixdb-commits-archive@asterixdb.apache.org Received: (qmail 98997 invoked by uid 500); 25 Aug 2015 16:50:47 -0000 Mailing-List: contact commits-help@asterixdb.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.incubator.apache.org Delivered-To: mailing list commits@asterixdb.incubator.apache.org Received: (qmail 98946 invoked by uid 99); 25 Aug 2015 16:50:47 -0000 Received: from Unknown (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Aug 2015 16:50:47 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id A6895C009B for ; Tue, 25 Aug 2015 16:50:46 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -0.006 X-Spam-Level: X-Spam-Status: No, score=-0.006 tagged_above=-999 required=6.31 tests=[RP_MATCHES_RCVD=-0.006] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id xFGLPpku7K7q for ; Tue, 25 Aug 2015 16:50:31 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 2F471506FC for ; Tue, 25 Aug 2015 16:50:31 +0000 (UTC) Received: (qmail 75845 invoked by uid 99); 25 Aug 2015 16:43:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Aug 2015 16:43:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 51BB3E7140; Tue, 25 Aug 2015 16:43:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: imaxon@apache.org To: commits@asterixdb.incubator.apache.org Date: Tue, 25 Aug 2015 16:44:26 -0000 Message-Id: <5b2867459945428fa93af1ec7ebf18f7@git.apache.org> In-Reply-To: <82235e2314bc4ce9bc242ea1878b4eae@git.apache.org> References: <82235e2314bc4ce9bc242ea1878b4eae@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [38/51] [partial] incubator-asterixdb git commit: Change folder structure for Java repackage http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java deleted file mode 100644 index 0360ffb..0000000 --- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java +++ /dev/null @@ -1,1608 +0,0 @@ -/* - * Copyright 2009-2013 by The Regents of the University of California - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * you may obtain a copy of the License from - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package edu.uci.ics.asterix.translator; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.lang3.mutable.Mutable; -import org.apache.commons.lang3.mutable.MutableObject; - -import edu.uci.ics.asterix.aql.base.Clause; -import edu.uci.ics.asterix.aql.base.Expression; -import edu.uci.ics.asterix.aql.base.Expression.Kind; -import edu.uci.ics.asterix.aql.expression.CallExpr; -import edu.uci.ics.asterix.aql.expression.CompactStatement; -import edu.uci.ics.asterix.aql.expression.ConnectFeedStatement; -import edu.uci.ics.asterix.aql.expression.CreateDataverseStatement; -import edu.uci.ics.asterix.aql.expression.CreateFeedPolicyStatement; -import edu.uci.ics.asterix.aql.expression.CreateFunctionStatement; -import edu.uci.ics.asterix.aql.expression.CreateIndexStatement; -import edu.uci.ics.asterix.aql.expression.CreatePrimaryFeedStatement; -import edu.uci.ics.asterix.aql.expression.CreateSecondaryFeedStatement; -import edu.uci.ics.asterix.aql.expression.DatasetDecl; -import edu.uci.ics.asterix.aql.expression.DataverseDecl; -import edu.uci.ics.asterix.aql.expression.DataverseDropStatement; -import edu.uci.ics.asterix.aql.expression.DeleteStatement; -import edu.uci.ics.asterix.aql.expression.DisconnectFeedStatement; -import edu.uci.ics.asterix.aql.expression.DistinctClause; -import edu.uci.ics.asterix.aql.expression.DropStatement; -import edu.uci.ics.asterix.aql.expression.FLWOGRExpression; -import edu.uci.ics.asterix.aql.expression.FeedDropStatement; -import edu.uci.ics.asterix.aql.expression.FeedPolicyDropStatement; -import edu.uci.ics.asterix.aql.expression.FieldAccessor; -import edu.uci.ics.asterix.aql.expression.FieldBinding; -import edu.uci.ics.asterix.aql.expression.ForClause; -import edu.uci.ics.asterix.aql.expression.FunctionDecl; -import edu.uci.ics.asterix.aql.expression.FunctionDropStatement; -import edu.uci.ics.asterix.aql.expression.GbyVariableExpressionPair; -import edu.uci.ics.asterix.aql.expression.GroupbyClause; -import edu.uci.ics.asterix.aql.expression.IfExpr; -import edu.uci.ics.asterix.aql.expression.IndexAccessor; -import edu.uci.ics.asterix.aql.expression.IndexDropStatement; -import edu.uci.ics.asterix.aql.expression.InsertStatement; -import edu.uci.ics.asterix.aql.expression.LetClause; -import edu.uci.ics.asterix.aql.expression.LimitClause; -import edu.uci.ics.asterix.aql.expression.ListConstructor; -import edu.uci.ics.asterix.aql.expression.ListConstructor.Type; -import edu.uci.ics.asterix.aql.expression.LiteralExpr; -import edu.uci.ics.asterix.aql.expression.LoadStatement; -import edu.uci.ics.asterix.aql.expression.NodeGroupDropStatement; -import edu.uci.ics.asterix.aql.expression.NodegroupDecl; -import edu.uci.ics.asterix.aql.expression.OperatorExpr; -import edu.uci.ics.asterix.aql.expression.OperatorType; -import edu.uci.ics.asterix.aql.expression.OrderbyClause; -import edu.uci.ics.asterix.aql.expression.OrderbyClause.OrderModifier; -import edu.uci.ics.asterix.aql.expression.OrderedListTypeDefinition; -import edu.uci.ics.asterix.aql.expression.QuantifiedExpression; -import edu.uci.ics.asterix.aql.expression.QuantifiedExpression.Quantifier; -import edu.uci.ics.asterix.aql.expression.QuantifiedPair; -import edu.uci.ics.asterix.aql.expression.Query; -import edu.uci.ics.asterix.aql.expression.RecordConstructor; -import edu.uci.ics.asterix.aql.expression.RecordTypeDefinition; -import edu.uci.ics.asterix.aql.expression.SetStatement; -import edu.uci.ics.asterix.aql.expression.TypeDecl; -import edu.uci.ics.asterix.aql.expression.TypeDropStatement; -import edu.uci.ics.asterix.aql.expression.TypeReferenceExpression; -import edu.uci.ics.asterix.aql.expression.UnaryExpr; -import edu.uci.ics.asterix.aql.expression.UnaryExpr.Sign; -import edu.uci.ics.asterix.aql.expression.UnionExpr; -import edu.uci.ics.asterix.aql.expression.UnorderedListTypeDefinition; -import edu.uci.ics.asterix.aql.expression.UpdateClause; -import edu.uci.ics.asterix.aql.expression.UpdateStatement; -import edu.uci.ics.asterix.aql.expression.VariableExpr; -import edu.uci.ics.asterix.aql.expression.WhereClause; -import edu.uci.ics.asterix.aql.expression.WriteStatement; -import edu.uci.ics.asterix.aql.expression.visitor.IAqlExpressionVisitor; -import edu.uci.ics.asterix.aql.util.FunctionUtils; -import edu.uci.ics.asterix.aql.util.RangeMapBuilder; -import edu.uci.ics.asterix.common.config.AsterixMetadataProperties; -import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType; -import edu.uci.ics.asterix.common.exceptions.AsterixException; -import edu.uci.ics.asterix.common.functions.FunctionConstants; -import edu.uci.ics.asterix.common.functions.FunctionSignature; -import edu.uci.ics.asterix.metadata.MetadataException; -import edu.uci.ics.asterix.metadata.MetadataManager; -import edu.uci.ics.asterix.metadata.declared.AqlDataSource.AqlDataSourceType; -import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider; -import edu.uci.ics.asterix.metadata.declared.AqlSourceId; -import edu.uci.ics.asterix.metadata.declared.DatasetDataSource; -import edu.uci.ics.asterix.metadata.declared.LoadableDataSource; -import edu.uci.ics.asterix.metadata.declared.ResultSetDataSink; -import edu.uci.ics.asterix.metadata.declared.ResultSetSinkId; -import edu.uci.ics.asterix.metadata.entities.Dataset; -import edu.uci.ics.asterix.metadata.entities.Function; -import edu.uci.ics.asterix.metadata.functions.ExternalFunctionCompilerUtil; -import edu.uci.ics.asterix.metadata.utils.DatasetUtils; -import edu.uci.ics.asterix.om.base.AOrderedList; -import edu.uci.ics.asterix.om.base.AString; -import edu.uci.ics.asterix.om.constants.AsterixConstantValue; -import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions; -import edu.uci.ics.asterix.om.functions.AsterixFunctionInfo; -import edu.uci.ics.asterix.om.types.BuiltinType; -import edu.uci.ics.asterix.om.types.IAType; -import edu.uci.ics.asterix.om.util.AsterixAppContextInfo; -import edu.uci.ics.asterix.runtime.formats.FormatUtils; -import edu.uci.ics.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement; -import edu.uci.ics.asterix.translator.CompiledStatements.ICompiledDmlStatement; -import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; -import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException; -import edu.uci.ics.hyracks.algebricks.common.utils.Pair; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.Counter; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable; -import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; -import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions; -import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; -import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; -import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl; -import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty; -import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn; -import edu.uci.ics.hyracks.api.io.FileReference; -import edu.uci.ics.hyracks.dataflow.std.file.FileSplit; - -/** - * Each visit returns a pair of an operator and a variable. The variable - * corresponds to the new column, if any, added to the tuple flow. E.g., for - * Unnest, the column is the variable bound to the elements in the list, for - * Subplan it is null. The first argument of a visit method is the expression - * which is translated. The second argument of a visit method is the tuple - * source for the current subtree. - */ - -public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator implements - IAqlExpressionVisitor, Mutable> { - - private final AqlMetadataProvider metadataProvider; - private final TranslationContext context; - private final String outputDatasetName; - private final ICompiledDmlStatement stmt; - private static AtomicLong outputFileID = new AtomicLong(0); - private static final String OUTPUT_FILE_PREFIX = "OUTPUT_"; - private static LogicalVariable METADATA_DUMMY_VAR = new LogicalVariable(-1); - - public AqlExpressionToPlanTranslator(AqlMetadataProvider metadataProvider, int currentVarCounter, - String outputDatasetName, ICompiledDmlStatement stmt) throws AlgebricksException { - this.context = new TranslationContext(new Counter(currentVarCounter)); - this.outputDatasetName = outputDatasetName; - this.stmt = stmt; - this.metadataProvider = metadataProvider; - FormatUtils.getDefaultFormat().registerRuntimeFunctions(); - } - - public int getVarCounter() { - return context.getVarCounter(); - } - - public ILogicalPlan translateLoad() throws AlgebricksException { - CompiledLoadFromFileStatement clffs = (CompiledLoadFromFileStatement) stmt; - Dataset dataset = metadataProvider.findDataset(clffs.getDataverseName(), clffs.getDatasetName()); - if (dataset == null) { - // This would never happen since we check for this in AqlTranslator - throw new AlgebricksException("Unable to load dataset " + clffs.getDatasetName() - + " since it does not exist"); - } - IAType itemType = metadataProvider.findType(clffs.getDataverseName(), dataset.getItemTypeName()); - DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), - stmt.getDatasetName()); - List> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset()); - - LoadableDataSource lds; - try { - lds = new LoadableDataSource(dataset, itemType, clffs.getAdapter(), clffs.getProperties()); - } catch (IOException e) { - throw new AlgebricksException(e); - } - - // etsOp is a dummy input operator used to keep the compiler happy. it - // could be removed but would result in - // the need to fix many rewrite rules that assume that datasourcescan - // operators always have input. - ILogicalOperator etsOp = new EmptyTupleSourceOperator(); - - // Add a logical variable for the record. - List payloadVars = new ArrayList(); - payloadVars.add(context.newVar()); - - // Create a scan operator and make the empty tuple source its input - DataSourceScanOperator dssOp = new DataSourceScanOperator(payloadVars, lds); - dssOp.getInputs().add(new MutableObject(etsOp)); - ILogicalExpression payloadExpr = new VariableReferenceExpression(payloadVars.get(0)); - Mutable payloadRef = new MutableObject(payloadExpr); - - // Creating the assign to extract the PK out of the record - ArrayList pkVars = new ArrayList(); - ArrayList> pkExprs = new ArrayList>(); - List> varRefsForLoading = new ArrayList>(); - LogicalVariable payloadVar = payloadVars.get(0); - for (List keyFieldName : partitionKeys) { - prepareVarAndExpression(keyFieldName, payloadVar, pkVars, pkExprs, varRefsForLoading); - } - - AssignOperator assign = new AssignOperator(pkVars, pkExprs); - assign.getInputs().add(new MutableObject(dssOp)); - - // If the input is pre-sorted, we set the ordering property explicitly in the assign - if (clffs.alreadySorted()) { - List orderColumns = new ArrayList(); - for (int i = 0; i < pkVars.size(); ++i) { - orderColumns.add(new OrderColumn(pkVars.get(i), OrderKind.ASC)); - } - assign.setExplicitOrderingProperty(new LocalOrderProperty(orderColumns)); - } - - List additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset()); - List additionalFilteringVars = null; - List> additionalFilteringAssignExpressions = null; - List> additionalFilteringExpressions = null; - AssignOperator additionalFilteringAssign = null; - if (additionalFilteringField != null) { - additionalFilteringVars = new ArrayList(); - additionalFilteringAssignExpressions = new ArrayList>(); - additionalFilteringExpressions = new ArrayList>(); - prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars, - additionalFilteringAssignExpressions, additionalFilteringExpressions); - additionalFilteringAssign = new AssignOperator(additionalFilteringVars, - additionalFilteringAssignExpressions); - } - - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, payloadRef, varRefsForLoading, - InsertDeleteOperator.Kind.INSERT, true); - insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - - if (additionalFilteringAssign != null) { - additionalFilteringAssign.getInputs().add(new MutableObject(assign)); - insertOp.getInputs().add(new MutableObject(additionalFilteringAssign)); - } else { - insertOp.getInputs().add(new MutableObject(assign)); - } - - ILogicalOperator leafOperator = new SinkOperator(); - leafOperator.getInputs().add(new MutableObject(insertOp)); - return new ALogicalPlanImpl(new MutableObject(leafOperator)); - } - - public ILogicalPlan translate(Query expr) throws AlgebricksException, AsterixException { - Pair p = expr.accept(this, new MutableObject( - new EmptyTupleSourceOperator())); - ArrayList> globalPlanRoots = new ArrayList>(); - ILogicalOperator topOp = p.first; - ProjectOperator project = (ProjectOperator) topOp; - LogicalVariable resVar = project.getVariables().get(0); - - if (outputDatasetName == null) { - FileSplit outputFileSplit = metadataProvider.getOutputFile(); - if (outputFileSplit == null) { - outputFileSplit = getDefaultOutputFileLocation(); - } - metadataProvider.setOutputFile(outputFileSplit); - - List> writeExprList = new ArrayList>(1); - writeExprList.add(new MutableObject(new VariableReferenceExpression(resVar))); - ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId()); - ResultSetDataSink sink = new ResultSetDataSink(rssId, null); - topOp = new DistributeResultOperator(writeExprList, sink); - topOp.getInputs().add(new MutableObject(project)); - - // Retrieve the Output RecordType (if any) and store it on - // the DistributeResultOperator - IAType outputRecordType = metadataProvider.findOutputRecordType(); - if (outputRecordType != null) { - topOp.getAnnotations().put("output-record-type", outputRecordType); - } - } else { - /** - * add the collection-to-sequence right before the final project, - * because dataset only accept non-collection records - */ - LogicalVariable seqVar = context.newVar(); - @SuppressWarnings("unchecked") - /** This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest it; otherwise do nothing. */ - AssignOperator assignCollectionToSequence = new AssignOperator(seqVar, - new MutableObject(new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.COLLECTION_TO_SEQUENCE), - new MutableObject(new VariableReferenceExpression(resVar))))); - assignCollectionToSequence.getInputs().add( - new MutableObject(project.getInputs().get(0).getValue())); - project.getInputs().get(0).setValue(assignCollectionToSequence); - project.getVariables().set(0, seqVar); - resVar = seqVar; - - DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), - stmt.getDatasetName()); - ArrayList vars = new ArrayList(); - ArrayList> exprs = new ArrayList>(); - List> varRefsForLoading = new ArrayList>(); - List> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset()); - for (List keyFieldName : partitionKeys) { - prepareVarAndExpression(keyFieldName, resVar, vars, exprs, varRefsForLoading); - } - - List additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset()); - List additionalFilteringVars = null; - List> additionalFilteringAssignExpressions = null; - List> additionalFilteringExpressions = null; - AssignOperator additionalFilteringAssign = null; - if (additionalFilteringField != null) { - additionalFilteringVars = new ArrayList(); - additionalFilteringAssignExpressions = new ArrayList>(); - additionalFilteringExpressions = new ArrayList>(); - - prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars, - additionalFilteringAssignExpressions, additionalFilteringExpressions); - - additionalFilteringAssign = new AssignOperator(additionalFilteringVars, - additionalFilteringAssignExpressions); - } - - AssignOperator assign = new AssignOperator(vars, exprs); - - if (additionalFilteringAssign != null) { - additionalFilteringAssign.getInputs().add(new MutableObject(project)); - assign.getInputs().add(new MutableObject(additionalFilteringAssign)); - } else { - assign.getInputs().add(new MutableObject(project)); - } - - Mutable varRef = new MutableObject(new VariableReferenceExpression( - resVar)); - ILogicalOperator leafOperator = null; - - switch (stmt.getKind()) { - case INSERT: { - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false); - insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - insertOp.getInputs().add(new MutableObject(assign)); - leafOperator = new SinkOperator(); - leafOperator.getInputs().add(new MutableObject(insertOp)); - break; - } - case DELETE: { - InsertDeleteOperator deleteOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.DELETE, false); - deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - deleteOp.getInputs().add(new MutableObject(assign)); - leafOperator = new SinkOperator(); - leafOperator.getInputs().add(new MutableObject(deleteOp)); - break; - } - case CONNECT_FEED: { - InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false); - insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - insertOp.getInputs().add(new MutableObject(assign)); - leafOperator = new SinkOperator(); - leafOperator.getInputs().add(new MutableObject(insertOp)); - break; - } - case SUBSCRIBE_FEED: { - ILogicalOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, varRefsForLoading, - InsertDeleteOperator.Kind.INSERT, false); - insertOp.getInputs().add(new MutableObject(assign)); - leafOperator = new SinkOperator(); - leafOperator.getInputs().add(new MutableObject(insertOp)); - break; - } - } - topOp = leafOperator; - } - globalPlanRoots.add(new MutableObject(topOp)); - ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots); - return plan; - } - - @SuppressWarnings("unchecked") - private void prepareVarAndExpression(List field, LogicalVariable resVar, - List additionalFilteringVars, - List> additionalFilteringAssignExpressions, - List> varRefs) { - IFunctionInfo finfoAccess; - ScalarFunctionCallExpression f; - if (field.size() > 1) { - finfoAccess = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED); - f = new ScalarFunctionCallExpression(finfoAccess, new MutableObject( - new VariableReferenceExpression(METADATA_DUMMY_VAR)), new MutableObject( - new ConstantExpression(new AsterixConstantValue(new AOrderedList(field))))); - } else { - finfoAccess = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME); - f = new ScalarFunctionCallExpression(finfoAccess, new MutableObject( - new VariableReferenceExpression(METADATA_DUMMY_VAR)), new MutableObject( - new ConstantExpression(new AsterixConstantValue(new AString(field.get(0)))))); - } - f.substituteVar(METADATA_DUMMY_VAR, resVar); - additionalFilteringAssignExpressions.add(new MutableObject(f)); - LogicalVariable v = context.newVar(); - additionalFilteringVars.add(v); - varRefs.add(new MutableObject(new VariableReferenceExpression(v))); - } - - private DatasetDataSource validateDatasetInfo(AqlMetadataProvider metadataProvider, String dataverseName, - String datasetName) throws AlgebricksException { - Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); - if (dataset == null) { - throw new AlgebricksException("Cannot find dataset " + datasetName + " in dataverse " + dataverseName); - } - if (dataset.getDatasetType() == DatasetType.EXTERNAL) { - throw new AlgebricksException("Cannot write output to an external dataset."); - } - AqlSourceId sourceId = new AqlSourceId(dataverseName, datasetName); - String itemTypeName = dataset.getItemTypeName(); - IAType itemType = metadataProvider.findType(dataverseName, itemTypeName); - DatasetDataSource dataSource = new DatasetDataSource(sourceId, dataset.getDataverseName(), - dataset.getDatasetName(), itemType, AqlDataSourceType.INTERNAL_DATASET); - - return dataSource; - } - - private FileSplit getDefaultOutputFileLocation() throws MetadataException { - String outputDir = System.getProperty("java.io.tmpDir"); - String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX - + outputFileID.incrementAndGet(); - AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties(); - return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath))); - } - - @Override - public Pair visitForClause(ForClause fc, Mutable tupSource) - throws AsterixException { - LogicalVariable v = context.newVar(fc.getVarExpr()); - Expression inExpr = fc.getInExpr(); - Pair> eo = aqlExprToAlgExpression(inExpr, tupSource); - ILogicalOperator returnedOp; - - if (fc.getPosVarExpr() == null) { - returnedOp = new UnnestOperator(v, new MutableObject(makeUnnestExpression(eo.first))); - } else { - LogicalVariable pVar = context.newVar(fc.getPosVarExpr()); - // We set the positional variable type as INT64 type. - returnedOp = new UnnestOperator(v, new MutableObject(makeUnnestExpression(eo.first)), - pVar, BuiltinType.AINT64, new AqlPositionWriter()); - } - returnedOp.getInputs().add(eo.second); - - return new Pair(returnedOp, v); - } - - @Override - public Pair visitLetClause(LetClause lc, Mutable tupSource) - throws AsterixException { - LogicalVariable v; - ILogicalOperator returnedOp; - - switch (lc.getBindingExpr().getKind()) { - case VARIABLE_EXPRESSION: { - v = context.newVar(lc.getVarExpr()); - LogicalVariable prev = context.getVar(((VariableExpr) lc.getBindingExpr()).getVar().getId()); - returnedOp = new AssignOperator(v, new MutableObject( - new VariableReferenceExpression(prev))); - returnedOp.getInputs().add(tupSource); - break; - } - default: { - v = context.newVar(lc.getVarExpr()); - Pair> eo = aqlExprToAlgExpression(lc.getBindingExpr(), - tupSource); - returnedOp = new AssignOperator(v, new MutableObject(eo.first)); - returnedOp.getInputs().add(eo.second); - break; - } - } - return new Pair(returnedOp, v); - } - - @Override - public Pair visitFlworExpression(FLWOGRExpression flwor, - Mutable tupSource) throws AsterixException { - Mutable flworPlan = tupSource; - boolean isTop = context.isTopFlwor(); - if (isTop) { - context.setTopFlwor(false); - } - for (Clause c : flwor.getClauseList()) { - Pair pC = c.accept(this, flworPlan); - flworPlan = new MutableObject(pC.first); - } - - Expression r = flwor.getReturnExpr(); - boolean noFlworClause = flwor.noForClause(); - - if (r.getKind() == Kind.VARIABLE_EXPRESSION) { - VariableExpr v = (VariableExpr) r; - LogicalVariable var = context.getVar(v.getVar().getId()); - - return produceFlwrResult(noFlworClause, isTop, flworPlan, var); - - } else { - Mutable baseOp = new MutableObject(flworPlan.getValue()); - Pair rRes = r.accept(this, baseOp); - ILogicalOperator rOp = rRes.first; - ILogicalOperator resOp; - if (expressionNeedsNoNesting(r)) { - baseOp.setValue(flworPlan.getValue()); - resOp = rOp; - } else { - SubplanOperator s = new SubplanOperator(rOp); - s.getInputs().add(flworPlan); - resOp = s; - baseOp.setValue(new NestedTupleSourceOperator(new MutableObject(s))); - } - Mutable resOpRef = new MutableObject(resOp); - return produceFlwrResult(noFlworClause, isTop, resOpRef, rRes.second); - } - } - - @Override - public Pair visitFieldAccessor(FieldAccessor fa, - Mutable tupSource) throws AsterixException { - Pair> p = aqlExprToAlgExpression(fa.getExpr(), tupSource); - LogicalVariable v = context.newVar(); - AbstractFunctionCallExpression fldAccess = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)); - fldAccess.getArguments().add(new MutableObject(p.first)); - ILogicalExpression faExpr = new ConstantExpression(new AsterixConstantValue(new AString(fa.getIdent() - .getValue()))); - fldAccess.getArguments().add(new MutableObject(faExpr)); - AssignOperator a = new AssignOperator(v, new MutableObject(fldAccess)); - a.getInputs().add(p.second); - return new Pair(a, v); - } - - @Override - public Pair visitIndexAccessor(IndexAccessor ia, - Mutable tupSource) throws AsterixException { - Pair> p = aqlExprToAlgExpression(ia.getExpr(), tupSource); - LogicalVariable v = context.newVar(); - AbstractFunctionCallExpression f; - if (ia.isAny()) { - f = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.ANY_COLLECTION_MEMBER)); - f.getArguments().add(new MutableObject(p.first)); - } else { - Pair> indexPair = aqlExprToAlgExpression(ia.getIndexExpr(), - tupSource); - f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM)); - f.getArguments().add(new MutableObject(p.first)); - f.getArguments().add(new MutableObject(indexPair.first)); - } - AssignOperator a = new AssignOperator(v, new MutableObject(f)); - a.getInputs().add(p.second); - return new Pair(a, v); - } - - @Override - public Pair visitCallExpr(CallExpr fcall, Mutable tupSource) - throws AsterixException { - LogicalVariable v = context.newVar(); - FunctionSignature signature = fcall.getFunctionSignature(); - List> args = new ArrayList>(); - Mutable topOp = tupSource; - - for (Expression expr : fcall.getExprList()) { - switch (expr.getKind()) { - case VARIABLE_EXPRESSION: { - LogicalVariable var = context.getVar(((VariableExpr) expr).getVar().getId()); - args.add(new MutableObject(new VariableReferenceExpression(var))); - break; - } - case LITERAL_EXPRESSION: { - LiteralExpr val = (LiteralExpr) expr; - args.add(new MutableObject(new ConstantExpression(new AsterixConstantValue( - ConstantHelper.objectFromLiteral(val.getValue()))))); - break; - } - default: { - Pair> eo = aqlExprToAlgExpression(expr, topOp); - AbstractLogicalOperator o1 = (AbstractLogicalOperator) eo.second.getValue(); - args.add(new MutableObject(eo.first)); - if (o1 != null && !(o1.getOperatorTag() == LogicalOperatorTag.ASSIGN && hasOnlyChild(o1, topOp))) { - topOp = eo.second; - } - break; - } - } - } - - AbstractFunctionCallExpression f; - if ((f = lookupUserDefinedFunction(signature, args)) == null) { - f = lookupBuiltinFunction(signature.getName(), signature.getArity(), args); - } - - if (f == null) { - throw new AsterixException(" Unknown function " + signature.getName() + "@" + signature.getArity()); - } - - // Put hints into function call expr. - if (fcall.hasHints()) { - for (IExpressionAnnotation hint : fcall.getHints()) { - f.getAnnotations().put(hint, hint); - } - } - - AssignOperator op = new AssignOperator(v, new MutableObject(f)); - if (topOp != null) { - op.getInputs().add(topOp); - } - - return new Pair(op, v); - } - - private AbstractFunctionCallExpression lookupUserDefinedFunction(FunctionSignature signature, - List> args) throws MetadataException { - if (signature.getNamespace() == null) { - return null; - } - Function function = MetadataManager.INSTANCE.getFunction(metadataProvider.getMetadataTxnContext(), signature); - if (function == null) { - return null; - } - AbstractFunctionCallExpression f = null; - if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_JAVA)) { - IFunctionInfo finfo = ExternalFunctionCompilerUtil.getExternalFunctionInfo( - metadataProvider.getMetadataTxnContext(), function); - f = new ScalarFunctionCallExpression(finfo, args); - } else if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) { - IFunctionInfo finfo = FunctionUtils.getFunctionInfo(signature); - f = new ScalarFunctionCallExpression(finfo, args); - } else { - throw new MetadataException(" User defined functions written in " + function.getLanguage() - + " are not supported"); - } - return f; - } - - private AbstractFunctionCallExpression lookupBuiltinFunction(String functionName, int arity, - List> args) { - AbstractFunctionCallExpression f = null; - FunctionIdentifier fi = new FunctionIdentifier(AlgebricksBuiltinFunctions.ALGEBRICKS_NS, functionName, arity); - AsterixFunctionInfo afi = AsterixBuiltinFunctions.lookupFunction(fi); - FunctionIdentifier builtinAquafi = afi == null ? null : afi.getFunctionIdentifier(); - - if (builtinAquafi != null) { - fi = builtinAquafi; - } else { - fi = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, functionName, arity); - afi = AsterixBuiltinFunctions.lookupFunction(fi); - if (afi == null) { - return null; - } - } - if (AsterixBuiltinFunctions.isBuiltinAggregateFunction(fi)) { - f = AsterixBuiltinFunctions.makeAggregateFunctionExpression(fi, args); - } else if (AsterixBuiltinFunctions.isBuiltinUnnestingFunction(fi)) { - UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression( - FunctionUtils.getFunctionInfo(fi), args); - ufce.setReturnsUniqueValues(AsterixBuiltinFunctions.returnsUniqueValues(fi)); - f = ufce; - } else { - f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fi), args); - } - return f; - } - - @Override - public Pair visitFunctionDecl(FunctionDecl fd, - Mutable tupSource) { - // TODO Auto-generated method stub - throw new NotImplementedException(); - } - - @Override - public Pair visitGroupbyClause(GroupbyClause gc, - Mutable tupSource) throws AsterixException { - GroupByOperator gOp = new GroupByOperator(); - Mutable topOp = tupSource; - for (GbyVariableExpressionPair ve : gc.getGbyPairList()) { - LogicalVariable v; - VariableExpr vexpr = ve.getVar(); - if (vexpr != null) { - v = context.newVar(vexpr); - } else { - v = context.newVar(); - } - Pair> eo = aqlExprToAlgExpression(ve.getExpr(), topOp); - gOp.addGbyExpression(v, eo.first); - topOp = eo.second; - } - for (GbyVariableExpressionPair ve : gc.getDecorPairList()) { - LogicalVariable v; - VariableExpr vexpr = ve.getVar(); - if (vexpr != null) { - v = context.newVar(vexpr); - } else { - v = context.newVar(); - } - Pair> eo = aqlExprToAlgExpression(ve.getExpr(), topOp); - gOp.addDecorExpression(v, eo.first); - topOp = eo.second; - } - gOp.getInputs().add(topOp); - - for (VariableExpr var : gc.getWithVarList()) { - LogicalVariable aggVar = context.newVar(); - LogicalVariable oldVar = context.getVar(var); - List> flArgs = new ArrayList>(1); - flArgs.add(new MutableObject(new VariableReferenceExpression(oldVar))); - AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions.makeAggregateFunctionExpression( - AsterixBuiltinFunctions.LISTIFY, flArgs); - AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), - (List) mkSingletonArrayList(new MutableObject(fListify))); - - agg.getInputs().add( - new MutableObject(new NestedTupleSourceOperator( - new MutableObject(gOp)))); - ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject(agg)); - gOp.getNestedPlans().add(plan); - // Hide the variable that was part of the "with", replacing it with - // the one bound by the aggregation op. - context.setVar(var, aggVar); - } - - gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint()); - return new Pair(gOp, null); - } - - @Override - public Pair visitIfExpr(IfExpr ifexpr, Mutable tupSource) - throws AsterixException { - // In the most general case, IfThenElse is translated in the following - // way. - // - // We assign the result of the condition to one variable varCond. - // We create one subplan which contains the plan for the "then" branch, - // on top of which there is a selection whose condition is varCond. - // Similarly, we create one subplan for the "else" branch, in which the - // selection is not(varCond). - // Finally, we concatenate the results. (??) - - Pair pCond = ifexpr.getCondExpr().accept(this, tupSource); - ILogicalOperator opCond = pCond.first; - LogicalVariable varCond = pCond.second; - - SubplanOperator sp = new SubplanOperator(); - Mutable nestedSource = new MutableObject(new NestedTupleSourceOperator( - new MutableObject(sp))); - - Pair pThen = ifexpr.getThenExpr().accept(this, nestedSource); - SelectOperator sel1 = new SelectOperator(new MutableObject(new VariableReferenceExpression( - varCond)), false, null); - sel1.getInputs().add(new MutableObject(pThen.first)); - - Pair pElse = ifexpr.getElseExpr().accept(this, nestedSource); - AbstractFunctionCallExpression notVarCond = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), new MutableObject( - new VariableReferenceExpression(varCond))); - SelectOperator sel2 = new SelectOperator(new MutableObject(notVarCond), false, null); - sel2.getInputs().add(new MutableObject(pElse.first)); - - ILogicalPlan p1 = new ALogicalPlanImpl(new MutableObject(sel1)); - sp.getNestedPlans().add(p1); - ILogicalPlan p2 = new ALogicalPlanImpl(new MutableObject(sel2)); - sp.getNestedPlans().add(p2); - - Mutable opCondRef = new MutableObject(opCond); - sp.getInputs().add(opCondRef); - - LogicalVariable resV = context.newVar(); - AbstractFunctionCallExpression concatNonNull = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CONCAT_NON_NULL), - new MutableObject(new VariableReferenceExpression(pThen.second)), - new MutableObject(new VariableReferenceExpression(pElse.second))); - AssignOperator a = new AssignOperator(resV, new MutableObject(concatNonNull)); - a.getInputs().add(new MutableObject(sp)); - - return new Pair(a, resV); - } - - @Override - public Pair visitLiteralExpr(LiteralExpr l, Mutable tupSource) { - LogicalVariable var = context.newVar(); - AssignOperator a = new AssignOperator(var, new MutableObject(new ConstantExpression( - new AsterixConstantValue(ConstantHelper.objectFromLiteral(l.getValue()))))); - if (tupSource != null) { - a.getInputs().add(tupSource); - } - return new Pair(a, var); - } - - @Override - public Pair visitOperatorExpr(OperatorExpr op, - Mutable tupSource) throws AsterixException { - ArrayList ops = op.getOpList(); - int nOps = ops.size(); - - if (nOps > 0 && (ops.get(0) == OperatorType.AND || ops.get(0) == OperatorType.OR)) { - return visitAndOrOperator(op, tupSource); - } - - ArrayList exprs = op.getExprList(); - - Mutable topOp = tupSource; - - ILogicalExpression currExpr = null; - for (int i = 0; i <= nOps; i++) { - - Pair> p = aqlExprToAlgExpression(exprs.get(i), topOp); - topOp = p.second; - ILogicalExpression e = p.first; - // now look at the operator - if (i < nOps) { - if (OperatorExpr.opIsComparison(ops.get(i))) { - AbstractFunctionCallExpression c = createComparisonExpression(ops.get(i)); - - // chain the operators - if (i == 0) { - c.getArguments().add(new MutableObject(e)); - currExpr = c; - if (op.isBroadcastOperand(i)) { - BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation(); - bcast.setObject(BroadcastSide.LEFT); - c.getAnnotations().put(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast); - } - } else { - ((AbstractFunctionCallExpression) currExpr).getArguments().add( - new MutableObject(e)); - c.getArguments().add(new MutableObject(currExpr)); - currExpr = c; - if (i == 1 && op.isBroadcastOperand(i)) { - BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation(); - bcast.setObject(BroadcastSide.RIGHT); - c.getAnnotations().put(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast); - } - } - } else { - AbstractFunctionCallExpression f = createFunctionCallExpressionForBuiltinOperator(ops.get(i)); - - if (i == 0) { - f.getArguments().add(new MutableObject(e)); - currExpr = f; - } else { - ((AbstractFunctionCallExpression) currExpr).getArguments().add( - new MutableObject(e)); - f.getArguments().add(new MutableObject(currExpr)); - currExpr = f; - } - } - } else { // don't forget the last expression... - ((AbstractFunctionCallExpression) currExpr).getArguments() - .add(new MutableObject(e)); - if (i == 1 && op.isBroadcastOperand(i)) { - BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation(); - bcast.setObject(BroadcastSide.RIGHT); - ((AbstractFunctionCallExpression) currExpr).getAnnotations().put( - BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY, bcast); - } - } - } - - // Add hints as annotations. - if (op.hasHints() && currExpr instanceof AbstractFunctionCallExpression) { - AbstractFunctionCallExpression currFuncExpr = (AbstractFunctionCallExpression) currExpr; - for (IExpressionAnnotation hint : op.getHints()) { - currFuncExpr.getAnnotations().put(hint, hint); - } - } - - LogicalVariable assignedVar = context.newVar(); - AssignOperator a = new AssignOperator(assignedVar, new MutableObject(currExpr)); - - a.getInputs().add(topOp); - - return new Pair(a, assignedVar); - } - - @Override - public Pair visitOrderbyClause(OrderbyClause oc, - Mutable tupSource) throws AsterixException { - OrderOperator ord = new OrderOperator(); - Iterator modifIter = oc.getModifierList().iterator(); - Mutable topOp = tupSource; - for (Expression e : oc.getOrderbyList()) { - Pair> p = aqlExprToAlgExpression(e, topOp); - OrderModifier m = modifIter.next(); - OrderOperator.IOrder comp = (m == OrderModifier.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER; - ord.getOrderExpressions() - .add(new Pair>(comp, new MutableObject( - p.first))); - topOp = p.second; - } - ord.getInputs().add(topOp); - if (oc.getNumTuples() > 0) { - ord.getAnnotations().put(OperatorAnnotations.CARDINALITY, oc.getNumTuples()); - } - if (oc.getNumFrames() > 0) { - ord.getAnnotations().put(OperatorAnnotations.MAX_NUMBER_FRAMES, oc.getNumFrames()); - } - if (oc.getRangeMap() != null) { - Iterator orderModifIter = oc.getModifierList().iterator(); - boolean ascending = (orderModifIter.next() == OrderModifier.ASC); - RangeMapBuilder.verifyRangeOrder(oc.getRangeMap(), ascending); - ord.getAnnotations().put(OperatorAnnotations.USE_RANGE_CONNECTOR, oc.getRangeMap()); - } - return new Pair(ord, null); - } - - @Override - public Pair visitQuantifiedExpression(QuantifiedExpression qe, - Mutable tupSource) throws AsterixException { - Mutable topOp = tupSource; - - ILogicalOperator firstOp = null; - Mutable lastOp = null; - - for (QuantifiedPair qt : qe.getQuantifiedList()) { - Pair> eo1 = aqlExprToAlgExpression(qt.getExpr(), topOp); - topOp = eo1.second; - LogicalVariable uVar = context.newVar(qt.getVarExpr()); - ILogicalOperator u = new UnnestOperator(uVar, new MutableObject( - makeUnnestExpression(eo1.first))); - - if (firstOp == null) { - firstOp = u; - } - if (lastOp != null) { - u.getInputs().add(lastOp); - } - lastOp = new MutableObject(u); - } - - // We make all the unnest correspond. to quantif. vars. sit on top - // in the hope of enabling joins & other optimiz. - firstOp.getInputs().add(topOp); - topOp = lastOp; - - Pair> eo2 = aqlExprToAlgExpression(qe.getSatisfiesExpr(), topOp); - - AggregateFunctionCallExpression fAgg; - SelectOperator s; - if (qe.getQuantifier() == Quantifier.SOME) { - s = new SelectOperator(new MutableObject(eo2.first), false, null); - s.getInputs().add(eo2.second); - fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.NON_EMPTY_STREAM, - new ArrayList>()); - } else { // EVERY - List> satExprList = new ArrayList>(1); - satExprList.add(new MutableObject(eo2.first)); - s = new SelectOperator(new MutableObject(new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AlgebricksBuiltinFunctions.NOT), satExprList)), false, null); - s.getInputs().add(eo2.second); - fAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression(AsterixBuiltinFunctions.EMPTY_STREAM, - new ArrayList>()); - } - LogicalVariable qeVar = context.newVar(); - AggregateOperator a = new AggregateOperator(mkSingletonArrayList(qeVar), - (List) mkSingletonArrayList(new MutableObject(fAgg))); - a.getInputs().add(new MutableObject(s)); - return new Pair(a, qeVar); - } - - @Override - public Pair visitQuery(Query q, Mutable tupSource) - throws AsterixException { - return q.getBody().accept(this, tupSource); - } - - @Override - public Pair visitRecordConstructor(RecordConstructor rc, - Mutable tupSource) throws AsterixException { - AbstractFunctionCallExpression f = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)); - LogicalVariable v1 = context.newVar(); - AssignOperator a = new AssignOperator(v1, new MutableObject(f)); - Mutable topOp = tupSource; - for (FieldBinding fb : rc.getFbList()) { - Pair> eo1 = aqlExprToAlgExpression(fb.getLeftExpr(), topOp); - f.getArguments().add(new MutableObject(eo1.first)); - topOp = eo1.second; - Pair> eo2 = aqlExprToAlgExpression(fb.getRightExpr(), topOp); - f.getArguments().add(new MutableObject(eo2.first)); - topOp = eo2.second; - } - a.getInputs().add(topOp); - return new Pair(a, v1); - } - - @Override - public Pair visitListConstructor(ListConstructor lc, - Mutable tupSource) throws AsterixException { - FunctionIdentifier fid = (lc.getType() == Type.ORDERED_LIST_CONSTRUCTOR) ? AsterixBuiltinFunctions.ORDERED_LIST_CONSTRUCTOR - : AsterixBuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR; - AbstractFunctionCallExpression f = new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fid)); - LogicalVariable v1 = context.newVar(); - AssignOperator a = new AssignOperator(v1, new MutableObject(f)); - Mutable topOp = tupSource; - for (Expression expr : lc.getExprList()) { - Pair> eo = aqlExprToAlgExpression(expr, topOp); - f.getArguments().add(new MutableObject(eo.first)); - topOp = eo.second; - } - a.getInputs().add(topOp); - return new Pair(a, v1); - } - - @Override - public Pair visitUnaryExpr(UnaryExpr u, Mutable tupSource) - throws AsterixException { - Expression expr = u.getExpr(); - Pair> eo = aqlExprToAlgExpression(expr, tupSource); - LogicalVariable v1 = context.newVar(); - AssignOperator a; - if (u.getSign() == Sign.POSITIVE) { - a = new AssignOperator(v1, new MutableObject(eo.first)); - } else { - AbstractFunctionCallExpression m = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NUMERIC_UNARY_MINUS)); - m.getArguments().add(new MutableObject(eo.first)); - a = new AssignOperator(v1, new MutableObject(m)); - } - a.getInputs().add(eo.second); - return new Pair(a, v1); - } - - @Override - public Pair visitVariableExpr(VariableExpr v, Mutable tupSource) { - // Should we ever get to this method? - LogicalVariable var = context.newVar(); - LogicalVariable oldV = context.getVar(v.getVar().getId()); - AssignOperator a = new AssignOperator(var, new MutableObject( - new VariableReferenceExpression(oldV))); - a.getInputs().add(tupSource); - return new Pair(a, var); - } - - @Override - public Pair visitWhereClause(WhereClause w, Mutable tupSource) - throws AsterixException { - Pair> p = aqlExprToAlgExpression(w.getWhereExpr(), tupSource); - SelectOperator s = new SelectOperator(new MutableObject(p.first), false, null); - s.getInputs().add(p.second); - - return new Pair(s, null); - } - - @Override - public Pair visitLimitClause(LimitClause lc, Mutable tupSource) - throws AsterixException { - Pair> p1 = aqlExprToAlgExpression(lc.getLimitExpr(), tupSource); - LimitOperator opLim; - Expression offset = lc.getOffset(); - if (offset != null) { - Pair> p2 = aqlExprToAlgExpression(offset, p1.second); - opLim = new LimitOperator(p1.first, p2.first); - opLim.getInputs().add(p2.second); - } else { - opLim = new LimitOperator(p1.first); - opLim.getInputs().add(p1.second); - } - return new Pair(opLim, null); - } - - @Override - public Pair visitDistinctClause(DistinctClause dc, - Mutable tupSource) throws AsterixException { - List> exprList = new ArrayList>(); - Mutable input = null; - for (Expression expr : dc.getDistinctByExpr()) { - Pair> p = aqlExprToAlgExpression(expr, tupSource); - exprList.add(new MutableObject(p.first)); - input = p.second; - } - DistinctOperator opDistinct = new DistinctOperator(exprList); - opDistinct.getInputs().add(input); - return new Pair(opDistinct, null); - } - - @Override - public Pair visitUnionExpr(UnionExpr unionExpr, - Mutable tupSource) throws AsterixException { - //Translate the AQL union into an assign [var] <- [function-call: asterix:union, Args:[..]] - //The rule "IntroduceUnionRule" will translates this assign operator into the UnionAll operator. - Mutable ts = tupSource; - LogicalVariable assignedVar = context.newVar(); - List> inputVars = new ArrayList>(); - for (Expression e : unionExpr.getExprs()) { - Pair op_var = e.accept(this, ts); - ts = new MutableObject(op_var.first); - VariableReferenceExpression var = new VariableReferenceExpression(op_var.second); - inputVars.add(new MutableObject(var)); - } - AbstractFunctionCallExpression union = new ScalarFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.UNION), inputVars); - AssignOperator a = new AssignOperator(assignedVar, new MutableObject(union)); - a.getInputs().add(ts); - return new Pair(a, assignedVar); - } - - private AbstractFunctionCallExpression createComparisonExpression(OperatorType t) { - FunctionIdentifier fi = operatorTypeToFunctionIdentifier(t); - IFunctionInfo finfo = FunctionUtils.getFunctionInfo(fi); - return new ScalarFunctionCallExpression(finfo); - } - - private FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) { - switch (t) { - case EQ: { - return AlgebricksBuiltinFunctions.EQ; - } - case NEQ: { - return AlgebricksBuiltinFunctions.NEQ; - } - case GT: { - return AlgebricksBuiltinFunctions.GT; - } - case GE: { - return AlgebricksBuiltinFunctions.GE; - } - case LT: { - return AlgebricksBuiltinFunctions.LT; - } - case LE: { - return AlgebricksBuiltinFunctions.LE; - } - default: { - throw new IllegalStateException(); - } - } - } - - private AbstractFunctionCallExpression createFunctionCallExpressionForBuiltinOperator(OperatorType t) - throws AsterixException { - - FunctionIdentifier fid = null; - switch (t) { - case PLUS: { - fid = AlgebricksBuiltinFunctions.NUMERIC_ADD; - break; - } - case MINUS: { - fid = AsterixBuiltinFunctions.NUMERIC_SUBTRACT; - break; - } - case MUL: { - fid = AsterixBuiltinFunctions.NUMERIC_MULTIPLY; - break; - } - case DIV: { - fid = AsterixBuiltinFunctions.NUMERIC_DIVIDE; - break; - } - case MOD: { - fid = AsterixBuiltinFunctions.NUMERIC_MOD; - break; - } - case IDIV: { - fid = AsterixBuiltinFunctions.NUMERIC_IDIV; - break; - } - case CARET: { - fid = AsterixBuiltinFunctions.CARET; - break; - } - case AND: { - fid = AlgebricksBuiltinFunctions.AND; - break; - } - case OR: { - fid = AlgebricksBuiltinFunctions.OR; - break; - } - case FUZZY_EQ: { - fid = AsterixBuiltinFunctions.FUZZY_EQ; - break; - } - - default: { - throw new NotImplementedException("Operator " + t + " is not yet implemented"); - } - } - return new ScalarFunctionCallExpression(FunctionUtils.getFunctionInfo(fid)); - } - - private static boolean hasOnlyChild(ILogicalOperator parent, Mutable childCandidate) { - List> inp = parent.getInputs(); - if (inp == null || inp.size() != 1) { - return false; - } - return inp.get(0) == childCandidate; - } - - private Pair> aqlExprToAlgExpression(Expression expr, - Mutable topOp) throws AsterixException { - switch (expr.getKind()) { - case VARIABLE_EXPRESSION: { - VariableReferenceExpression ve = new VariableReferenceExpression(context.getVar(((VariableExpr) expr) - .getVar().getId())); - return new Pair>(ve, topOp); - } - case LITERAL_EXPRESSION: { - LiteralExpr val = (LiteralExpr) expr; - return new Pair>(new ConstantExpression( - new AsterixConstantValue(ConstantHelper.objectFromLiteral(val.getValue()))), topOp); - } - default: { - // Mutable src = new - // Mutable(); - // Mutable src = topOp; - if (expressionNeedsNoNesting(expr)) { - Pair p = expr.accept(this, topOp); - ILogicalExpression exp = ((AssignOperator) p.first).getExpressions().get(0).getValue(); - return new Pair>(exp, p.first.getInputs().get(0)); - } else { - Mutable src = new MutableObject(); - - Pair p = expr.accept(this, src); - - if (((AbstractLogicalOperator) p.first).getOperatorTag() == LogicalOperatorTag.SUBPLAN) { - src.setValue(topOp.getValue()); - Mutable top2 = new MutableObject(p.first); - return new Pair>(new VariableReferenceExpression( - p.second), top2); - } else { - SubplanOperator s = new SubplanOperator(); - s.getInputs().add(topOp); - src.setValue(new NestedTupleSourceOperator(new MutableObject(s))); - Mutable planRoot = new MutableObject(p.first); - s.setRootOp(planRoot); - return new Pair>(new VariableReferenceExpression( - p.second), new MutableObject(s)); - } - } - } - } - - } - - private Pair produceFlwrResult(boolean noForClause, boolean isTop, - Mutable resOpRef, LogicalVariable resVar) { - if (isTop) { - ProjectOperator pr = new ProjectOperator(resVar); - pr.getInputs().add(resOpRef); - return new Pair(pr, resVar); - - } else if (noForClause) { - return new Pair(resOpRef.getValue(), resVar); - } else { - return aggListify(resVar, resOpRef, false); - } - } - - private Pair aggListify(LogicalVariable var, Mutable opRef, - boolean bProject) { - AggregateFunctionCallExpression funAgg = AsterixBuiltinFunctions.makeAggregateFunctionExpression( - AsterixBuiltinFunctions.LISTIFY, new ArrayList>()); - funAgg.getArguments().add(new MutableObject(new VariableReferenceExpression(var))); - LogicalVariable varListified = context.newVar(); - AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(varListified), - (List) mkSingletonArrayList(new MutableObject(funAgg))); - agg.getInputs().add(opRef); - ILogicalOperator res; - if (bProject) { - ProjectOperator pr = new ProjectOperator(varListified); - pr.getInputs().add(new MutableObject(agg)); - res = pr; - } else { - res = agg; - } - return new Pair(res, varListified); - } - - private Pair visitAndOrOperator(OperatorExpr op, - Mutable tupSource) throws AsterixException { - ArrayList ops = op.getOpList(); - int nOps = ops.size(); - - ArrayList exprs = op.getExprList(); - - Mutable topOp = tupSource; - - OperatorType opLogical = ops.get(0); - AbstractFunctionCallExpression f = createFunctionCallExpressionForBuiltinOperator(opLogical); - - for (int i = 0; i <= nOps; i++) { - Pair> p = aqlExprToAlgExpression(exprs.get(i), topOp); - topOp = p.second; - // now look at the operator - if (i < nOps) { - if (ops.get(i) != opLogical) { - throw new TranslationException("Unexpected operator " + ops.get(i) - + " in an OperatorExpr starting with " + opLogical); - } - } - f.getArguments().add(new MutableObject(p.first)); - } - - LogicalVariable assignedVar = context.newVar(); - AssignOperator a = new AssignOperator(assignedVar, new MutableObject(f)); - a.getInputs().add(topOp); - - return new Pair(a, assignedVar); - - } - - private static boolean expressionNeedsNoNesting(Expression expr) { - Kind k = expr.getKind(); - return k == Kind.LITERAL_EXPRESSION || k == Kind.LIST_CONSTRUCTOR_EXPRESSION - || k == Kind.RECORD_CONSTRUCTOR_EXPRESSION || k == Kind.VARIABLE_EXPRESSION - || k == Kind.CALL_EXPRESSION || k == Kind.OP_EXPRESSION || k == Kind.FIELD_ACCESSOR_EXPRESSION - || k == Kind.INDEX_ACCESSOR_EXPRESSION || k == Kind.UNARY_EXPRESSION || k == Kind.UNION_EXPRESSION; - } - - private ArrayList mkSingletonArrayList(T item) { - ArrayList array = new ArrayList(1); - array.add(item); - return array; - } - - @Override - public Pair visitTypeDecl(TypeDecl td, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitRecordTypeDefiniton(RecordTypeDefinition tre, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitTypeReferenceExpression(TypeReferenceExpression tre, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitNodegroupDecl(NodegroupDecl ngd, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitLoadStatement(LoadStatement stmtLoad, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDropStatement(DropStatement del, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDisconnectFeedStatement(DisconnectFeedStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCreateIndexStatement(CreateIndexStatement cis, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitOrderedListTypeDefiniton(OrderedListTypeDefinition olte, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitUnorderedListTypeDefiniton(UnorderedListTypeDefinition ulte, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - private ILogicalExpression makeUnnestExpression(ILogicalExpression expr) { - switch (expr.getExpressionTag()) { - case VARIABLE: { - return new UnnestingFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SCAN_COLLECTION), - new MutableObject(expr)); - } - case FUNCTION_CALL: { - AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr; - if (fce.getKind() == FunctionKind.UNNEST) { - return expr; - } else { - return new UnnestingFunctionCallExpression( - FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SCAN_COLLECTION), - new MutableObject(expr)); - } - } - default: { - return expr; - } - } - } - - @Override - public Pair visitInsertStatement(InsertStatement insert, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDeleteStatement(DeleteStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitUpdateStatement(UpdateStatement update, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitUpdateClause(UpdateClause del, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDataverseDecl(DataverseDecl dv, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDatasetDecl(DatasetDecl dd, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitSetStatement(SetStatement ss, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitWriteStatement(WriteStatement ws, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCreateDataverseStatement(CreateDataverseStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitIndexDropStatement(IndexDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitNodeGroupDropStatement(NodeGroupDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDataverseDropStatement(DataverseDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitTypeDropStatement(TypeDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visit(CreateFunctionStatement cfs, Mutable arg) - throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitFunctionDropStatement(FunctionDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCreatePrimaryFeedStatement(CreatePrimaryFeedStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCreateSecondaryFeedStatement(CreateSecondaryFeedStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitConnectFeedStatement(ConnectFeedStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitDropFeedStatement(FeedDropStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCompactStatement(CompactStatement del, - Mutable arg) throws AsterixException { - // TODO Auto-generated method stub - return null; - } - - @Override - public Pair visitCreateFeedPolicyStatement(CreateFeedPolicyStatement cfps, - Mutable arg) throws AsterixException { - return null; - } - - @Override - public Pair visitDropFeedPolicyStatement(FeedPolicyDropStatement dfs, - Mutable arg) throws AsterixException { - return null; - } -}