asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sjaco...@apache.org
Subject [07/12] asterixdb-bad git commit: Initial commit
Date Fri, 09 Sep 2016 20:41:46 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb-bad/blob/bdba1b86/target/generated-resources/javacc/grammar.jj
----------------------------------------------------------------------
diff --git a/target/generated-resources/javacc/grammar.jj b/target/generated-resources/javacc/grammar.jj
new file mode 100644
index 0000000..ad77885
--- /dev/null
+++ b/target/generated-resources/javacc/grammar.jj
@@ -0,0 +1,2949 @@
+options{
+
+
+       STATIC = false;
+
+}
+PARSER_BEGIN(BADAQLParser)
+
+package org.apache.asterix.bad.lang;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.asterix.bad.lang.statement.BrokerDropStatement;
+import org.apache.asterix.bad.lang.statement.ChannelDropStatement;
+import org.apache.asterix.bad.lang.statement.ChannelSubscribeStatement;
+import org.apache.asterix.bad.lang.statement.ChannelUnsubscribeStatement;
+import org.apache.asterix.bad.lang.statement.CreateBrokerStatement;
+import org.apache.asterix.bad.lang.statement.CreateChannelStatement;
+import org.apache.asterix.common.annotations.AutoDataGen;
+import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.FieldIntervalDataGen;
+import org.apache.asterix.common.annotations.FieldValFileDataGen;
+import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import org.apache.asterix.common.annotations.IRecordFieldDataGen;
+import org.apache.asterix.common.annotations.InsertRandIntDataGen;
+import org.apache.asterix.common.annotations.ListDataGen;
+import org.apache.asterix.common.annotations.ListValFileDataGen;
+import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.util.RangeMapBuilder;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.clause.LimitClause;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.clause.UpdateClause;
+import org.apache.asterix.lang.common.clause.WhereClause;
+import org.apache.asterix.lang.common.context.RootScopeFactory;
+import org.apache.asterix.lang.common.context.Scope;
+import org.apache.asterix.lang.common.expression.AbstractAccessor;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.IfExpr;
+import org.apache.asterix.lang.common.expression.IndexAccessor;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.OperatorExpr;
+import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
+import org.apache.asterix.lang.common.expression.UnaryExpr;
+import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.DoubleLiteral;
+import org.apache.asterix.lang.common.literal.FalseLiteral;
+import org.apache.asterix.lang.common.literal.FloatLiteral;
+import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
+import org.apache.asterix.lang.common.literal.MissingLiteral;
+import org.apache.asterix.lang.common.literal.NullLiteral;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.literal.TrueLiteral;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
+import org.apache.asterix.lang.common.statement.CompactStatement;
+import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
+import org.apache.asterix.lang.common.statement.CreateIndexStatement;
+import org.apache.asterix.lang.common.statement.CreatePrimaryFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateSecondaryFeedStatement;
+import org.apache.asterix.lang.common.statement.DatasetDecl;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.DataverseDropStatement;
+import org.apache.asterix.lang.common.statement.DeleteStatement;
+import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FeedDropStatement;
+import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.FunctionDropStatement;
+import org.apache.asterix.lang.common.statement.IndexDropStatement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.LoadStatement;
+import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
+import org.apache.asterix.lang.common.statement.NodegroupDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
+import org.apache.asterix.lang.common.statement.RunStatement;
+import org.apache.asterix.lang.common.statement.SetStatement;
+import org.apache.asterix.lang.common.statement.TypeDecl;
+import org.apache.asterix.lang.common.statement.TypeDropStatement;
+import org.apache.asterix.lang.common.statement.UpdateStatement;
+import org.apache.asterix.lang.common.statement.UpsertStatement;
+import org.apache.asterix.lang.common.statement.WriteStatement;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.xerces.util.IntStack;
+
+class BADAQLParser extends ScopeChecker implements IParser {
+
+    // optimizer hints
+    private static final String AUTO_HINT = "auto";
+    private static final String BROADCAST_JOIN_HINT = "bcast";
+    private static final String COMPOSE_VAL_FILES_HINT = "compose-val-files";
+    private static final String DATE_BETWEEN_YEARS_HINT = "date-between-years";
+    private static final String DATETIME_ADD_RAND_HOURS_HINT = "datetime-add-rand-hours";
+    private static final String DATETIME_BETWEEN_YEARS_HINT = "datetime-between-years";
+    private static final String HASH_GROUP_BY_HINT = "hash";
+    private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
+    private static final String INMEMORY_HINT = "inmem";
+    private static final String INSERT_RAND_INT_HINT = "insert-rand-int";
+    private static final String INTERVAL_HINT = "interval";
+    private static final String LIST_HINT = "list";
+    private static final String LIST_VAL_FILE_HINT = "list-val-file";
+    private static final String RANGE_HINT = "range";
+    private static final String SKIP_SECONDARY_INDEX_SEARCH_HINT = "skip-index";
+    private static final String VAL_FILE_HINT = "val-files";
+    private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
+    private static final String GEN_FIELDS_HINT = "gen-fields";
+    // data generator hints
+    private static final String DGEN_HINT = "dgen";
+
+    private static class IndexParams {
+      public IndexType type;
+      public int gramLength;
+
+      public IndexParams(IndexType type, int gramLength) {
+        this.type = type;
+        this.gramLength = gramLength;
+      }
+    };
+
+    private static class FunctionName {
+       public String dataverse = null;
+       public String library = null;
+       public String function = null;
+       public String hint = null;
+    }
+
+    private static String getHint(Token t) {
+        if (t.specialToken == null) {
+            return null;
+        }
+        String s = t.specialToken.image;
+        int n = s.length();
+        if (n < 2) {
+            return null;
+        }
+        return s.substring(1).trim();
+    }
+
+    private static IRecordFieldDataGen parseFieldDataGen(String hint) throws ParseException {
+      IRecordFieldDataGen rfdg = null;
+      String splits[] = hint.split(" +");
+      if (splits[0].equals(VAL_FILE_HINT)) {
+        File[] valFiles = new File[splits.length - 1];
+        for (int k=1; k<splits.length; k++) {
+          valFiles[k-1] = new File(splits[k]);
+        }
+        rfdg = new FieldValFileDataGen(valFiles);
+      } else if (splits[0].equals(VAL_FILE_SAME_INDEX_HINT)) {
+        rfdg = new FieldValFileSameIndexDataGen(new File(splits[1]), splits[2]);
+      } else if (splits[0].equals(LIST_VAL_FILE_HINT)) {
+        rfdg = new ListValFileDataGen(new File(splits[1]), Integer.parseInt(splits[2]), Integer.parseInt(splits[3]));
+      } else if (splits[0].equals(LIST_HINT)) {
+        rfdg = new ListDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+      } else if (splits[0].equals(INTERVAL_HINT)) {
+        FieldIntervalDataGen.ValueType vt;
+        if (splits[1].equals("int")) {
+          vt = FieldIntervalDataGen.ValueType.INT;
+        } else if (splits[1].equals("long")) {
+          vt = FieldIntervalDataGen.ValueType.LONG;
+        } else if (splits[1].equals("float")) {
+          vt = FieldIntervalDataGen.ValueType.FLOAT;
+        } else if (splits[1].equals("double")) {
+          vt = FieldIntervalDataGen.ValueType.DOUBLE;
+        } else {
+          throw new ParseException("Unknown type for interval data gen: " + splits[1]);
+        }
+        rfdg = new FieldIntervalDataGen(vt, splits[2], splits[3]);
+      } else if (splits[0].equals(INSERT_RAND_INT_HINT)) {
+        rfdg = new InsertRandIntDataGen(splits[1], splits[2]);
+      } else if (splits[0].equals(DATE_BETWEEN_YEARS_HINT)) {
+        rfdg = new DateBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+      } else if (splits[0].equals(DATETIME_BETWEEN_YEARS_HINT)) {
+        rfdg = new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+      } else if (splits[0].equals(DATETIME_ADD_RAND_HOURS_HINT)) {
+        rfdg = new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
+      } else if (splits[0].equals(AUTO_HINT)) {
+        rfdg = new AutoDataGen(splits[1]);
+      }
+      return rfdg;
+    }
+
+    public BADAQLParser(String s){
+        this(new StringReader(s));
+        super.setInput(s);
+    }
+
+    public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
+        File file = new File(args[0]);
+        Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+        BADAQLParser parser = new BADAQLParser(fis);
+        List<Statement> st = parser.parse();
+        //st.accept(new AQLPrintVisitor(), 0);
+    }
+
+    public List<Statement> parse() throws AsterixException {
+        try {
+            return Statement();
+        } catch (Error e) {
+            // this is here as the JavaCharStream that's below the lexer somtimes throws Errors that are not handled
+            // by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
+            throw new AsterixException(new ParseException(e.getMessage()));
+        } catch (ParseException e){
+            throw new AsterixException(e.getMessage());
+        }
+    }
+}
+
+PARSER_END(BADAQLParser)
+
+
+LimitClause LimitClause() throws ParseException:
+{
+    LimitClause lc = new LimitClause();
+    Expression expr;
+    pushForbiddenScope(getCurrentScope());
+}
+{
+    <LIMIT> expr = Expression()    { lc.setLimitExpr(expr);    }
+    (<OFFSET> expr = Expression() { lc.setOffset(expr);    })?
+
+  {
+    popForbiddenScope();
+    return lc;
+  }
+}
+
+String StringLiteral() throws ParseException:
+{
+}
+{
+  <STRING_LITERAL>
+    {
+      return removeQuotesAndEscapes(token.image);
+    }
+}
+
+RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
+{
+  RefreshExternalDatasetStatement redss = new RefreshExternalDatasetStatement();
+  Pair<Identifier,Identifier> nameComponents = null;
+  String datasetName = null;
+}
+{
+  (
+    <REFRESH> <EXTERNAL> <DATASET> nameComponents = QualifiedName()
+    {
+    redss.setDataverseName(nameComponents.first);
+    redss.setDatasetName(nameComponents.second);
+    return redss;
+    }
+  )
+}
+
+Statement CompactStatement() throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  Statement stmt = null;
+}
+{
+  <COMPACT> <DATASET> nameComponents = QualifiedName()
+    {
+      stmt = new CompactStatement(nameComponents.first, nameComponents.second);
+    }
+    {
+      return stmt;
+    }
+}
+
+Expression AndExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+}
+{
+    operand = RelExpr()
+    (
+
+      <AND>
+      {
+        if (op == null) {
+          op = new OperatorExpr();
+          op.addOperand(operand);
+        op.setCurrentop(true);
+        }
+        try{
+          op.addOperator(token.image);
+        } catch (AsterixException e){
+          throw new ParseException(e.getMessage());
+        }
+    }
+
+    operand = RelExpr()
+    {
+      op.addOperand(operand);
+    }
+
+    )*
+
+    {
+      return op==null? operand: op;
+    }
+}
+
+TypeDecl TypeSpecification(String hint, boolean dgen) throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  boolean ifNotExists = false;
+  TypeExpression typeExpr = null;
+}
+{
+  <TYPE> nameComponents = TypeName() ifNotExists = IfNotExists()
+  <AS> typeExpr = TypeExpr()
+    {
+      long numValues = -1;
+      String filename = null;
+      if (dgen) {
+        String splits[] = hint.split(" +");
+        if (splits.length != 3) {
+          throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
+        }
+        filename = splits[1];
+        numValues = Long.parseLong(splits[2]);
+      }
+      TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
+      return new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
+    }
+}
+
+TOKEN_MGR_DECLS:
+{
+    public int commentDepth = 0;
+    public IntStack lexerStateStack = new IntStack();
+
+    public void pushState() {
+      lexerStateStack.push( curLexState );
+    }
+
+    public void popState(String token) {
+      if (lexerStateStack.size() > 0) {
+         SwitchTo( lexerStateStack.pop() );
+      } else {
+         int errorLine = input_stream.getEndLine();
+         int errorColumn = input_stream.getEndColumn();
+         String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
+             + "\" but state stack is empty.";
+         throw new TokenMgrError(msg, -1);
+      }
+    }
+}
+
+Statement WriteStatement() throws ParseException:
+{
+  String nodeName = null;
+  String fileName = null;
+  Query query;
+  String writerClass = null;
+  Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+  <WRITE> <OUTPUT> <TO> nodeName = Identifier() <COLON> fileName = StringLiteral()
+    ( <USING> writerClass = StringLiteral() )?
+    {
+      return new WriteStatement(new Identifier(nodeName), fileName, writerClass);
+    }
+}
+
+Pair<Integer, Pair<List<String>, TypeExpression>>OpenField() throws ParseException:
+{
+  TypeExpression fieldType = null;
+  Pair<Integer, List<String>> fieldList = null;
+}
+{
+  fieldList = NestedField()
+  ( <COLON> fieldType =  IndexedTypeExpr() <QUES>)?
+  {
+    return new Pair<Integer, Pair<List<String>, TypeExpression>>
+           (fieldList.first, new Pair<List<String>, TypeExpression>(fieldList.second, fieldType));
+  }
+}
+
+boolean IfExists() throws ParseException:
+{
+}
+{
+  (
+    LOOKAHEAD(1)
+    <IF> <EXISTS>
+    {
+      return true;
+    }
+  )?
+    {
+      return false;
+    }
+}
+
+Expression Index() throws ParseException:
+{
+    Expression expr = null;
+}
+{
+  <LEFTBRACKET> ( expr = Expression()
+    {
+        if(expr.getKind() == Expression.Kind.LITERAL_EXPRESSION)
+        {
+            Literal lit = ((LiteralExpr)expr).getValue();
+            if(lit.getLiteralType() != Literal.Type.INTEGER &&
+               lit.getLiteralType() != Literal.Type.LONG) {
+                throw new ParseException("Index should be an INTEGER");
+            }
+        }
+    }
+
+      | <QUES> // ANY
+
+      )
+
+   <RIGHTBRACKET>
+    {
+      return expr;
+    }
+}
+
+IndexParams IndexType() throws ParseException:
+{
+  IndexType type = null;
+  int gramLength = 0;
+}
+{
+  (
+    <BTREE>
+    {
+      type = IndexType.BTREE;
+    }
+   |<RTREE>
+    {
+      type = IndexType.RTREE;
+    }
+   |<KEYWORD>
+    {
+      type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
+    }
+   |<NGRAM> <LEFTPAREN> <INTEGER_LITERAL>
+    {
+      type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
+      gramLength = Integer.valueOf(token.image);
+    }
+    <RIGHTPAREN>
+  )
+  {
+    return new IndexParams(type, gramLength);
+  }
+}
+
+CreateBrokerStatement BrokerSpecification() throws ParseException:
+{
+  CreateBrokerStatement cbs = null;
+  Pair<Identifier,Identifier> name = null;
+  String endPoint = null;
+}
+{
+  (
+    <BROKER>  name = QualifiedName()
+    <AT>  endPoint = StringLiteral()
+    {
+      cbs = new CreateBrokerStatement(name.first, name.second,endPoint);
+    }
+  )
+    {
+      return cbs;
+    }
+}
+
+FunctionDecl FunctionDeclaration() throws ParseException:
+{
+  FunctionDecl funcDecl;
+  FunctionSignature signature;
+  String functionName;
+  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+  Expression funcBody;
+  createNewScope();
+}
+{
+  <DECLARE> <FUNCTION> functionName = Identifier()
+  paramList = ParameterList()
+  <LEFTBRACE> funcBody = Expression() <RIGHTBRACE>
+    {
+      signature = new FunctionSignature(defaultDataverse, functionName, paramList.size());
+      getCurrentScope().addFunctionDescriptor(signature, false);
+      funcDecl = new FunctionDecl(signature, paramList, funcBody);
+      removeCurrentScope();
+      return funcDecl;
+    }
+}
+
+Triple<Identifier,Identifier,Identifier>DoubleQualifiedName() throws ParseException:
+{
+  String first = null;
+  String second = null;
+  String third = null;
+}
+{
+  first = Identifier() <DOT> second = Identifier() (<DOT> third = Identifier())?
+  {
+    Identifier id1 = null;
+    Identifier id2 = null;
+    Identifier id3 = null;
+    if (third == null) {
+      id2 = new Identifier(first);
+      id3 = new Identifier(second);
+    } else {
+      id1 = new Identifier(first);
+      id2 = new Identifier(second);
+      id3 = new Identifier(third);
+    }
+    return new Triple<Identifier,Identifier,Identifier>(id1, id2, id3);
+  }
+}
+
+Expression MultExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+}
+{
+    operand = ExponentExpr()
+
+    (( <MUL> | <DIV> | <MOD> | <IDIV>)
+      {
+        if (op == null) {
+          op = new OperatorExpr();
+          op.addOperand(operand);
+          op.setCurrentop(true);
+        }
+        try{
+          op.addOperator(token.image);
+        } catch (AsterixException e){
+          throw new ParseException(e.getMessage());
+        }
+    }
+    operand = ExponentExpr()
+    {
+       op.addOperand(operand);
+    }
+    )*
+
+     {
+       return op==null?operand:op;
+     }
+}
+
+InsertStatement InsertStatement() throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  Pair<Integer, List<String>> returnField = new Pair<Integer, List<String>>(0, null);
+  boolean returnRecord = false;
+  Query query;
+  boolean upsert = false;
+}
+{
+  (<INSERT>|<UPSERT>{ upsert = true; }) <INTO> <DATASET> nameComponents = QualifiedName() 
+  query = Query() (<RETURN> "records"{returnRecord = true;} | "returning" returnField = NestedField())?
+    {
+      query.setTopLevel(true);
+      if(upsert){
+        return new UpsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
+      } else{
+        return new InsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter(),
+        returnRecord, returnField.second);
+      }
+    }
+}
+
+Expression FunctionCallExpr() throws ParseException:
+{
+  CallExpr callExpr;
+  List<Expression> argList = new ArrayList<Expression>();
+  Expression tmp;
+  int arity = 0;
+  FunctionName funcName = null;
+  String hint = null;
+}
+{
+  funcName = FunctionName()
+    {
+      hint = funcName.hint;
+    }
+  <LEFTPAREN> (tmp = Expression()
+    {
+      argList.add(tmp);
+      arity ++;
+    }
+  (<COMMA> tmp = Expression()
+    {
+      argList.add(tmp);
+      arity++;
+    }
+  )*)? <RIGHTPAREN>
+    {
+      // TODO use funcName.library
+      String fqFunctionName = funcName.library == null ? funcName.function : funcName.library + "#" + funcName.function;
+      FunctionSignature signature
+        = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
+      if (signature == null) {
+        signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
+      }
+      callExpr = new CallExpr(signature,argList);
+      if (hint != null) {
+        if (hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+          callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
+        } else if (hint.startsWith(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+          callExpr.addHint(SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE);
+        }
+      }
+      return callExpr;
+    }
+}
+
+Clause Clause() throws ParseException:
+{
+  Clause clause;
+}
+{
+    (
+         clause = ForClause()
+       | clause = LetClause()
+       | clause = WhereClause()
+       | clause = OrderbyClause()
+       | clause = GroupClause()
+       | clause = LimitClause()
+       | clause = DistinctClause()
+    )
+    {
+      return clause;
+    }
+}
+
+Pair<Integer, List<String>>NestedField() throws ParseException:
+{
+  List<String> exprList = new ArrayList<String>();
+  String lit = null;
+  int source = 0;
+}
+{
+  lit = Identifier()
+  {
+    boolean meetParens = false;
+  }
+  (
+    LOOKAHEAD(1)
+    <LEFTPAREN><RIGHTPAREN>
+    {
+        if(!lit.equals("meta")){
+            throw new ParseException("The string before () has to be \"meta\".");
+        }
+        meetParens = true;
+        source = 1;
+    }
+  )?
+  {
+    if(!meetParens){
+        exprList.add(lit);
+    }
+  }
+  (<DOT>
+    lit = Identifier()
+    {
+      exprList.add(lit);
+    }
+  )*
+  {
+    return new Pair<Integer, List<String>>(source, exprList);
+  }
+}
+
+Expression Literal() throws ParseException:
+{
+  LiteralExpr lit = new LiteralExpr();
+  String str = null;
+}
+{
+  ( str = StringLiteral()
+    {
+      lit.setValue(new StringLiteral(str));
+    }
+  | <INTEGER_LITERAL>
+    {
+      lit.setValue(new LongIntegerLiteral(new Long(token.image)));
+    }
+  | <FLOAT_LITERAL>
+    {
+      lit.setValue(new FloatLiteral(new Float(token.image)));
+    }
+  | <DOUBLE_LITERAL>
+    {
+      lit.setValue(new DoubleLiteral(new Double(token.image)));
+    }
+  | <MISSING>
+    {
+      lit.setValue(MissingLiteral.INSTANCE);
+    }
+  | <NULL>
+    {
+      lit.setValue(NullLiteral.INSTANCE);
+    }
+  | <TRUE>
+    {
+      lit.setValue(TrueLiteral.INSTANCE);
+    }
+  | <FALSE>
+    {
+      lit.setValue(FalseLiteral.INSTANCE);
+    }
+  )
+    {
+      return lit;
+    }
+}
+
+Expression OperatorExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+}
+{
+    operand = AndExpr()
+    (
+
+      <OR>
+      {
+        if (op == null) {
+          op = new OperatorExpr();
+          op.addOperand(operand);
+        op.setCurrentop(true);
+        }
+        try{
+          op.addOperator(token.image);
+        } catch (AsterixException e){
+          throw new ParseException(e.getMessage());
+        }
+    }
+
+    operand = AndExpr()
+    {
+      op.addOperand(operand);
+    }
+
+    )*
+
+    {
+      return op==null? operand: op;
+    }
+}
+
+Expression AddExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+}
+{
+    operand = MultExpr()
+
+    ( (<PLUS> | <MINUS>)
+      {
+        if (op == null) {
+          op = new OperatorExpr();
+        op.addOperand(operand);
+        op.setCurrentop(true);
+        }
+        try{
+          ((OperatorExpr)op).addOperator(token.image);
+        } catch (AsterixException e){
+          throw new ParseException(e.getMessage());
+        }
+    }
+
+    operand = MultExpr()
+    {
+      op.addOperand(operand);
+    }
+    )*
+
+    {
+       return op==null? operand: op;
+     }
+}
+
+Statement SetStatement() throws ParseException:
+{
+  String pn = null;
+  String pv = null;
+}
+{
+  <SET> pn = Identifier() pv = StringLiteral()
+    {
+      return new SetStatement(pn, pv);
+    }
+}
+
+RunStatement RunStatement() throws ParseException:
+{
+  String system = null;
+  String tmp;
+  ArrayList<String> parameters = new  ArrayList<String>();
+  Pair<Identifier,Identifier> nameComponentsFrom = null;
+  Pair<Identifier,Identifier> nameComponentsTo = null;
+}
+{
+  <RUN> system = Identifier()<LEFTPAREN> ( tmp = Identifier() [<COMMA>]
+    {
+      parameters.add(tmp);
+    }
+  )*<RIGHTPAREN>
+  <FROM> <DATASET> nameComponentsFrom  = QualifiedName()
+  <TO> <DATASET> nameComponentsTo  = QualifiedName()
+    {
+      return new RunStatement(system, parameters, nameComponentsFrom.first, nameComponentsFrom.second, nameComponentsTo.first, nameComponentsTo.second);
+    }
+}
+
+List<Statement>Statement() throws ParseException:
+{
+  scopeStack.push(RootScopeFactory.createRootScope(this));
+  List<Statement> decls = new ArrayList<Statement>();
+  Statement stmt = null;
+}
+{
+  ( stmt = SingleStatement() (";") ?
+    {
+      decls.add(stmt);
+    }
+  )*
+  <EOF>
+  {
+    return decls;
+  }
+}
+
+String GetPolicy() throws ParseException:
+{
+  String policy = null;
+}
+{
+   <USING> <POLICY> policy = Identifier()
+   {
+     return policy;
+   }
+
+}
+
+Expression FLWOGR() throws ParseException:
+{
+    FLWOGRExpression flworg = new FLWOGRExpression();
+    List<Clause> clauseList = new ArrayList<Clause>();
+    Expression returnExpr;
+    Clause tmp;
+    createNewScope();
+}
+{
+     (tmp = ForClause()  {clauseList.add(tmp);} | tmp = LetClause() {clauseList.add(tmp);})
+      (tmp = Clause() {clauseList.add(tmp);})* (<RETURN>|<SELECT>) returnExpr = Expression()
+
+     {
+       flworg.setClauseList(clauseList);
+       flworg.setReturnExpr(returnExpr);
+       removeCurrentScope();
+       return flworg;
+     }
+}
+
+DistinctClause DistinctClause() throws ParseException:
+{
+  List<Expression> exprs = new ArrayList<Expression>();
+  Expression expr;
+}
+{
+  <DISTINCT> <BY> expr = Expression()
+  {
+    exprs.add(expr);
+  }
+  (<COMMA> expr = Expression()
+      {
+          exprs.add(expr);
+      }
+  )*
+  {
+      return new DistinctClause(exprs);
+  }
+}
+
+Identifier Field() throws ParseException:
+{
+  String ident = null;
+}
+{
+  <DOT> ident = Identifier()
+    {
+      return new Identifier(ident);
+    }
+}
+
+String Identifier() throws ParseException:
+{
+  String lit = null;
+}
+{
+  ((<IDENTIFIER>)
+    {
+      return token.image;
+    }
+  | lit = StringLiteral()
+    {
+      return lit;
+    }
+  )
+}
+
+// Merged Non-terminal
+Statement SingleStatement() throws ParseException:
+{
+  Statement stmt = null;
+}
+{
+  (
+    stmt = DataverseDeclaration()
+    | stmt = FunctionDeclaration()
+    | stmt = CreateStatement()
+    | stmt = LoadStatement()
+    | stmt = DropStatement()
+    | stmt = WriteStatement()
+    | stmt = SetStatement()
+    | stmt = InsertStatement()
+    | stmt = DeleteStatement()
+    | stmt = UpdateStatement()
+    | stmt = FeedStatement()
+    | stmt = CompactStatement()
+    | stmt = Query()
+    | stmt = RefreshExternalDatasetStatement()
+    | stmt = RunStatement()
+    | stmt = ChannelSubscriptionStatement()
+  )
+  {
+  return stmt;
+  }
+}
+
+Clause LetClause() throws ParseException:
+{
+    LetClause lc = new LetClause();
+    VariableExpr varExp;
+    Expression beExp;
+    extendCurrentScope();
+}
+{
+    (<LET>|<WITH>) varExp = Variable() <ASSIGN> beExp = Expression()
+    {
+      getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+      lc.setVarExpr(varExp);
+      lc.setBindingExpr(beExp);
+      return lc;
+    }
+}
+
+ListConstructor UnorderedListConstructor() throws ParseException:
+{
+      ListConstructor expr = new ListConstructor();
+      List<Expression> exprList = null;
+      expr.setType(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR);
+}
+{
+    <LEFTDBLBRACE> exprList = ExpressionList() <RIGHTDBLBRACE>
+    {
+      expr.setExprList(exprList);
+      return expr;
+    }
+}
+
+Expression PrimaryExpr() throws ParseException:
+{
+  Expression expr = null;
+}
+{
+  ( LOOKAHEAD(2)
+    expr = FunctionCallExpr()
+  | expr = Literal()
+  | expr = DatasetAccessExpression()
+  | expr = VariableRef()
+    {
+      if(((VariableExpr)expr).getIsNewVar() == true)
+        throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
+    }
+  | expr = ListConstructor()
+  | expr = RecordConstructor()
+  | expr = ParenthesizedExpression()
+  )
+    {
+      return expr;
+    }
+}
+
+Expression DatasetAccessExpression() throws ParseException:
+{
+  String funcName;
+  String arg1 = null;
+  String arg2 = null;
+  Expression nameArg;
+}
+{
+  <DATASET>
+    {
+      funcName = token.image;
+    }
+  ( ( arg1 = Identifier() ( <DOT> arg2 = Identifier() )? )
+    {
+      String name = arg2 == null ? arg1 : arg1 + "." + arg2;
+      LiteralExpr ds = new LiteralExpr();
+      ds.setValue( new StringLiteral(name) );
+      nameArg = ds;
+      if(arg2 != null){
+          addDataverse(arg1.toString());
+          addDataset(name);
+      } else {
+          addDataset(defaultDataverse + "." + name);
+      }
+    }
+  | ( <LEFTPAREN> nameArg = Expression() <RIGHTPAREN> ) )
+    {
+      String dataverse = MetadataConstants.METADATA_DATAVERSE_NAME;
+      FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, 1);
+      if (signature == null) {
+        signature = new FunctionSignature(dataverse, funcName, 1);
+      }
+      List<Expression> argList = new ArrayList<Expression>();
+      argList.add(nameArg);
+      return new CallExpr(signature, argList);
+    }
+}
+
+Pair<Identifier,Identifier>QualifiedName() throws ParseException:
+{
+  String first = null;
+  String second = null;
+}
+{
+  first = Identifier() (<DOT> second = Identifier())?
+  {
+    Identifier id1 = null;
+    Identifier id2 = null;
+    if (second == null) {
+      id2 = new Identifier(first);
+    } else
+    {
+      id1 = new Identifier(first);
+      id2 = new Identifier(second);
+    }
+    return new Pair<Identifier,Identifier>(id1, id2);
+  }
+}
+
+UpdateClause UpdateClause() throws ParseException:
+{
+  Expression target = null;
+  Expression value = null ;
+  InsertStatement is = null;
+  DeleteStatement ds = null;
+  UpdateStatement us = null;
+  Expression condition = null;
+  UpdateClause ifbranch = null;
+  UpdateClause elsebranch = null;
+}
+{
+   (<SET> target = Expression() <ASSIGN> value = Expression()
+   | is = InsertStatement()
+   | ds = DeleteStatement()
+   | us = UpdateStatement()
+   | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
+     <THEN> ifbranch = UpdateClause()
+     [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
+     {
+       return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
+     }
+   )
+}
+
+Clause ForClause() throws ParseException:
+{
+    ForClause fc = new ForClause();
+    VariableExpr varExp;
+    VariableExpr varPos = null;
+    Expression inExp;
+    extendCurrentScope();
+}
+{
+    (<FOR>|<FROM>) varExp = Variable() (<AT> varPos = Variable())?  <IN> ( inExp = Expression() )
+    {
+      fc.setVarExpr(varExp);
+      getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+      fc.setInExpr(inExp);
+      if (varPos != null) {
+        fc.setPosExpr(varPos);
+        getCurrentScope().addNewVarSymbolToScope(varPos.getVar());
+      }
+      return fc;
+    }
+}
+
+Clause OrderbyClause() throws ParseException:
+{
+    OrderbyClause oc = new OrderbyClause();
+    Expression orderbyExpr;
+    List<Expression> orderbyList = new ArrayList<Expression>();
+    List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier >();
+    int numOfOrderby = 0;
+}
+{
+  (
+    <ORDER>
+      {
+        String hint = getHint(token);
+        if (hint != null) {
+          if (hint.startsWith(INMEMORY_HINT)) {
+            String splits[] = hint.split(" +");
+            int numFrames = Integer.parseInt(splits[1]);
+            int numTuples = Integer.parseInt(splits[2]);
+            oc.setNumFrames(numFrames);
+            oc.setNumTuples(numTuples);
+          }
+          if (hint.startsWith(RANGE_HINT)) {
+            try{
+              oc.setRangeMap(RangeMapBuilder.parseHint(hint.substring(RANGE_HINT.length())));
+            } catch (AsterixException e) {
+              throw new ParseException(e.getMessage());
+            }
+          }
+        }
+      }
+    <BY> orderbyExpr = Expression()
+    {
+      orderbyList.add(orderbyExpr);
+      OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
+    }
+    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+    {
+      modifierList.add(modif);
+    }
+
+    (<COMMA> orderbyExpr = Expression()
+    {
+      orderbyList.add(orderbyExpr);
+      modif = OrderbyClause.OrderModifier.ASC;
+    }
+    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+    {
+      modifierList.add(modif);
+    }
+    )*
+)
+    {
+      oc.setModifierList(modifierList);
+      oc.setOrderbyList(orderbyList);
+      return oc;
+    }
+}
+
+Expression IfThenElse() throws ParseException:
+{
+  Expression condExpr;
+  Expression thenExpr;
+  Expression elseExpr;
+  IfExpr ifExpr = new IfExpr();
+}
+{
+    <IF> <LEFTPAREN> condExpr = Expression() <RIGHTPAREN> <THEN> thenExpr = Expression() <ELSE> elseExpr = Expression()
+
+    {
+      ifExpr.setCondExpr(condExpr);
+      ifExpr.setThenExpr(thenExpr);
+      ifExpr.setElseExpr(elseExpr);
+      return ifExpr;
+    }
+}
+
+void RecordField(RecordTypeDefinition recType) throws ParseException:
+{
+  String fieldName;
+  TypeExpression type = null;
+  boolean nullable = false;
+}
+{
+  fieldName = Identifier()
+    {
+      String hint = getHint(token);
+      IRecordFieldDataGen rfdg = hint != null ? parseFieldDataGen(hint) : null;
+    }
+  <COLON> type =  TypeExpr() (<QUES> { nullable = true; } )?
+    {
+      recType.addField(fieldName, type, nullable, rfdg);
+    }
+}
+
+Expression UnionExpr() throws ParseException:
+{
+    UnionExpr union = null;
+    Expression operand1 = null;
+    Expression operand2 = null;
+}
+{
+   operand1 = UnaryExpr()
+   (<UNION>
+       (operand2 = UnaryExpr()) {
+          if (union == null) {
+             union = new UnionExpr();
+             union.addExpr(operand1);
+          }
+          union.addExpr(operand2);
+       } )*
+   {
+     return (union == null)? operand1: union;
+   }
+}
+
+Expression RelExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+  boolean broadcast = false;
+  IExpressionAnnotation annotation = null;
+}
+{
+    operand = AddExpr()
+    {
+      if (operand instanceof VariableExpr) {
+        String hint = getHint(token);
+        if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+          broadcast = true;
+        }
+      }
+    }
+
+    (
+      LOOKAHEAD(2)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> |<SIMILAR>)
+        {
+          String mhint = getHint(token);
+          if (mhint != null) {
+            if (mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+            annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
+          } else if (mhint.equals(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+            annotation = SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE;
+          }
+        }
+          if (op == null) {
+            op = new OperatorExpr();
+            op.addOperand(operand, broadcast);
+          op.setCurrentop(true);
+          broadcast = false;
+          }
+         try{
+           op.addOperator(token.image);
+         } catch (AsterixException e){
+           throw new ParseException(e.getMessage());
+         }
+      }
+
+       operand = AddExpr()
+      {
+         broadcast = false;
+         if (operand instanceof VariableExpr) {
+           String hint = getHint(token);
+           if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+             broadcast = true;
+           }
+         }
+         op.addOperand(operand, broadcast);
+      }
+    )?
+
+     {
+       if (annotation != null) {
+         op.addHint(annotation);
+       }
+       return op==null? operand: op;
+     }
+}
+
+String FilterField() throws ParseException:
+{
+  String filterField = null;
+}
+{
+  filterField = Identifier()
+    {
+      return filterField;
+    }
+}
+
+FunctionSignature ApplyFunction() throws ParseException:
+{
+  FunctionName functioName = null;
+  FunctionSignature funcSig = null;
+}
+{
+  <APPLY> <FUNCTION> functioName = FunctionName()
+    {
+       String fqFunctionName = functioName.library == null ? functioName.function : functioName.library + "#" + functioName.function;
+       return new FunctionSignature(functioName.dataverse, fqFunctionName, 1);
+    }
+}
+
+String CompactionPolicy() throws ParseException:
+{
+  String compactionPolicy = null;
+}
+{
+  compactionPolicy = Identifier()
+    {
+      return compactionPolicy;
+    }
+}
+
+FunctionName FunctionName() throws ParseException:
+{
+  String first = null;
+  String second = null;
+  String third = null;
+  boolean secondAfterDot = false;
+}
+{
+  first = Identifier()
+  {
+    FunctionName result = new FunctionName();
+    result.hint = getHint(token);
+  }
+  ( <DOT> second = Identifier()
+    {
+      secondAfterDot = true;
+    }
+  ( <SYMBOLHASH> third = Identifier())? | <SYMBOLHASH> second = Identifier() )?
+    {
+      if (second == null) {
+        result.dataverse = defaultDataverse;
+        result.library = null;
+        result.function = first;
+      } else if (third == null) {
+        if (secondAfterDot) {
+          result.dataverse = first;
+          result.library   = null;
+          result.function = second;
+        } else {
+          result.dataverse = defaultDataverse;
+          result.library   = first;
+          result.function = second;
+        }
+      } else {
+        result.dataverse = first;
+        result.library   = second;
+        result.function  = third;
+      }
+
+      if (result.function.equalsIgnoreCase("int")) {
+            result.function = "int64";
+      }
+      return result;
+    }
+}
+
+List<Expression>ExpressionList() throws ParseException:
+{
+      Expression expr = null;
+      List<Expression> list = null;
+      List<Expression> exprList = new ArrayList<Expression>();
+}
+{
+    (
+      expr = Expression() { exprList.add(expr); }
+      (LOOKAHEAD(1) <COMMA> list = ExpressionList() { exprList.addAll(list); })?
+    )?
+    (LOOKAHEAD(1) Comma())?
+    {
+        return exprList;
+    }
+}
+
+DeleteStatement DeleteStatement() throws ParseException:
+{
+  VariableExpr var = null;
+  Expression condition = null;
+  Pair<Identifier, Identifier> nameComponents;
+  // This is related to the new metadata lock management
+  setDataverses(new ArrayList<String>());
+  setDatasets(new ArrayList<String>());
+
+}
+{
+  <DELETE> var = Variable()
+    {
+      getCurrentScope().addNewVarSymbolToScope(var.getVar());
+    }
+  <FROM> <DATASET> nameComponents  = QualifiedName()
+  (<WHERE> condition = Expression())?
+    {
+      // First we get the dataverses and datasets that we want to lock
+      List<String> dataverses = getDataverses();
+      List<String> datasets = getDatasets();
+      // we remove the pointer to the dataverses and datasets
+      setDataverses(null);
+      setDatasets(null);
+      return new DeleteStatement(var, nameComponents.first, nameComponents.second,
+          condition, getVarCounter(), dataverses, datasets);
+    }
+}
+
+Map<String,String>Properties() throws ParseException:
+{
+  Map<String,String> properties = new HashMap<String,String>();
+  Pair<String, String> property;
+}
+{
+  ( <LEFTPAREN> property = Property()
+    {
+      properties.put(property.first, property.second);
+    }
+  ( <COMMA> property = Property()
+    {
+      properties.put(property.first, property.second);
+    }
+  )* <RIGHTPAREN> )?
+    {
+      return properties;
+    }
+}
+
+Clause GroupClause() throws ParseException:
+{
+    GroupbyClause gbc = new GroupbyClause();
+    // GbyVariableExpressionPair pair = new GbyVariableExpressionPair();
+    List<GbyVariableExpressionPair> vePairList = new ArrayList<GbyVariableExpressionPair>();
+    List<GbyVariableExpressionPair> decorPairList = new ArrayList<GbyVariableExpressionPair>();
+    Map<Expression, VariableExpr> withVarMap = new HashMap<Expression, VariableExpr>();
+    VariableExpr var = null;
+    VariableExpr withVar = null;
+    Expression expr = null;
+    VariableExpr decorVar = null;
+    Expression decorExpr = null;
+}
+{
+      {
+        Scope newScope = extendCurrentScopeNoPush(true);
+        // extendCurrentScope(true);
+      }
+    <GROUP>
+      {
+         String hint = getHint(token);
+         if (hint != null && hint.equals(HASH_GROUP_BY_HINT)) {
+           gbc.setHashGroupByHint(true);
+         }
+      }
+    <BY> (LOOKAHEAD(2)  var = Variable()
+    {
+      newScope.addNewVarSymbolToScope(var.getVar());
+    } <ASSIGN>)?
+    expr = Expression()
+       {
+         GbyVariableExpressionPair pair1 = new GbyVariableExpressionPair(var, expr);
+         vePairList.add(pair1);
+       }
+    (<COMMA> ( LOOKAHEAD(2) var = Variable()
+    {
+      newScope.addNewVarSymbolToScope(var.getVar());
+    } <ASSIGN>)?
+        expr = Expression()
+         {
+           GbyVariableExpressionPair pair2 = new GbyVariableExpressionPair(var, expr);
+           vePairList.add(pair2);
+         }
+        )*
+    (<DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+       {
+         newScope.addNewVarSymbolToScope(decorVar.getVar());
+         GbyVariableExpressionPair pair3 = new GbyVariableExpressionPair(decorVar, decorExpr);
+         decorPairList.add(pair3);
+       }
+      (<COMMA> <DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+           {
+             newScope.addNewVarSymbolToScope(decorVar.getVar());
+             GbyVariableExpressionPair pair4 = new GbyVariableExpressionPair(decorVar, decorExpr);
+             decorPairList.add(pair4);
+           }
+       )*
+    )?
+    (<WITH>|<KEEPING>) withVar = VariableRef()
+    {
+      if(withVar.getIsNewVar()==true)
+          throw new ParseException("can't find variable " + withVar.getVar());
+      withVarMap.put(withVar, withVar);
+      newScope.addNewVarSymbolToScope(withVar.getVar());
+    }
+    (<COMMA> withVar = VariableRef()
+    {
+      if(withVar.getIsNewVar()==true)
+          throw new ParseException("can't find variable " + withVar.getVar());
+      withVarMap.put(withVar, withVar);
+      newScope.addNewVarSymbolToScope(withVar.getVar());
+    })*
+    {
+      gbc.setGbyPairList(vePairList);
+      gbc.setDecorPairList(decorPairList);
+      gbc.setWithVarMap(withVarMap);
+      replaceCurrentScope(newScope);
+      return gbc;
+    }
+}
+
+OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
+{
+  TypeExpression type = null;
+}
+{
+  <LEFTBRACKET>
+    ( type =  TypeExpr() )
+  <RIGHTBRACKET>
+  {
+    return new OrderedListTypeDefinition(type);
+  }
+}
+
+boolean IfNotExists() throws ParseException:
+{
+}
+{
+  (
+    LOOKAHEAD(1)
+    <IF>
+    <IDENTIFIER>
+    (
+      {
+        if(!token.image.equals("not")){
+           throw new ParseException("Expect word \"not\" at line " + token.beginLine + ", column "
+                         + token.beginColumn +"!");
+        }
+      }
+    )
+    <EXISTS>
+    {
+      return true;
+    }
+  )?
+    {
+      return false;
+    }
+}
+
+Expression ListConstructor() throws ParseException:
+{
+    Expression expr = null;
+}
+{
+    (
+        expr = OrderedListConstructor() | expr = UnorderedListConstructor()
+    )
+
+    {
+      return expr;
+    }
+}
+
+LoadStatement LoadStatement() throws ParseException:
+{
+  Identifier dataverseName = null;
+  Identifier datasetName = null;
+  boolean alreadySorted = false;
+  String adapterName;
+  Map<String,String> properties;
+  Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+  <LOAD> <DATASET> nameComponents = QualifiedName()
+    {
+      dataverseName = nameComponents.first;
+      datasetName = nameComponents.second;
+    }
+  <USING> adapterName = AdapterName() properties = Configuration()
+  (<PRESORTED>
+    {
+      alreadySorted = true;
+    }
+  )?
+    {
+      return new LoadStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
+    }
+}
+
+Statement ChannelSubscriptionStatement() throws ParseException:
+{
+  Statement stmt = null;
+  Pair<Identifier,Identifier> nameComponents = null;
+  List<Expression> argList = new ArrayList<Expression>();
+  Expression tmp = null;
+  String id = null;
+  String subscriptionId = null;
+  Pair<Identifier,Identifier> brokerName = null;
+}
+{
+  (
+  "subscribe" <TO> nameComponents = QualifiedName()
+   <LEFTPAREN> (tmp = Expression()
+   {
+      argList.add(tmp);
+   }
+   (<COMMA> tmp = Expression()
+   {
+      argList.add(tmp);
+   }
+   )*)? <RIGHTPAREN> <ON> brokerName = QualifiedName()
+   {
+      stmt = new ChannelSubscribeStatement(nameComponents.first, nameComponents.second, argList, getVarCounter(), brokerName.first, brokerName.second, subscriptionId);
+   }
+   | "unsubscribe" id = StringLiteral() <FROM> nameComponents = QualifiedName()
+      {
+        setDataverses(new ArrayList<String>());
+        setDatasets(new ArrayList<String>());
+        VariableExpr varExp = new VariableExpr();
+        VarIdentifier var = new VarIdentifier();
+        varExp.setVar(var);
+        var.setValue("$subscriptionPlaceholder");
+        getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+        List<String> dataverses = getDataverses();
+        List<String> datasets = getDatasets();
+        // we remove the pointer to the dataverses and datasets
+        setDataverses(null);
+        setDatasets(null);
+        stmt = new ChannelUnsubscribeStatement(varExp, nameComponents.first, nameComponents.second, id, getVarCounter(), dataverses, datasets);
+      }
+     | "change" "subscription" subscriptionId = StringLiteral()  <ON> nameComponents = QualifiedName()
+       <LEFTPAREN> (tmp = Expression()
+       {
+         argList.add(tmp);
+       }
+       (<COMMA> tmp = Expression()
+       {
+         argList.add(tmp);
+       }
+       )*)? <RIGHTPAREN>
+        <TO> brokerName = QualifiedName()
+      {
+        stmt = new ChannelSubscribeStatement(nameComponents.first, nameComponents.second, argList, getVarCounter(), brokerName.first, brokerName.second, subscriptionId);
+      }
+    )
+    {
+      return stmt;
+    }
+}
+
+Expression UnaryExpr() throws ParseException:
+{
+    UnaryExpr uexpr = null;
+    Expression expr = null;
+}
+{
+    ( (<PLUS> | <MINUS>)
+    {
+          uexpr = new UnaryExpr();
+          try{
+            uexpr.setExprType(token.image);
+          } catch (AsterixException e){
+            throw new ParseException(e.getMessage());
+          }
+    }
+    )?
+
+    expr = ValueExpr()
+    {
+        if(uexpr!=null){
+            ((UnaryExpr)uexpr).setExpr(expr);
+            return uexpr;
+        }
+        else{
+            return expr;
+        }
+    }
+}
+
+Query Query() throws ParseException:
+{
+  Query query = new Query(false);
+  // we set the pointers to the dataverses and datasets lists to fill them with entities to be locked
+  setDataverses(query.getDataverses());
+  setDatasets(query.getDatasets());
+  Expression expr;
+}
+{
+  expr = Expression()
+    {
+      query.setBody(expr);
+      query.setVarCounter(getVarCounter());
+      // we remove the pointers to the locked entities before we return the query object
+      setDataverses(null);
+      setDatasets(null);
+      return query;
+    }
+}
+
+Expression ValueExpr() throws ParseException:
+{
+  Expression expr = null;
+  Identifier ident = null;
+  AbstractAccessor fa = null;
+  Expression indexExpr = null;
+}
+{
+  expr = PrimaryExpr() ( ident = Field()
+    {
+      fa = (fa == null ? new FieldAccessor(expr, ident)
+                       : new FieldAccessor(fa, ident));
+    }
+  | indexExpr = Index()
+    {
+      fa = (fa == null ? new IndexAccessor(expr, indexExpr)
+                       : new IndexAccessor(fa, indexExpr));
+     }
+  )*
+    {
+      return fa == null ? expr : fa;
+    }
+}
+
+TypeReferenceExpression TypeReference() throws ParseException:
+{
+  String id = null;
+}
+{
+ id = Identifier()
+   {
+     if (id.equalsIgnoreCase("int")) {
+        id = "int64";
+     }
+
+     return new TypeReferenceExpression(new Identifier(id));
+   }
+}
+
+// Merged Non-terminal
+Statement DropStatement() throws ParseException:
+{
+  String id = null;
+  Pair<Identifier,Identifier> pairId = null;
+  Triple<Identifier,Identifier,Identifier> tripleId = null;
+  FunctionSignature funcSig = null;
+  boolean ifExists = false;
+  Statement stmt = null;
+}
+{
+  <DROP>
+  (
+    <DATASET> pairId = QualifiedName() ifExists = IfExists()
+      {
+        stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
+      }
+    | <INDEX> tripleId = DoubleQualifiedName() ifExists = IfExists()
+      {
+        stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
+      }
+    | <NODEGROUP> id = Identifier() ifExists = IfExists()
+      {
+        stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
+      }
+    | <TYPE> pairId = TypeName() ifExists = IfExists()
+      {
+        stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
+      }
+    | <DATAVERSE> id = Identifier() ifExists = IfExists()
+      {
+        stmt = new DataverseDropStatement(new Identifier(id), ifExists);
+      }
+    | <FUNCTION> funcSig = FunctionSignature() ifExists = IfExists()
+      {
+        stmt = new FunctionDropStatement(funcSig, ifExists);
+      }
+    | <FEED> pairId = QualifiedName() ifExists = IfExists()
+      {
+        stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
+      }
+    | <INGESTION> <POLICY> pairId = QualifiedName() ifExists = IfExists()
+      {
+        stmt = new FeedPolicyDropStatement(pairId.first, pairId.second, ifExists);
+      }
+    | <CHANNEL> pairId = QualifiedName() ifExists = IfExists()
+      {
+        stmt = new ChannelDropStatement(pairId.first, pairId.second, ifExists);
+      }
+      	      | <BROKER> pairId = QualifiedName() ifExists = IfExists()	
+      {	
+        stmt = new BrokerDropStatement(pairId.first, pairId.second, ifExists);	
+      }
+      
+  )
+  {
+  return stmt;
+  }
+}
+
+Expression ParenthesizedExpression() throws ParseException:
+{
+  Expression expr;
+}
+{
+    <LEFTPAREN> expr = Expression() <RIGHTPAREN>
+    {
+      return expr;
+    }
+}
+
+// Merged Non-terminal
+Statement CreateStatement() throws ParseException:
+{
+  String hint = null;
+  boolean dgen = false;
+  Statement stmt = null;
+}
+{
+  <CREATE>
+  (
+    {
+      hint = getHint(token);
+      if (hint != null && hint.startsWith(DGEN_HINT)) {
+        dgen = true;
+      }
+    }
+    stmt = TypeSpecification(hint, dgen)
+    | stmt = NodegroupSpecification()
+    | stmt = DatasetSpecification()
+    | stmt = IndexSpecification()
+    | stmt = DataverseSpecification()
+    | stmt = FunctionSpecification()
+    | stmt = FeedSpecification()
+    | stmt = FeedPolicySpecification()
+    | stmt = ChannelSpecification() | stmt = BrokerSpecification()
+  )
+  {
+  return stmt;
+  }
+}
+
+QuantifiedExpression QuantifiedExpression() throws ParseException:
+{
+  QuantifiedExpression qc = new QuantifiedExpression();
+  List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
+  Expression satisfiesExpr;
+  VariableExpr var;
+  Expression inExpr;
+  QuantifiedPair pair;
+}
+{
+  {
+    createNewScope();
+  }
+
+   (      (<SOME>  {  qc.setQuantifier(QuantifiedExpression.Quantifier.SOME);    })
+        | (<EVERY> {  qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY);    }))
+    var = Variable() <IN> inExpr = Expression()
+    {
+      pair = new QuantifiedPair(var, inExpr);
+      getCurrentScope().addNewVarSymbolToScope(var.getVar());
+      quantifiedList.add(pair);
+    }
+    (
+    <COMMA> var = Variable() <IN> inExpr = Expression()
+    {
+      pair = new QuantifiedPair(var, inExpr);
+      getCurrentScope().addNewVarSymbolToScope(var.getVar());
+      quantifiedList.add(pair);
+    }
+    )*
+     <SATISFIES> satisfiesExpr = Expression()
+     {
+       qc.setSatisfiesExpr(satisfiesExpr);
+       qc.setQuantifiedList(quantifiedList);
+       removeCurrentScope();
+       return qc;
+     }
+}
+
+String AdapterName() throws ParseException:
+{
+  String adapterName = null;
+}
+{
+  adapterName = Identifier()
+    {
+      return adapterName;
+    }
+}
+
+DataverseDecl DataverseDeclaration() throws ParseException:
+{
+  String dvName = null;
+}
+{
+  <USE> <DATAVERSE> dvName = Identifier()
+    {
+      defaultDataverse = dvName;
+      return new DataverseDecl(new Identifier(dvName));
+    }
+}
+
+CreateFeedStatement FeedSpecification() throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  boolean ifNotExists = false;
+  String adapterName = null;
+  Map<String,String> properties = null;
+  FunctionSignature appliedFunction = null;
+  CreateFeedStatement cfs = null;
+  Pair<Identifier,Identifier> sourceNameComponents = null;
+
+}
+{
+  (
+    <SECONDARY> <FEED>  nameComponents = QualifiedName() ifNotExists = IfNotExists()
+    <FROM> <FEED> sourceNameComponents = QualifiedName() (appliedFunction = ApplyFunction())?
+    {
+      cfs = new CreateSecondaryFeedStatement(nameComponents, sourceNameComponents, appliedFunction, ifNotExists);
+    }
+   |
+    (<PRIMARY>)? <FEED> nameComponents = QualifiedName() ifNotExists = IfNotExists()
+    <USING> adapterName = AdapterName() properties = Configuration() (appliedFunction = ApplyFunction())?
+     {
+      cfs = new CreatePrimaryFeedStatement(nameComponents, adapterName, properties, appliedFunction, ifNotExists);
+     }
+  )
+  {
+    return cfs;
+  }
+}
+
+FunctionSignature FunctionSignature() throws ParseException:
+{
+  FunctionName fctName = null;
+  int arity = 0;
+}
+{
+  fctName = FunctionName() <SYMBOLAT> <INTEGER_LITERAL>
+    {
+      arity = new Integer(token.image);
+      if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
+        throw new ParseException(" invalid arity:" + arity);
+      }
+
+      // TODO use fctName.library
+      String fqFunctionName = fctName.library == null ? fctName.function : fctName.library + "#" + fctName.function;
+      return new FunctionSignature(fctName.dataverse, fqFunctionName, arity);
+    }
+}
+
+FieldBinding FieldBinding() throws ParseException:
+{
+    FieldBinding fb = new FieldBinding();
+    Expression left, right;
+}
+{
+    left = Expression() <COLON> right = Expression()
+    {
+      fb.setLeftExpr(left);
+      fb.setRightExpr(right);
+      return fb;
+    }
+}
+
+DatasetDecl DatasetSpecification() throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  boolean ifNotExists = false;
+  Pair<Identifier,Identifier> typeComponents = null;
+  String adapterName = null;
+  Map<String,String> properties = null;
+  Map<String,String> compactionPolicyProperties = null;
+  FunctionSignature appliedFunction = null;
+  Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
+  String nodeGroupName = null;
+  Map<String,String> hints = new HashMap<String,String>();
+  DatasetDecl dsetDecl = null;
+  boolean autogenerated = false;
+  String compactionPolicy = null;
+  boolean temp = false;
+  Pair<Integer, List<String>> filterField = null;
+  Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
+}
+{
+  (
+    <EXTERNAL> <DATASET> nameComponents = QualifiedName()
+    <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+    ifNotExists = IfNotExists()
+    <USING> adapterName = AdapterName() properties = Configuration()
+    ( <ON> nodeGroupName = Identifier() )?
+    ( <HINTS> hints = Properties() )?
+    ( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+      {
+        ExternalDetailsDecl edd = new ExternalDetailsDecl();
+        edd.setAdapter(adapterName);
+        edd.setProperties(properties);
+        dsetDecl = new DatasetDecl(nameComponents.first,
+                                   nameComponents.second,
+                                   typeComponents.first,
+                                   typeComponents.second,
+                                   metaTypeComponents.first,
+                                   metaTypeComponents.second,
+                                   nodeGroupName != null? new Identifier(nodeGroupName): null,
+                                   compactionPolicy,
+                                   compactionPolicyProperties,
+                                   hints,
+                                   DatasetType.EXTERNAL,
+                                   edd,
+                                   ifNotExists);
+      }
+
+    | (<INTERNAL> | <TEMPORARY> {
+            temp = token.image.toLowerCase().equals("temporary");
+        }
+      )?
+    <DATASET> nameComponents = QualifiedName()
+    <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+    (
+        { String name; }
+        <WITH>
+        name = Identifier()
+        {
+            if(!name.equals("meta")){
+                throw new ParseException("We can only support one additional associated field called \"meta\".");
+            }
+        }
+        <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN>
+    )?
+    ifNotExists = IfNotExists()
+    primaryKeyFields = PrimaryKey()
+    ( <AUTOGENERATED> { autogenerated = true; } )?
+    ( <ON> nodeGroupName = Identifier() )?
+    ( <HINTS> hints = Properties() )?
+    ( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+    ( <WITH> <FILTER> <ON> filterField = NestedField() )?
+      {
+        if(filterField!=null && filterField.first!=0){
+          throw new ParseException("A filter field can only be a field in the main record of the dataset.");
+        }
+        InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
+                                                          primaryKeyFields.first,
+                                                          autogenerated,
+                                                          filterField == null? null : filterField.second,
+                                                          temp);
+        dsetDecl = new DatasetDecl(nameComponents.first,
+                                   nameComponents.second,
+                                   typeComponents.first,
+                                   typeComponents.second,
+                                   metaTypeComponents.first,
+                                   metaTypeComponents.second,
+                                   nodeGroupName != null ? new Identifier(nodeGroupName) : null,
+                                   compactionPolicy,
+                                   compactionPolicyProperties,
+                                   hints,
+                                   DatasetType.INTERNAL,
+                                   idd,
+                                   ifNotExists);
+      }
+  )
+    {
+      return dsetDecl;
+    }
+}
+
+Clause WhereClause() throws ParseException:
+{
+  WhereClause wc = new WhereClause();
+  Expression whereExpr;
+}
+{
+    <WHERE> whereExpr = Expression()
+    {
+      wc.setWhereExpr(whereExpr);
+      return wc;
+    }
+}
+
+Pair<String, String>KeyValuePair() throws ParseException:
+{
+  String key;
+  String value;
+}
+{
+  <LEFTPAREN> key = StringLiteral() <EQ> value = StringLiteral() <RIGHTPAREN>
+    {
+      return new Pair<String, String>(key, value);
+    }
+}
+
+Statement FeedStatement() throws ParseException:
+{
+  Pair<Identifier,Identifier> feedNameComponents = null;
+  Pair<Identifier,Identifier> datasetNameComponents = null;
+
+  Map<String,String> configuration = null;
+  Statement stmt = null;
+  String policy = null;
+}
+{
+  (
+    <CONNECT> <FEED> feedNameComponents = QualifiedName() <TO> <DATASET> datasetNameComponents = QualifiedName() (policy = GetPolicy())?
+      {
+        stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, policy, getVarCounter());
+      }
+    | <DISCONNECT> <FEED> feedNameComponents = QualifiedName() <FROM> <DATASET> datasetNameComponents = QualifiedName()
+      {
+        stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
+      }
+  )
+    {
+      return stmt;
+    }
+}
+
+CreateFeedPolicyStatement FeedPolicySpecification() throws ParseException:
+{
+  String policyName = null;
+  String basePolicyName = null;
+  String sourcePolicyFile = null;
+  String definition = null;
+  boolean ifNotExists = false;
+  Map<String,String> properties = null;
+  CreateFeedPolicyStatement cfps = null;
+}
+{
+  (
+    <INGESTION> <POLICY>  policyName = Identifier() ifNotExists = IfNotExists()
+    <FROM>
+    (
+      <POLICY> basePolicyName = Identifier() properties = Configuration() ( <DEFINITION> definition = StringLiteral())?
+      {
+        cfps = new CreateFeedPolicyStatement(policyName, basePolicyName, properties, definition, ifNotExists);
+      }
+     |<PATH> sourcePolicyFile = StringLiteral() (<DEFINITION> definition = StringLiteral())?
+      {
+        cfps = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
+      }
+    )
+  )
+  {
+    return cfps;
+  }
+}
+
+VariableExpr VariableRef() throws ParseException:
+{
+    VariableExpr varExp = new VariableExpr();
+    VarIdentifier var = new VarIdentifier();
+}
+{
+  <VARIABLE>
+    {
+     String varName = token.image;
+     Identifier ident = lookupSymbol(varName);
+     if (isInForbiddenScopes(varName)) {
+       throw new ParseException("Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
+     }
+     if(ident != null) { // exist such ident
+       varExp.setIsNewVar(false);
+       varExp.setVar((VarIdentifier)ident);
+     } else {
+       varExp.setVar(var);
+     }
+     var.setValue(varName);
+     return varExp;
+    }
+}
+
+ListConstructor OrderedListConstructor() throws ParseException:
+{
+      ListConstructor expr = new ListConstructor();
+      List<Expression> exprList = null;
+      expr.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
+}
+{
+    <LEFTBRACKET> exprList = ExpressionList() <RIGHTBRACKET>
+    {
+      expr.setExprList(exprList);
+      return expr;
+    }
+}
+
+VariableExpr Variable() throws ParseException:
+{
+    VariableExpr varExp = new VariableExpr();
+    VarIdentifier var = new VarIdentifier();
+}
+{
+  <VARIABLE>
+    {
+     Identifier ident = lookupSymbol(token.image);
+     if(ident != null) { // exist such ident
+       varExp.setIsNewVar(false);
+     }
+     varExp.setVar(var);
+     var.setValue(token.image);
+     return varExp;
+    }
+}
+
+CreateChannelStatement ChannelSpecification() throws ParseException:
+{
+  Pair<Identifier,Identifier> nameComponents = null;
+  FunctionSignature appliedFunction = null;
+  CreateChannelStatement ccs = null;
+  String fqFunctionName = null;
+  Expression period = null;
+}
+{
+  (
+    "repetitive" <CHANNEL>  nameComponents = QualifiedName()
+    <USING> appliedFunction = FunctionSignature()
+    "period" period = FunctionCallExpr()
+    {
+      ccs = new CreateChannelStatement(nameComponents.first,
+                                   nameComponents.second, appliedFunction, period);
+    }
+  )
+    {
+      return ccs;
+    }
+}
+
+UpdateStatement UpdateStatement() throws ParseException:
+{
+  VariableExpr vars;
+  Expression target;
+  Expression condition;
+  UpdateClause uc;
+  List<UpdateClause> ucs = new ArrayList<UpdateClause>();
+}
+{
+  <UPDATE> vars = Variable() <IN> target = Expression()
+  <WHERE> condition = Expression()
+  <LEFTPAREN> (uc = UpdateClause()
+    {
+      ucs.add(uc);
+    }
+  (<COMMA> uc = UpdateClause()
+    {
+      ucs.add(uc);
+    }
+  )*) <RIGHTPAREN>
+    {
+      return new UpdateStatement(vars, target, condition, ucs);
+    }
+}
+
+RecordConstructor RecordConstructor() throws ParseException:
+{
+      RecordConstructor expr = new RecordConstructor();
+      FieldBinding tmp = null;
+      List<FieldBinding> fbList = new ArrayList<FieldBinding>();
+}
+{
+    <LEFTBRACE> (tmp = FieldBinding()
+    {
+      fbList.add(tmp);
+    }
+    (<COMMA> tmp = FieldBinding() { fbList.add(tmp);  })*)? <RIGHTBRACE>
+    {
+      expr.setFbList(fbList);
+      return expr;
+    }
+}
+
+Expression ExponentExpr() throws ParseException:
+{
+  OperatorExpr op = null;
+  Expression operand = null;
+}
+{
+    operand = UnionExpr()
+
+    ( <CARET>
+      {
+        if (op == null) {
+          op = new OperatorExpr();
+          op.addOperand(operand);
+          op.setCurrentop(true);
+        }
+        try{
+          op.addOperator(token.image);
+        } catch (AsterixException e){
+          throw new ParseException(e.getMessage());
+        }
+    }
+    operand = UnionExpr()
+    {
+       op.addOperand(operand);
+    }
+    )?
+
+    {
+       return op==null?operand:op;
+    }
+}
+
+CreateFunctionStatement FunctionSpecification() throws ParseException:
+{
+  FunctionSignature signature;
+  boolean ifNotExists = false;
+  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+  String functionBody;
+  VarIdentifier var = null;
+  Expression functionBodyExpr;
+  Token beginPos;
+  Token endPos;
+  FunctionName fctName = null;
+
+  createNewScope();
+}
+{
+  <FUNCTION> fctName = FunctionName()
+  ifNotExists = IfNotExists()
+  paramList = ParameterList()
+  <LEFTBRACE>
+  {
+     beginPos = token;
+  }
+  functionBodyExpr = Expression() <RIGHTBRACE>
+    {
+      endPos = token;
+      functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
+      // TODO use fctName.library
+      signature = new FunctionSignature(fctName.dataverse, fctName.function, paramList.size());
+      getCurrentScope().addFunctionDescriptor(signature, false);
+      removeCurrentScope();
+      return new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
+    }
+}
+
+List<VarIdentifier>ParameterList() throws ParseException:
+{
+  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+  VarIdentifier var = null;
+}
+{
+  <LEFTPAREN> (<VARIABLE>
+    {
+      var = new VarIdentifier();
+      var.setValue(token.image);
+      paramList.add(var);
+      getCurrentScope().addNewVarSymbolToScope(var);
+    }
+  ( <COMMA> <VARIABLE>
+    {
+      var = new VarIdentifier();
+      var.setValue(token.image);
+      paramList.add(var);
+      getCurrentScope().addNewVarSymbolToScope(var);
+    }
+  )*)? <RIGHTPAREN>
+    {
+      return paramList;
+    }
+}
+
+CreateDataverseStatement DataverseSpecification() throws ParseException:
+{
+  String dvName = null;
+  boolean ifNotExists = false;
+  String format = null;
+}
+{
+  <DATAVERSE> dvName = Identifier()
+  ifNotExists = IfNotExists()
+  ( <WITH> <FORMAT> format = StringLiteral() )?
+    {
+      return new CreateDataverseStatement(new Identifier(dvName), format, ifNotExists);
+    }
+}
+
+void Comma() :
+{}
+{
+   <COMMA>
+}
+
+Pair<String, String>Property() throws ParseException:
+{
+  String key;
+  String value;
+}
+{
+  key = Identifier() <EQ> ( value = StringLiteral() | <INTEGER_LITERAL>
+    {
+      try {
+        value = "" + Long.valueOf(token.image);
+      } catch (NumberFormatException nfe) {
+        throw new ParseException("inapproriate value: " + token.image);
+      }
+    }
+  )
+    {
+      return new Pair<String, String>(key.toUpperCase(), value);
+    }
+}
+
+Expression Expression() :
+{
+  Expression expr = null;
+  Expression exprP = null;
+}
+{
+(
+
+//OperatorExpr | IfThenElse | FLWOGRExpression | QuantifiedExpression
+    expr = OperatorExpr()
+    | expr = IfThenElse()
+    | expr = FLWOGR()
+    | expr = QuantifiedExpression()
+
+
+)
+    {
+      return (exprP==null) ? expr : exprP;
+    }
+}
+
+CreateIndexStatement IndexSpecification() throws ParseException:
+{
+  CreateIndexStatement cis = new CreateIndexStatement();
+  String indexName = null;
+  boolean ifNotExists = false;
+  Pair<Identifier,Identifier> nameComponents = null;
+  Pair<Integer, Pair<List<String>, TypeExpression>> fieldPair = null;
+  IndexParams indexType = null;
+  boolean enforced = false;
+}
+{
+  <INDEX> indexName = Identifier()
+  ifNotExists = IfNotExists()
+  <ON> nameComponents = QualifiedName()
+  <LEFTPAREN> ( fieldPair = OpenField()
+    {
+       cis.addFieldExprPair(fieldPair.second);
+       cis.addFieldIndexIndicator(fieldPair.first);
+    }
+  ) (<COMMA> fieldPair = OpenField()
+    {
+       cis.addFieldExprPair(fieldPair.second);
+       cis.addFieldIndexIndicator(fieldPair.first);
+    }
+  )* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced = true; } )?
+    {
+      cis.setIndexName(new Identifier(indexName));
+      cis.setIfNotExists(ifNotExists);
+      cis.setDataverseName(nameComponents.first);
+      cis.setDatasetName(nameComponents.second);
+      if (indexType != null) {
+        cis.setIndexType(indexType.type);
+        cis.setGramLength(indexType.gramLength);
+      }
+      cis.setEnforced(enforced);
+      return cis;
+    }
+}
+
+RecordTypeDefinition RecordTypeDef() throws ParseException:
+{
+  RecordTypeDefinition recType = new RecordTypeDefinition();
+  RecordTypeDefinition.RecordKind recordKind = null;
+}
+{
+  ( <CLOSED>{ recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
+    | <OPEN>{ recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
+   <LEFTBRACE>
+    {
+      String hint = getHint(token);
+      if (hint != null) {
+        String splits[] = hint.split(" +");
+        if (splits[0].equals(GEN_FIELDS_HINT)) {
+          if (splits.length != 5) {
+            throw new ParseException("Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
+          }
+          if (!splits[1].equals("int")) {
+            throw new ParseException("The only supported type for gen-fields is int.");
+          }
+          UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
+             Integer.parseInt(splits[2]), Integer.parseInt(splits[3]), splits[4]);
+          recType.setUndeclaredFieldsDataGen(ufdg);
+        }
+      }
+
+    }
+        (
+          RecordField(recType)
+          ( <COMMA>  RecordField(recType) )*
+        )?
+   <RIGHTBRACE>
+   {
+      if (recordKind == null) {
+        recordKind = RecordTypeDefinition.RecordKind.OPEN;
+      }
+      recType.setRecordKind(recordKind);
+      return recType;
+   }
+}
+
+UnorderedListTypeDefinition UnorderedListTypeDef() throws ParseException:
+{
+  TypeExpression type = null;
+}
+{
+  <LEFTDBLBRACE>
+    ( type =  TypeExpr() )
+  <RIGHTDBLBRACE>
+  {
+    return new UnorderedListTypeDefinition(type);
+  }
+}
+
+TypeExpression IndexedTypeExpr() throws ParseException:
+{
+  TypeExpression typeExpr = null;
+}
+{
+  (
+      typeExpr = TypeReference()
+    | typeExpr = OrderedListTypeDef()
+    | typeExpr = UnorderedListTypeDef()
+  )
+  {
+    return typeExpr;
+  }
+}
+
+NodegroupDecl NodegroupSpecification() throws ParseException:
+{
+  String name = null;
+  String tmp = null;
+  boolean ifNotExists = false;
+  List<Identifier>ncNames = null;
+}
+{
+  <NODEGROUP> name = Identifier()
+  ifNotExists = IfNotExists() <ON> tmp = Identifier()
+    {
+      ncNames = new ArrayList<Identifier>();
+      ncNames.add(new Identifier(tmp));
+    }
+  ( <COMMA> tmp = Identifier()
+    {
+      ncNames.add(new Identifier(tmp));
+    }
+  )*
+    {
+      return new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
+    }
+}
+
+Pair<Identifier,Identifier>TypeName() throws ParseException:
+{
+  Pair<Identifier,Identifier> name = null;
+}
+{
+  name = QualifiedName()
+    {
+      if (name.first == null) {
+        name.first = new Identifier(defaultDataverse);
+      }
+      return name;
+    }
+}
+
+Pair<List<Integer>, List<List<String>>>PrimaryKey() throws ParseException:
+{
+   Pair<Integer, List<String>> tmp = null;
+   List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
+   List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
+}
+{
+  <PRIMARY> <KEY> tmp = NestedField()
+    {
+      keyFieldSourceIndicators.add(tmp.first);
+      primaryKeyFields.add(tmp.second);
+    }
+  ( <COMMA> tmp = NestedField()
+    {
+      keyFieldSourceIndicators.add(tmp.first);
+      primaryKeyFields.add(tmp.second);
+    }
+  )*
+    {
+      return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
+    }
+}
+
+TypeExpression TypeExpr() throws ParseException:
+{
+  TypeExpression typeExpr = null;
+}
+{
+  (
+      typeExpr = RecordTypeDef()
+    | typeExpr = TypeReference()
+    | typeExpr = OrderedListTypeDef()
+    | typeExpr = UnorderedListTypeDef()
+  )
+  {
+    return typeExpr;
+  }
+}
+
+Map<String,String>Configuration() throws ParseException:
+{
+    Map<String,String> configuration = new LinkedHashMap<String,String>();
+    Pair<String, String> keyValuePair = null;
+}
+{
+  <LEFTPAREN> ( keyValuePair = KeyValuePair()
+    {
+      configuration.put(keyValuePair.first, keyValuePair.second);
+    }
+  ( <COMMA> keyValuePair = KeyValuePair()
+    {
+      configuration.put(keyValuePair.first, keyValuePair.second);
+    }
+  )* )? <RIGHTPAREN>
+    {
+      return configuration;
+    }
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN [IGNORE_CASE] :
+{
+    <BROKER : "broker">
+  | <CHANNEL : "channel">
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <APPLY : "apply">
+  | <AS : "as">
+  | <ASC : "asc">
+  | <AT : "at">
+  | <AUTOGENERATED : "autogenerated">
+  | <BTREE : "btree">
+  | <BY : "by">
+  | <CLOSED : "closed">
+  | <COMPACT : "compact">
+  | <COMPACTION : "compaction">
+  | <CONNECT : "connect">
+  | <CREATE : "create">
+  | <DATASET : "dataset">
+  | <DATAVERSE : "dataverse">
+  | <DECLARE : "declare">
+  | <DECOR : "decor">
+  | <DEFINITION : "definition">
+  | <DELETE : "delete">
+  | <DESC : "desc">
+  | <DISCONNECT : "disconnect">
+  | <DISTINCT : "distinct">
+  | <DROP : "drop">
+  | <ELSE : "else">
+  | <ENFORCED : "enforced">
+  | <EVERY : "every">
+  | <EXISTS : "exists">
+  | <EXTERNAL : "external">
+  | <FEED : "feed">
+  | <FILTER : "filter">
+  | <FOR : "for">
+  | <FORMAT : "format">
+  | <FROM : "from">
+  | <FUNCTION : "function">
+  | <GROUP : "group">
+  | <HINTS : "hints">
+  | <IF : "if">
+  | <IN : "in">
+  | <INDEX : "index">
+  | <INGESTION : "ingestion">
+  | <INSERT : "insert">
+  | <INTERNAL : "internal">
+  | <INTO : "into">
+  | <KEY : "key">
+  | <KEYWORD : "keyword">
+  | <KEEPING : "keeping">
+  | <LET : "let">
+  | <LIMIT : "limit">
+  | <LOAD : "load">
+  | <NGRAM : "ngram">
+  | <NODEGROUP : "nodegroup">
+  | <OFFSET : "offset">
+  | <ON : "on">
+  | <OPEN : "open">
+  | <ORDER : "order">
+  | <OUTPUT : "output">
+  | <PATH : "path">
+  | <POLICY : "policy">
+  | <PRESORTED : "pre-sorted">
+  | <PRIMARY : "primary">
+  | <REFRESH : "refresh">
+  | <RETURN : "return">
+  | <RTREE : "rtree">
+  | <RUN : "run">
+  | <SATISFIES : "satisfies">
+  | <SECONDARY : "secondary">
+  | <SELECT : "select">
+  | <SET : "set">
+  | <SOME : "some">
+  | <TEMPORARY : "temporary">
+  | <THEN : "then">
+  | <TO : "to">
+  | <TYPE : "type">
+  | <UNION : "union">
+  | <UPDATE : "update">
+  | <UPSERT : "upsert">
+  | <USE : "use">
+  | <USING : "using">
+  | <WHERE : "where">
+  | <WITH : "with">
+  | <WRITE : "write">
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <CARET : "^">
+  | <DIV : "/">
+  | <IDIV : "idiv">
+  | <MINUS : "-">
+  | <MOD : "%">
+  | <MUL : "*">
+  | <PLUS : "+">
+
+  | <LEFTPAREN : "(">
+  | <RIGHTPAREN : ")">
+  | <LEFTBRACKET : "[">
+  | <RIGHTBRACKET : "]">
+
+  | <COLON : ":">
+  | <COMMA : ",">
+  | <DOT : ".">
+  | <QUES : "?">
+
+  | <LT : "<">
+  | <GT : ">">
+  | <LE : "<=">
+  | <GE : ">=">
+  | <EQ : "=">
+  | <NE : "!=">
+  | <SIMILAR : "~=">
+  | <ASSIGN : ":=">
+
+  | <AND : "and">
+  | <OR : "or">
+
+  | <SYMBOLAT : "@">
+  | <SYMBOLHASH : "#">
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <LEFTBRACE : "{"> { pushState(); } : DEFAULT
+}
+< DEFAULT >
+ TOKEN :
+{
+    <RIGHTBRACE : "}"> { popState("}"); }
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
+}
+< IN_DBL_BRACE >
+ TOKEN :
+{
+    <RIGHTDBLBRACE : "}}"> { popState("}}"); }
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <INTEGER_LITERAL : (<DIGIT>)+ >
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+  < MISSING : "missing">
+  | <NULL : "null">
+  | <TRUE : "true">
+  | <FALSE : "false">
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <#DIGIT : ["0" - "9"]>
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    < DOUBLE_LITERAL: <DIGITS>
+        | <DIGITS> ( "." <DIGITS> )?
+        | "." <DIGITS>
+    >
+  | < FLOAT_LITERAL: <DIGITS> ( "f" | "F" )
+        | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
+        | "." <DIGITS> ( "f" | "F" )
+    >
+  | <DIGITS : (<DIGIT>)+ >
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <#LETTER : ["A" - "Z", "a" - "z"]>
+  | <SPECIALCHARS : ["$", "_", "-"]>
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
+    <STRING_LITERAL : ("\"" (
+          <EscapeQuot>
+        | <EscapeBslash>
+        | <EscapeSlash>
+        | <EscapeBspace>
+        | <EscapeFormf>
+        | <EscapeNl>
+        | <EscapeCr>
+        | <EscapeTab>
+        | ~["\"","\\"])* "\"")
+      | ("\'"(
+          <EscapeApos>
+        | <EscapeBslash>
+        | <EscapeSlash>
+        | <EscapeBspace>
+        | <EscapeFormf>
+        | <EscapeNl>
+        | <EscapeCr>
+        | <EscapeTab>
+        | ~["\'","\\"])* "\'")>
+  | < #EscapeQuot: "\\\"" >
+  | < #EscapeApos: "\\\'" >
+  | < #EscapeBslash: "\\\\" >
+  | < #EscapeSlash: "\\/" >
+  | < #EscapeBspace: "\\b" >
+  | < #EscapeFormf: "\\f" >
+  | < #EscapeNl: "\\n" >
+  | < #EscapeCr: "\\r" >
+  | < #EscapeTab: "\\t" >
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <IDENTIFIER : <LETTER> (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
+}
+< DEFAULT,IN_DBL_BRACE >
+ TOKEN :
+{
+    <VARIABLE : "$" <LETTER> (<LETTER> | <DIGIT> | "_")*>
+}
+< DEFAULT,IN_DBL_BRACE >
+ SKIP :
+{
+    " "
+  | "\t"
+  | "\r"
+  | "\n"
+}
+< DEFAULT,IN_DBL_BRACE >
+ SKIP :
+{
+    <"//" (~["\n"])* "\n">
+}
+< DEFAULT,IN_DBL_BRACE >
+ SKIP :
+{
+    <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
+}
+< DEFAULT,IN_DBL_BRACE >
+ SKIP :
+{
+    <"/*"> { pushState(); } : INSIDE_COMMENT
+}
+< INSIDE_COMMENT >
+ SPECIAL_TOKEN :
+{
+    <"+"(" ")*(~["*"])*>
+}
+< INSIDE_COMMENT >
+ SKIP :
+{
+    <"/*"> { pushState(); }
+}
+< INSIDE_COMMENT >
+ SKIP :
+{
+    <"*/"> { popState("*/"); }
+  | <~[]>
+}


Mime
View raw message