asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wangs...@apache.org
Subject asterixdb git commit: Add some functionalities to Grammar Extension plugin
Date Thu, 12 Jan 2017 21:49:48 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 599ef8fb2 -> 456ff9ded


Add some functionalities to Grammar Extension plugin

 - Can add a Java method at the end of the class definition
 - Extend @merge replace "oldphrase" with "newphrase"
 - Can add a construct at the end of the file
 - Can deal with class modifier

Change-Id: Icafc01ebe591b81b3c3f69ea9da123dbfad19bb0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1431
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/456ff9de
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/456ff9de
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/456ff9de

Branch: refs/heads/master
Commit: 456ff9ded50f2d6ed40ec9ba537fcd847810b584
Parents: 599ef8f
Author: Taewoo Kim <wangsaeu@yahoo.com>
Authored: Wed Jan 11 14:16:16 2017 -0800
Committer: Taewoo Kim <wangsaeu@yahoo.com>
Committed: Thu Jan 12 13:49:16 2017 -0800

----------------------------------------------------------------------
 .../extension/grammar/GrammarExtensionMojo.java | 187 +++++++++++++++++--
 .../src/test/resources/lang/extension.jj        |  55 +++++-
 .../basic-test/basic-test-plugin-config.xml     |   1 +
 3 files changed, 219 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
index 5b44e23..4c750e2 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
@@ -68,16 +68,23 @@ public class GrammarExtensionMojo extends AbstractMojo {
     private static final String KWUNIMPORT = "unimport";
     private static final String KWPACKAGE = "package";
     private static final String NEWPRODUCTION = "@new";
+    // Adds a construct at the end of the file.
+    private static final String NEW_AT_THE_END_PRODUCTION = "@new_at_the_end";
+    // Adds a method at the end of the class definition.
+    private static final String NEW_AT_THE_END_CLASS_DEFINITION = "@new_at_the_class_def";
     private static final String MERGEPRODUCTION = "@merge";
     private static final String OVERRIDEPRODUCTION = "@override";
     private static final String BEFORE = "before:";
     private static final String AFTER = "after:";
     private static final String REPLACE = "replace";
     private static final String WITH = "with";
+    private static final String OPTION_TRUE = "true";
+    private static final String OPTION_FALSE = "false";
     private static final List<String> KEYWORDS = Arrays
             .asList(new String[] { KWCLASS, KWIMPORT, KWPACKAGE, PARSER_BEGIN, PARSER_END
});
-    private static final List<String> EXTENSIONKEYWORDS = Arrays
-            .asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION, OVERRIDEPRODUCTION,
MERGEPRODUCTION });
+    private static final List<String> EXTENSIONKEYWORDS =
+            Arrays.asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION, NEW_AT_THE_END_PRODUCTION,
+                    NEW_AT_THE_END_CLASS_DEFINITION, OVERRIDEPRODUCTION, MERGEPRODUCTION
});
     private static final String REGEX_WS_DOT_SEMICOLON = "\\s|[.]|[;]";
     private static final String REGEX_WS_PAREN = "\\s|[(]|[)]";
     private static final String OPTIONS = "options";
@@ -87,14 +94,20 @@ public class GrammarExtensionMojo extends AbstractMojo {
     private Map<String, String[]> mergeElements = new HashMap<>();
     private List<Pair<String, String>> baseFinals = new ArrayList<>();
     private List<Pair<String, String>> extensionFinals = new ArrayList<>();
+    private List<Pair<String, String>> extensionFinalsAtTheEnd = new ArrayList<>();
+    private List<Pair<String, String>> extensionMethodsAtTheClassDef = new ArrayList<>();
     private List<List<String>> imports = new ArrayList<>();
     private String baseClassName;
     private String baseClassDef;
     private String optionsBlock;
     private boolean read = false;
     private boolean shouldReplace = false;
-    private String oldWord = null;
-    private String newWord = null;
+
+    // Used in @merge replace. If set to true, applies each block's changes and stores them.
+    private boolean shouldApplyEachBlockChange = true;
+
+    private String oldPhrase = null;
+    private String newPhrase = null;
 
     @Parameter(property = "grammarix.base")
     private String base;
@@ -115,12 +128,16 @@ public class GrammarExtensionMojo extends AbstractMojo {
     private String parserClassName;
     private String lastIdentifier;
 
+    @Parameter(property = "grammarix.parserClassModifier")
+    private String parserClassModifier;
+
     @Override
     public void execute() throws MojoExecutionException {
         base = new File(base).getAbsolutePath();
         getLog().info("Base dir: " + base);
         getLog().info("Grammar-base: " + gbase);
         getLog().info("Grammar-extension: " + gextension);
+        getLog().info("Output: " + output);
         processBase();
         processExtension();
         generateOutput();
@@ -166,7 +183,29 @@ public class GrammarExtensionMojo extends AbstractMojo {
             writer.newLine();
 
             // Class definition
-            writer.write(baseClassDef.replaceAll(baseClassName, parserClassName));
+            String classDef = baseClassDef.replaceAll(baseClassName, parserClassName);
+            if (parserClassModifier != null && parserClassModifier.length() >
0) {
+                // Adds a class modifier if any.
+                classDef = classDef.replaceFirst(KWCLASS, parserClassModifier + " " + KWCLASS);
+            }
+
+            // Process extensions at the end of the class definition
+            if (!extensionMethodsAtTheClassDef.isEmpty()) {
+                int index = classDef.lastIndexOf(CLOSE_BRACE);
+                if (index != -1) {
+                    String classDefExtension = "";
+                    for (Pair<String, String> element : extensionMethodsAtTheClassDef)
{
+                        classDefExtension += toOutput(element.first);
+                        classDefExtension += "\n";
+                        classDefExtension += element.second;
+                        classDefExtension += "\n";
+                    }
+                    classDef =
+                            classDef.substring(0, index) + "\n" + classDefExtension + "\n"
+ classDef.substring(index);
+                }
+            }
+
+            writer.write(classDef);
             writer.newLine();
 
             // Parser End
@@ -213,6 +252,13 @@ public class GrammarExtensionMojo extends AbstractMojo {
                 writer.newLine();
             }
 
+            for (Pair<String, String> element : extensionFinalsAtTheEnd) {
+                writer.write(toOutput(element.first));
+                writer.newLine();
+                writer.write(element.second);
+                writer.newLine();
+            }
+
         } catch (Exception e) {
             getLog().error(e);
             throw new MojoExecutionException(e.getMessage(), e);
@@ -671,9 +717,15 @@ public class GrammarExtensionMojo extends AbstractMojo {
                         case NEWPRODUCTION:
                             nextOperation = NEWPRODUCTION;
                             break;
+                        case NEW_AT_THE_END_PRODUCTION:
+                            nextOperation = NEW_AT_THE_END_PRODUCTION;
+                            break;
+                        case NEW_AT_THE_END_CLASS_DEFINITION:
+                            nextOperation = NEW_AT_THE_END_CLASS_DEFINITION;
+                            break;
                         case MERGEPRODUCTION:
                             nextOperation = MERGEPRODUCTION;
-                            shouldReplace = shouldReplace(tokens);
+                            shouldReplace = shouldReplace(tokens, position.line);
                             break;
                         case OVERRIDEPRODUCTION:
                             nextOperation = OVERRIDEPRODUCTION;
@@ -693,6 +745,10 @@ public class GrammarExtensionMojo extends AbstractMojo {
                         case NEWPRODUCTION:
                             handleNew(identifier, reader);
                             break;
+                        case NEW_AT_THE_END_CLASS_DEFINITION:
+                            readFinalProduction(identifier, reader);
+                            addFinalProduction(identifier, extensionMethodsAtTheClassDef);
+                            break;
                         case OVERRIDEPRODUCTION:
                             handleOverride(identifier, reader);
                             break;
@@ -704,12 +760,16 @@ public class GrammarExtensionMojo extends AbstractMojo {
                     }
                     nextOperation = NEWPRODUCTION;
                 } else if (openAngularIndex == 0) {
-                    if (nextOperation != NEWPRODUCTION) {
+                    if (nextOperation != NEWPRODUCTION && nextOperation != NEW_AT_THE_END_PRODUCTION)
{
                         throw new MojoExecutionException("Can only add new REGEX production
kind");
                     }
                     position.index = position.line.indexOf(OPEN_ANGULAR);
                     readFinalProduction(identifier, reader);
-                    addFinalProduction(identifier, extensionFinals);
+                    if (nextOperation == NEWPRODUCTION) {
+                        addFinalProduction(identifier, extensionFinals);
+                    } else if (nextOperation == NEW_AT_THE_END_PRODUCTION) {
+                        addFinalProduction(identifier, extensionFinalsAtTheEnd);
+                    }
                 } else if (identifier.length() > 0 || position.line.trim().length() >
0) {
                     identifier.append(position.line);
                     identifier.append('\n');
@@ -721,16 +781,64 @@ public class GrammarExtensionMojo extends AbstractMojo {
         }
     }
 
-    private boolean shouldReplace(String[] tokens) throws MojoExecutionException {
+    private boolean shouldReplace(String[] tokens, String currentLine) throws MojoExecutionException
{
         boolean replace = false;
-        if (tokens.length == 5) {
-            if (tokens[1].equals(REPLACE) && tokens[3].equals(WITH)) {
-                shouldReplace = true;
-                oldWord = tokens[2];
-                newWord = tokens[4];
+        String errMessage = "Allowed syntax after @merge: <REPLACE> \"oldPhrase\" <WITH>
\"newPhrase\" <TRUE|FALSE>.";
+        String errMessage1 = "The old phrase should be place between two quotes. (E.g., \"old\")";
+        String errMessage2 = "The new phrase should be place between two quotes. (E.g., \"new\")";
+        // @merge replace "oldphrase" with "newphrase" proceedBlock:true/false
+        if (tokens.length >= 6) {
+            // Checks whether "replace" exists.
+            if (!tokens[1].equalsIgnoreCase(REPLACE)) {
+                throw new MojoExecutionException(errMessage);
+            }
+
+            // Checks whether "with" exists.
+            boolean withFound = false;
+            for (int i = 3; i < tokens.length; i++) {
+                if (tokens[i].equalsIgnoreCase(WITH)) {
+                    withFound = true;
+                    break;
+                }
+            }
+            if (!withFound) {
+                throw new MojoExecutionException(errMessage);
+            }
+
+            // Check whether the last parameter is true/false.
+            // If this is true, then we check all blocks and process before: and after:.
+            // If not, we don't process all blocks and replace "old" with "new" for all instances
for
+            // the entire part of the given method.
+            if (tokens[tokens.length - 1].equalsIgnoreCase(OPTION_TRUE)) {
+                shouldApplyEachBlockChange = true;
+            } else if (tokens[tokens.length - 1].equalsIgnoreCase(OPTION_FALSE)) {
+                shouldApplyEachBlockChange = false;
             } else {
-                throw new MojoExecutionException("Allowed syntax after @merge: <REPLACE>
oldWord <WITH> newWord");
+                throw new MojoExecutionException(errMessage);
+            }
+
+            // Gets the old phrase.
+            int oldStart = findQuotePos(currentLine, 0);
+            if (oldStart < 0) {
+                throw new MojoExecutionException(errMessage1);
+            }
+            int oldEnd = findQuotePos(currentLine, oldStart + 1);
+            if (oldEnd < 0) {
+                throw new MojoExecutionException(errMessage1);
+            }
+            oldPhrase = currentLine.substring(oldStart + 1, oldEnd);
+
+            // Gets the new phrase.
+            int newStart = findQuotePos(currentLine, oldEnd + 1);
+            if (newStart < 0) {
+                throw new MojoExecutionException(errMessage2);
+            }
+            int newEnd = findQuotePos(currentLine, newStart + 1);
+            if (newEnd < 0) {
+                throw new MojoExecutionException(errMessage2);
             }
+            newPhrase = currentLine.substring(newStart + 1, newEnd);
+            replace = true;
         }
         return replace;
     }
@@ -776,26 +884,37 @@ public class GrammarExtensionMojo extends AbstractMojo {
             throw new MojoExecutionException(identifier.toString() + " doesn't exist in base
grammar");
         } else if (shouldReplace) {
             Pair<String, String> baseMethods = extensibles.get(sig);
-            baseMethods.first = baseMethods.first.replaceAll(oldWord, newWord);
-            baseMethods.second = baseMethods.second.replaceAll(oldWord, newWord);
+            // Literally replaces the old phrase with the new phrase.
+            baseMethods.first = stringReplaceAll(baseMethods.first, oldPhrase, newPhrase);
+            baseMethods.second = stringReplaceAll(baseMethods.second, oldPhrase, newPhrase);
             shouldReplace = false;
         }
         String[] amendments = new String[6];
-        mergeElements.put(sig, amendments);
+        if (shouldApplyEachBlockChange) {
+            // Applies and stores each block's change only if shouldApplyEachBlockChange
is set to true.
+            mergeElements.put(sig, amendments);
+        } else {
+            // If shouldApplyEachBlockChange is false, no result from each block's change
are not stored.
+            shouldApplyEachBlockChange = true;
+        }
         // we don't need the identifier anymore
         identifier.setLength(0);
+        // The first block
         readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
         String block = record.toString();
         extractBeforeAndAfter(block, amendments, 0, 1);
         record.reset();
         position.index = 0;
         position.line = reader.readLine();
+        // Skips empty lines.
         while (position.line != null && position.line.trim().length() == 0) {
             position.line = reader.readLine();
         }
+        // The second block
         int openBraceIndex = position.line.indexOf(OPEN_BRACE);
         if (openBraceIndex > -1) {
             position.index = openBraceIndex;
+            // Reads the entire part of the second block - two OPEN_BRACE and two CLOSE_BRACE.
             readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
         } else {
             throw new MojoExecutionException("merge element doesn't have a second block");
@@ -917,4 +1036,36 @@ public class GrammarExtensionMojo extends AbstractMojo {
         }
         return aString.toString();
     }
+
+    /**
+     * Literally replaces the given string.
+     */
+    private String stringReplaceAll(String baseStr, String oldPhrase, String newPhrase) {
+        String resultStr = baseStr;
+        String tempStr;
+        int index = resultStr.indexOf(oldPhrase);
+        while (index < resultStr.length()) {
+            if (index < 0) {
+                return resultStr;
+            }
+            tempStr = resultStr.substring(index, index + oldPhrase.length());
+            if (tempStr.equals(oldPhrase)) {
+                resultStr = resultStr.substring(0, index) + newPhrase + resultStr.substring(index
+ oldPhrase.length());
+            }
+            index = resultStr.indexOf(oldPhrase, index + 1);
+        }
+        return resultStr;
+    }
+
+    /**
+     * Finds and returns the first occurrence index of a quote from startingIndex.
+     *
+     * @param baseStr
+     * @param startingIndex
+     * @return the substring if found. If not, this returns null.
+     */
+    private int findQuotePos(String baseStr, int startingIndex) {
+        return baseStr.indexOf(ExternalDataConstants.QUOTE, startingIndex);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
index fb143a3..63bd38a 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
@@ -1,14 +1,29 @@
+// If you want to put an additional import, just add import statements like the following.
+// import package name
 import org.apache.asterix.lang.extension.EchoStatement;
-// to remove an import, we use the keyword, unimport
+
+// To remove an import, we use the keyword, unimport
 // unimport package name
 
+// If you want to add a method in the class definition (before PARSER_END), use the following
phrase and attach
+// new method right after the phrase.
+// @new_at_the_end
+@new_at_the_class_def
+    public void initScope() {
+            scopeStack.push(RootScopeFactory.createRootScope(this));
+    }
+
 // Merging of non-terminals can only be done on non-terminals which conform to the following
structure.
 // Content will simply be prepended or appended to the base blocks.
 // Note: refrain from using the strings "before:" and "after:" in the merge areas as that
will break the merge.
 // As a workaround, you can always override
-// one additional possible change is direct replacement and it can be done through
-// @merge replace "baseWord" with "extensionWord"
-
+// one additional possible change is direct replacement and it can be done through the followin
syntax:
+// @merge replace "base phrase" with "extension phrase" true/false
+// Here, true/false tells whether the tool needs to process the three blocks.
+// If true, like normal @merge case, before and after clause in each block will be processed
+// after "base phrase" in the blocks have been replaced with "new phrase".
+// If false, then it just expects the blank form that consists of three blocks and not process
them.
+// Only, "base phrase" in the blocks will be replaced with "new phrase".
 @merge
 Statement SingleStatement() throws ParseException:
 {
@@ -26,6 +41,24 @@ Statement SingleStatement() throws ParseException:
   }
 }
 
+// In the following case, all instances of the first phrase inside of "" will be replaced
with the second phrase in "".
+// Also, we don't check "before:" and "after:" section of each area. That check will be ignored
since
+// the last parameter is set to false.
+@merge replace "nameComponents = QualifiedName() (<AS> var = Variable())?" with "nameComponents
  =   QualifiedName()  (<AS> var = Variable())? " false
+InsertStatement InsertStatement() throws ParseException:
+{
+  // merge area 1
+}
+{
+    (
+    // merge area 2
+    )
+    {
+      // merge area 3
+    }
+}
+
+
 // The default
 // Adding a new node. if a node exists, it will throw an exception.
 @new
@@ -40,11 +73,21 @@ Statement EchoStatement() throws ParseException:
     }
 }
 
+// Overriding a non-terminal. if exists in base, it will be overriden, otherwise, it will
be added
+// @override
+
+
+// Terminals can be added like the following.
 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <ECHO : "echo">
 }
 
-// Overriding a non-terminal. if exists in base, it will be overriden, otherwise, it will
be added
-// @override
+// If something needs to be added at the end of file, you can use @new_at_the_end like the
following.
+@new_at_the_end
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+    <METAVARIABLE : "$$" <IDENTIFIER> >
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
index 5475fa4..3f929c1 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
@@ -41,6 +41,7 @@
           <gextension>src/test/resources/lang/extension.jj</gextension>
           <output>target/generated-sources/lang/grammar.jj</output>
           <parserClassName>ExtendedParser</parserClassName>
+          <parserClassModifier></parserClassModifier>
           <packageName>org.apache.asterix.lang.extension.parser</packageName>
         </configuration>
         <executions>


Mime
View raw message