freemarker-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ddek...@apache.org
Subject incubator-freemarker git commit: Continued work on the FM2 to FM3 converter
Date Fri, 30 Jun 2017 23:23:45 GMT
Repository: incubator-freemarker
Updated Branches:
  refs/heads/3 c5feb6328 -> 507b89bf5


Continued work on the FM2 to FM3 converter


Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/507b89bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/507b89bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/507b89bf

Branch: refs/heads/3
Commit: 507b89bf544a9dc35506ae40fa526dd137b59d58
Parents: c5feb63
Author: ddekany <ddekany@apache.org>
Authored: Sat Jul 1 01:23:31 2017 +0200
Committer: ddekany <ddekany@apache.org>
Committed: Sat Jul 1 01:23:31 2017 +0200

----------------------------------------------------------------------
 .../core/FM2ASTToFM3SourceConverter.java        | 209 +++++++++++++++++--
 .../core/UnexpectedNodeContentException.java    |  17 +-
 .../converter/FM2ToFM3ConverterTest.java        |  30 +++
 3 files changed, 234 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/507b89bf/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
index 13a80ac..b88f9f2 100644
--- a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
+++ b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
@@ -35,6 +35,7 @@ import org.apache.freemarker.core.util._StringUtil;
 
 import freemarker.template.Configuration;
 import freemarker.template.Template;
+import freemarker.template.utility.StringUtil;
 
 /**
  * Takes a FreeMarker 2 AST, and converts it to an FreeMarker 3 source code.
@@ -154,7 +155,7 @@ public class FM2ASTToFM3SourceConverter {
         int pos = getPositionAfterWSAndExpComments(getEndPositionExclusive(content));
         assertNodeContent(pos < src.length(), node, "Unexpected EOF", null);
         char c = src.charAt(pos);
-        assertNodeContent(c == ';' || c == '}', node, "Expected ';' or '}', found '{}'",
c);
+        assertNodeContent(c == ';' || c == '}', node, "Expected ';' or '}', found {}", c);
         if (c == ';') { // #{exp; m1M2} -> ${exp?string('0.0#')}
             int minDecimals = getParam(node, 1, ParameterRole.MINIMUM_DECIMALS, Integer.class);
             int maxDecimals = getParam(node, 2, ParameterRole.MAXIMUM_DECIMALS, Integer.class);
@@ -213,11 +214,143 @@ public class FM2ASTToFM3SourceConverter {
             printDirIfOrElseOrElseIf((ConditionalBlock) node);
         } else if (node instanceof UnifiedCall) {
             printDirCustom((UnifiedCall) node);
+        } else if (node instanceof Macro) {
+            printDirMacroOrFunction((Macro) node);
         } else {
             throw new ConverterException("Unhandled AST TemplateElement class: " + node.getClass().getName());
         }
     }
 
+    private void printDirMacroOrFunction(Macro node) throws ConverterException {
+        int paramCnt = node.getParameterCount();
+
+        int subtype = getParam(node, paramCnt - 1, ParameterRole.AST_NODE_SUBTYPE, Integer.class);
+        String tagName;
+        if (subtype == Macro.TYPE_MACRO) {
+            tagName = "#macro";
+        } else if (subtype == Macro.TYPE_FUNCTION) {
+            tagName = "#function";
+        } else {
+            throw new UnexpectedNodeContentException(node, "Unhandled node subtype: {}",
subtype);
+        }
+
+        print(tagBeginChar);
+        print(tagName);
+        int pos = getStartPosition(node) + 1;
+        assertNodeContent(src.substring(pos, pos + tagName.length()).equals(tagName), node,
+                "Tag name doesn't match {}", tagName);
+        pos += tagName.length();
+
+        {
+            String sep = readWSAndExpComments(pos);
+            printWithConvertedExpComments(sep);
+            pos += sep.length();
+        }
+
+        String assignedName = getParam(node, 0, ParameterRole.ASSIGNMENT_TARGET, String.class);
+        print(FTLUtil.escapeIdentifier(assignedName));
+        {
+            int lastPos = pos;
+            pos = getPositionAfterIdentifier(pos);
+            assertNodeContent(pos > lastPos, node, "Expected target name", null);
+        }
+
+        {
+            String sep = readWSAndExpComments(pos, '(', true);
+            printWithConvertedExpComments(sep);
+            pos += sep.length();
+        }
+
+        int paramIdx = 1;
+        while (node.getParameterRole(paramIdx) == ParameterRole.PARAMETER_NAME) {
+            String paramName = getParam(node, paramIdx++, ParameterRole.PARAMETER_NAME, String.class);
+            print(FTLUtil.escapeIdentifier(paramName));
+            {
+                int lastPos = pos;
+                pos = getPositionAfterIdentifier(pos);
+                assertNodeContent(pos > lastPos, node, "Expected parameter name", null);
+            }
+
+            Expression paramDefault = getParam(node, paramIdx++, ParameterRole.PARAMETER_DEFAULT,
Expression.class);
+            if (paramDefault != null) {
+                String sep = readWSAndExpComments(pos, '=', false);
+                printWithConvertedExpComments(sep);
+                printExp(paramDefault);
+                pos = getEndPositionExclusive(paramDefault);
+            }
+
+            {
+                String sep = readWSAndExpComments(pos);
+                printWithConvertedExpComments(sep);
+                pos += sep.length();
+            }
+            {
+                char c = src.charAt(pos);
+                assertNodeContent(
+                        c == ',' || c == ')' || c == tagEndChar
+                                || c == '\\' || StringUtil.isFTLIdentifierStart(c),
+                        node,
+                        "Unexpected character: {}", c);
+                if (c == ',') {
+                    print(c);
+                    pos++;
+
+                    String sep = readWSAndExpComments(pos);
+                    printWithConvertedExpComments(sep);
+                    pos += sep.length();
+                }
+                if (c == ')') {
+                    assertNodeContent(node.getParameterRole(paramIdx) != ParameterRole.PARAMETER_NAME,
node,
+                            "Expected no parameter after \"(\"", null);
+                }
+            }
+        }
+
+        {
+            ParameterRole parameterRole = node.getParameterRole(paramIdx);
+            assertNodeContent(parameterRole == ParameterRole.CATCH_ALL_PARAMETER_NAME, node,
+                    "Expected catch-all parameter role, but found {}", parameterRole);
+        }
+        String paramName = getParam(node, paramIdx++, ParameterRole.CATCH_ALL_PARAMETER_NAME,
String.class);
+        if (paramName != null) {
+            print(FTLUtil.escapeIdentifier(paramName));
+            {
+                int lastPos = pos;
+                pos = getPositionAfterIdentifier(pos);
+                assertNodeContent(pos > lastPos, node,
+                        "Expected catch-all parameter name", null);
+            }
+            {
+                String sep = readWSAndExpComments(pos);
+                printWithConvertedExpComments(sep);
+                pos += sep.length();
+            }
+            assertNodeContent(src.startsWith("...", pos), node,
+                    "Expected \"...\" after catch-all parameter name", null);
+            print("...");
+            pos += 3;
+        }
+
+        assertNodeContent(paramIdx == paramCnt - 1, node,
+                "Expected AST parameter at index {} to be the last one", paramIdx);
+
+        {
+            String sep = readWSAndExpComments(pos, ')', true);
+            printWithConvertedExpComments(sep);
+            pos += sep.length();
+        }
+        assertNodeContent(src.charAt(pos) == tagEndChar, node, "Tag end not found", null);
+        print(tagEndChar);
+
+        printChildrenElements(node);
+
+        print(tagBeginChar);
+        print('/');
+        print(tagName);
+        printEndTagSkippedTokens(node);
+        print(tagEndChar);
+    }
+
     private void printDirCustom(UnifiedCall node) throws ConverterException {
         print(tagBeginChar);
         print('@');
@@ -256,7 +389,7 @@ public class FM2ASTToFM3SourceConverter {
         int pos = getEndPositionExclusive(lastPrintedExp);
         boolean beforeFirstLoopVar = true;
         while (paramIdx < paramCount) {
-            String sep = readExpWSAndSeparator(pos, beforeFirstLoopVar ? ';' : ',', false);
+            String sep = readWSAndExpComments(pos, beforeFirstLoopVar ? ';' : ',', false);
             assertNodeContent(sep.length() != 0, node,
                     "Can't find loop variable separator", null);
             printWithConvertedExpComments(sep);
@@ -280,7 +413,7 @@ public class FM2ASTToFM3SourceConverter {
         {
             char c = src.charAt(elementEndPos);
             assertNodeContent(c == tagEndChar, node,
-                    "tagEndChar expected, found '{}'", c);
+                    "tagEndChar expected, found {}", c);
         }
         if (startTagEndPos != elementEndPos) { // We have an end-tag
             assertNodeContent(src.charAt(startTagEndPos - 1) != '/', node,
@@ -392,14 +525,49 @@ public class FM2ASTToFM3SourceConverter {
             printExpOr((OrExpression) node);
         } else if (node instanceof NotExpression) {
             printExpNot((NotExpression) node);
+        } else if (node instanceof DefaultToExpression) {
+            printExpDefault((DefaultToExpression) node);
+        } else if (node instanceof ExistsExpression) {
+            printExpExists((ExistsExpression) node);
         } else {
             throw new ConverterException("Unhandled AST node expression class: " + node.getClass().getName());
         }
     }
 
+    private void printPostfixOperator(Expression node, String operator) throws ConverterException
{
+        Expression lho = getParam(node, 0, ParameterRole.LEFT_HAND_OPERAND, Expression.class);
+        printExp(lho);
+
+        int wsStartPos = getEndPositionExclusive(lho);
+        int opPos = getPositionAfterWSAndExpComments(wsStartPos);
+        printWithConvertedExpComments(src.substring(wsStartPos, opPos));
+        String operatorInSrc = src.substring(opPos, opPos + operator.length());
+        assertNodeContent(operatorInSrc.equals(operator), node,
+                "Expected " + _StringUtil.jQuote(operator) + ", found {}", operatorInSrc);
+        print(operator);
+    }
+
+    private void printExpExists(ExistsExpression node) throws ConverterException {
+        assertParamCount(node, 1);
+        printPostfixOperator(node, "??");
+    }
+
+    private void printExpDefault(DefaultToExpression node) throws ConverterException {
+        assertParamCount(node, 2);
+        Expression rho = getParam(node, 1, ParameterRole.RIGHT_HAND_OPERAND, Expression.class);
+        if (rho != null) {
+            Expression lho = getParam(node, 0, ParameterRole.LEFT_HAND_OPERAND, Expression.class);
+            printExp(lho);
+            printParameterSeparatorSource(lho, rho);
+            printNode(rho);
+        } else {
+            printPostfixOperator(node, "!");
+        }
+    }
+
     private void printExpNot(NotExpression node) throws ConverterException {
         printWithParamsLeadingSkippedTokens("!", node);
-        printNode(getOnlyParam(node, ParameterRole.RIGHT_HAND_OPERAND, Expression.class));
+        printExp(getOnlyParam(node, ParameterRole.RIGHT_HAND_OPERAND, Expression.class));
     }
 
     private static final Map<String, String> COMPARATOR_OP_MAP;
@@ -481,7 +649,7 @@ public class FM2ASTToFM3SourceConverter {
 
     private void printExpBuiltinVariable(BuiltinVariable node) throws ConverterException
{
         int startPos = getStartPosition(node);
-        String sep = readExpWSAndSeparator(startPos, '.', false);
+        String sep = readWSAndExpComments(startPos, '.', false);
         printWithConvertedExpComments(sep);
         String name = src.substring(startPos + sep.length(), getEndPositionExclusive(node));
         print(convertBuiltInVariableName(name));
@@ -501,7 +669,7 @@ public class FM2ASTToFM3SourceConverter {
         String rho = getParam(node, 1, ParameterRole.RIGHT_HAND_OPERAND, String.class);
         printNode(lho);
         printWithConvertedExpComments(
-                readExpWSAndSeparator(getEndPositionExclusive(lho), '.', false));
+                readWSAndExpComments(getEndPositionExclusive(lho), '.', false));
         print(FTLUtil.escapeIdentifier(rho));
     }
 
@@ -681,7 +849,7 @@ public class FM2ASTToFM3SourceConverter {
                 break scanForRHO;
             } else {
                 throw new UnexpectedNodeContentException(node,
-                        "Unexpected character when scanning for for built-in key: '{}'",
c);
+                        "Unexpected character when scanning for for built-in key: {}", c);
             }
         }
         if (pos == endPos || !foundQuestionMark) {
@@ -833,12 +1001,12 @@ public class FM2ASTToFM3SourceConverter {
             {
                 char c = src.charAt(pos++);
                 assertNodeContent(c == tagBeginChar, node,
-                        "tagBeginChar expected, found '{}'", c);
+                        "tagBeginChar expected, found {}", c);
             }
             {
                 char c = src.charAt(pos++);
                 assertNodeContent(c == '#', node,
-                        "'#' expected, found '{}'", c);
+                        "'#' expected, found {}", c);
             }
             findNameEnd: while (pos < src.length()) {
                 char c = src.charAt(pos);
@@ -878,7 +1046,7 @@ public class FM2ASTToFM3SourceConverter {
             return pos;
         } else {
             throw new UnexpectedNodeContentException(node,
-                    "Unexpected character when scanning for tag end: '{}'", c);
+                    "Unexpected character when scanning for tag end: {}", c);
         }
     }
 
@@ -887,7 +1055,7 @@ public class FM2ASTToFM3SourceConverter {
         {
             char c = src.charAt(tagEndPos);
             assertNodeContent(c == tagEndChar, node,
-                    "tagEndChar expected, found '{}'", c);
+                    "tagEndChar expected, found {}", c);
         }
 
         int pos = tagEndPos - 1;
@@ -1047,8 +1215,12 @@ public class FM2ASTToFM3SourceConverter {
         return pos;
     }
 
+    private String readWSAndExpComments(int startPos)
+            throws ConverterException {
+        return src.substring(startPos, getPositionAfterWSAndExpComments(startPos));
+    }
 
-    private String readExpWSAndSeparator(int startPos, char separator, boolean separatorOptional)
+    private String readWSAndExpComments(int startPos, char separator, boolean separatorOptional)
             throws ConverterException {
         int pos = getPositionAfterWSAndExpComments(startPos);
 
@@ -1063,7 +1235,7 @@ public class FM2ASTToFM3SourceConverter {
         return src.substring(startPos, pos);
     }
 
-    private String readIdentifier(int startPos) throws ConverterException {
+    private int getPositionAfterIdentifier(int startPos) throws ConverterException {
         int pos = startPos;
         scanUntilIdentifierEnd: while (pos < src.length()) {
             char c = src.charAt(pos);
@@ -1071,18 +1243,19 @@ public class FM2ASTToFM3SourceConverter {
                 if (pos + 1 == src.length()) {
                     throw new ConverterException("Misplaced \"\\\" at position " + pos);
                 }
-                if (!FTLUtil.isEscapedIdentifierCharacter(src.charAt(pos + 1))) {
-                    throw new ConverterException("Invalid escape at position " + pos);
-                }
                 pos += 2; // to skip escaped character
             } else if (pos == startPos && FTLUtil.isNonEscapedIdentifierStart(c)
-                    || FTLUtil.isNonEscapedIdentifierPart(c)) {
+                    || StringUtil.isFTLIdentifierPart(c)) {
                 pos++;
             } else {
                 break scanUntilIdentifierEnd;
             }
         }
-        return src.substring(startPos, pos);
+        return pos;
+    }
+
+    private String readIdentifier(int startPos) throws ConverterException {
+        return src.substring(startPos, getPositionAfterIdentifier(startPos));
     }
 
     private String readUntilWSOrComment(int startPos) throws ConverterException {

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/507b89bf/freemarker-converter/src/main/java/freemarker/core/UnexpectedNodeContentException.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/freemarker/core/UnexpectedNodeContentException.java
b/freemarker-converter/src/main/java/freemarker/core/UnexpectedNodeContentException.java
index 8a4b474..c8bfe73 100644
--- a/freemarker-converter/src/main/java/freemarker/core/UnexpectedNodeContentException.java
+++ b/freemarker-converter/src/main/java/freemarker/core/UnexpectedNodeContentException.java
@@ -19,10 +19,8 @@
 
 package freemarker.core;
 
-import java.util.Objects;
-
-import org.apache.freemarker.core.util._StringUtil;
 import org.apache.freemarker.converter.ConverterException;
+import org.apache.freemarker.core.util._StringUtil;
 
 public class UnexpectedNodeContentException extends ConverterException {
     public UnexpectedNodeContentException(TemplateObject node, String errorMessage, Object
msgParam) {
@@ -36,6 +34,17 @@ public class UnexpectedNodeContentException extends ConverterException
{
         if (substIdx == -1) {
             return errorMessage;
         }
-        return errorMessage.substring(0, substIdx) + Objects.toString(msgParam) + errorMessage.substring(substIdx
+ 2);
+        return errorMessage.substring(0, substIdx) + formatParam(msgParam) + errorMessage.substring(substIdx
+ 2);
+    }
+
+    private static String formatParam(Object param) {
+        if (param == null) {
+            return "null";
+        }
+        if (param instanceof String || param instanceof Character) {
+            return _StringUtil.jQuote(param);
+        }
+        return param.toString();
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/507b89bf/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
index 54abfd2..9166e35 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
@@ -124,6 +124,15 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
                 + "${a<#-- C5 -->||<#-- C6 -->b}${a<#-- C7 -->|<#--
C8 -->b}");
 
         assertConvertedSame("${!a}${! foo}${! <#-- C1 --> bar}${!!c}");
+
+        assertConvertedSame("${a!} ${a!0}");
+        assertConvertedSame("${a <#-- C1 --> !} ${a <#-- C2 --> ! <#-- C3
--> 0}");
+        assertConvertedSame("${a!b.c(x!0, y!0)}");
+        assertConvertedSame("${(a.b)!x}");
+        // [FM3] Will be: a!(x+1)
+        assertConvertedSame("${a!x+1}");
+
+        assertConvertedSame("${a??} ${a <#-- C1 --> ??}");
     }
 
     @Test
@@ -155,6 +164,27 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
 
         assertConverted("<#if foo>1<#elseIf bar>2<#else>3</#if>",
"<#if foo>1<#elseif bar>2<#else>3</#if>");
         assertConvertedSame("<#if  foo >1<#elseIf  bar >2<#else >3</#if
>");
+
+        assertConvertedSame("<#macro m>body</#macro>");
+        assertConvertedSame("<#macro <#-- C1 --> m <#-- C2 -->></#macro
>");
+        assertConvertedSame("<#macro m()></#macro>");
+        assertConvertedSame("<#macro m <#-- C1 --> ( <#-- C2 --> ) <#--
C3 --> ></#macro>");
+        assertConvertedSame("<#macro m p1></#macro>");
+        assertConvertedSame("<#macro m(p1)></#macro>");
+        assertConvertedSame("<#macro m p1 p2 p3></#macro>");
+        assertConvertedSame("<#macro m p1 <#-- C1 --> p2 <#-- C2 --> p3 <#--
C3 -->></#macro>");
+        assertConvertedSame("<#macro m(p1<#-- C1 -->,<#-- C2 --> p2<#--
C3 -->,<#-- C4 -->"
+                + " p5<#-- C5 -->)<#-- C6 -->></#macro>");
+        assertConvertedSame("<#macro m p1=11 p2=foo p3=a+b></#macro>");
+        assertConvertedSame("<#macro m(p1=11, p2=foo, p3=a+b)></#macro>");
+        assertConvertedSame("<#macro m p1<#-- C1 -->=<#-- C2 -->11<#--
C3 -->,<#-- C4 -->p2=22></#macro>");
+        assertConvertedSame("<#macro m others...></#macro>");
+        assertConvertedSame("<#macro m p1 others...></#macro>");
+        assertConvertedSame("<#macro m p1 p2=22 others...></#macro>");
+        assertConvertedSame("<#macro m(others...)></#macro>");
+        assertConvertedSame("<#macro m(others <#-- C1 --> ... <#-- C2 -->
)></#macro>");
+        assertConvertedSame("<#function m x y>foo</#function>");
+        assertConvertedSame("<#macro m\\-1 p\\-1></#macro>");
     }
 
     @Test


Mime
View raw message