freemarker-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ddek...@apache.org
Subject [05/16] incubator-freemarker git commit: Reworked how ${...}-s are lexed a bit, to make the lexing of string literals (which may contain ${...}) more manageable.
Date Sun, 04 Oct 2015 19:17:09 GMT
Reworked how ${...}-s are lexed a bit, to make the lexing of string literals (which may contain
${...}) more manageable.


Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/d8487ffe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/d8487ffe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/d8487ffe

Branch: refs/heads/2.3
Commit: d8487ffee38a1fd3ab9291c9174d8f7e7ce15be8
Parents: 6802cc7
Author: ddekany <ddekany@apache.org>
Authored: Thu Oct 1 16:20:45 2015 +0200
Committer: ddekany <ddekany@apache.org>
Committed: Fri Oct 2 00:07:17 2015 +0200

----------------------------------------------------------------------
 src/main/javacc/FTL.jj                          | 57 +++++++++++++-------
 .../core/ParsingErrorMessagesTest.java          |  7 +++
 .../core/StringLiteralInterpolationTest.java    | 55 ++++++++++++-------
 3 files changed, 79 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/main/javacc/FTL.jj
----------------------------------------------------------------------
diff --git a/src/main/javacc/FTL.jj b/src/main/javacc/FTL.jj
index a3c5870..9c6bae1 100644
--- a/src/main/javacc/FTL.jj
+++ b/src/main/javacc/FTL.jj
@@ -255,10 +255,11 @@ public class FMParser {
     }
     
     void setupStringLiteralMode(FMParserTokenManager parentTokenSource) {
-        token_source.onlyTextOutput = true;
         token_source.initialNamingConvention = parentTokenSource.initialNamingConvention;
         token_source.namingConvention = parentTokenSource.namingConvention;
         token_source.namingConventionEstabilisher = parentTokenSource.namingConventionEstabilisher;
+        token_source.SwitchTo(NODIRECTIVE);
+        
         outputFormat = PlainTextOutputFormat.INSTANCE;
         recalculateAutoEscapingField();                                
         if (incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_24) {
@@ -578,12 +579,12 @@ TOKEN_MGR_DECLS:
      * distinguish the } used to close a hash literal and the one used to close a ${
      */
     private FMParser parser;
+    private int postInterpolationLexState = -1;
     private int hashLiteralNesting;
     private int parenthesisNesting;
     private int bracketNesting;
     private boolean inFTLHeader;
     boolean strictEscapeSyntax,
-            onlyTextOutput,
             squBracTagSyntax,
             autodetectTagSyntax,
             directiveSyntaxEstablished,
@@ -602,11 +603,6 @@ TOKEN_MGR_DECLS:
     // tag syntax detection. If you update this logic, take a look
     // at the UNKNOWN_DIRECTIVE token too.
     private void strictSyntaxCheck(Token tok, int tokenNamingConvention, int newLexState)
{
-        if (onlyTextOutput) {
-            tok.kind = STATIC_TEXT_NON_WS;
-            return;
-        }
-
         final String image = tok.image;
         
         // Non-strict syntax (deprecated) only supports legacy naming convention.
@@ -731,11 +727,6 @@ TOKEN_MGR_DECLS:
     }
 
     private void unifiedCall(Token tok) {
-        if (onlyTextOutput) {
-            tok.kind = STATIC_TEXT_NON_WS;
-            return;
-        }
-    
         char firstChar = tok.image.charAt(0);
         if (autodetectTagSyntax && !directiveSyntaxEstablished) {
             squBracTagSyntax = (firstChar == '[');
@@ -753,11 +744,6 @@ TOKEN_MGR_DECLS:
     }
 
     private void unifiedCallEnd(Token tok) {
-        if (onlyTextOutput) {
-            tok.kind = STATIC_TEXT_NON_WS;
-            return;
-        }
-    
         char firstChar = tok.image.charAt(0);
         if (squBracTagSyntax && firstChar == '<') {
             tok.kind = STATIC_TEXT_NON_WS;
@@ -781,6 +767,37 @@ TOKEN_MGR_DECLS:
             SwitchTo(DEFAULT);
         }
     }
+    
+    private void startInterpolation(Token tok) {
+        if (postInterpolationLexState != -1) {
+            char c = tok.image.charAt(0);
+            throw new TokenMgrError(
+                    "You can't start an interpolation (" + c + "{...}) here "
+                    + "as you are inside another interpolation.)",
+                    TokenMgrError.LEXICAL_ERROR,
+                    tok.beginLine, tok.beginColumn,
+                    tok.endLine, tok.endColumn);
+        }
+        postInterpolationLexState = curLexState;
+        SwitchTo(FM_EXPRESSION);
+    }
+
+    /**
+     * @param tok
+     *         Assumed to be an '}', or something that is the closing pair of another "mirror
image" character.
+     */
+    private void endInterpolation(Token tok) {
+        if (postInterpolationLexState == -1) {
+            char c = tok.image.charAt(0);
+            throw new TokenMgrError(
+                    "You can't have an \"" + c + "\" here, as there's nothing open that it
could close.",
+                    TokenMgrError.LEXICAL_ERROR,
+                    tok.beginLine, tok.beginColumn,
+                    tok.endLine, tok.endColumn);
+        }
+        SwitchTo(postInterpolationLexState);
+        postInterpolationLexState = -1;
+    }
 
     private void eatNewline() {
         int charsRead = 0;
@@ -1096,9 +1113,9 @@ TOKEN:
     |
     <STATIC_TEXT_FALSE_ALARM : "$" | "#" | "<" | "[" | "{"> // to handle a lone
dollar sign or "<" or "# or <@ with whitespace after"
     |
-    <DOLLAR_INTERPOLATION_OPENING : "${"> : FM_EXPRESSION
+    <DOLLAR_INTERPOLATION_OPENING : "${"> { startInterpolation(matchedToken); }
     |
-    <HASH_INTERPOLATION_OPENING : "#{"> : FM_EXPRESSION
+    <HASH_INTERPOLATION_OPENING : "#{"> { startInterpolation(matchedToken); }
 }
 
 <FM_EXPRESSION, IN_PAREN, NAMED_PARAMETER_EXPRESSION> SKIP :
@@ -1259,7 +1276,7 @@ TOKEN:
     |
     <CLOSING_CURLY_BRACKET : "}">
     {
-        if (hashLiteralNesting == 0) SwitchTo(DEFAULT);
+        if (hashLiteralNesting == 0) endInterpolation(matchedToken);
         else --hashLiteralNesting;
     }
     |

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/freemarker/core/ParsingErrorMessagesTest.java b/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
index d528675..f34ddf2 100644
--- a/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
+++ b/src/test/java/freemarker/core/ParsingErrorMessagesTest.java
@@ -74,6 +74,13 @@ public class ParsingErrorMessagesTest {
         assertErrorContains("${blah", "\"{\"", "unclosed");
     }
     
+    @Test
+    public void testInterpolatingClosingsErrors() {
+        assertErrorContains("${x", "unclosed");
+        assertErrorContains("<#assign x = x}>", "\"}\"", "open");
+        // TODO assertErrorContains("<#assign x = '${x'>", "unclosed");
+    }
+    
     private void assertErrorContains(String ftl, String... expectedSubstrings) {
         assertErrorContains(false, ftl, expectedSubstrings);
         assertErrorContains(true, ftl, expectedSubstrings);

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/d8487ffe/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/freemarker/core/StringLiteralInterpolationTest.java b/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
index e6d86e4..fd6fb14 100644
--- a/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
+++ b/src/test/java/freemarker/core/StringLiteralInterpolationTest.java
@@ -26,28 +26,33 @@ import freemarker.template.Configuration;
 import freemarker.template.TemplateException;
 import freemarker.test.TemplateTest;
 
+@SuppressWarnings("boxing")
 public class StringLiteralInterpolationTest extends TemplateTest {
 
     @Test
     public void basics() throws IOException, TemplateException {
-        assertOutput("<#assign x = 1>${'${x}'}", "1");
-        assertOutput("<#assign x = 1>${'${x} ${x}'}", "1 1");
-        assertOutput("<#assign x = 1>${'$\\{x}'}", "${x}");
-        assertOutput("<#assign x = 1>${'$\\{x} $\\{x}'}", "${x} ${x}");
-        assertOutput("<#assign x = 1>${'<#-- not a comment -->${x}'}", "<#--
not a comment -->1");
-        assertOutput("<#assign x = 1>${'<#-- not a comment -->$\\{x}'}", "<#--
not a comment -->${x}");
-        assertOutput("<#assign x = 1>${'<@x/>${x}'}", "<@x/>1");
-        assertOutput("<#assign x = 1>${'<@x/>$\\{x}'}", "<@x/>${x}");
-        assertOutput("<#assign x = 1>${'<@ ${x}<@'}", "<@ 1<@");
-        assertOutput("<#assign x = 1>${'<@ $\\{x}<@'}", "<@ ${x}<@");
-        assertOutput("<#assign x = 1>${'</@x>${x}'}", "</@x>1");
-        assertOutput("<#assign x = 1>${'</@x>$\\{x}'}", "</@x>${x}");
-        assertOutput("<#assign x = 1>${'</@ ${x}</@'}", "</@ 1</@");
-        assertOutput("<#assign x = 1>${'</@ $\\{x}</@'}", "</@ ${x}</@");
-        assertOutput("<#assign x = 1>${'[@ ${x}'}", "[@ 1");
-        assertOutput("<#assign x = 1>${'[@ $\\{x}'}", "[@ ${x}");
-        assertOutput("<#assign x = 1>${'<#assign x = 2> ${x}'}", "<#assign
x = 2> 1");
-        assertOutput("<#assign x = 1>${'<#assign x = 2> $\\{x}'}", "<#assign
x = 2> ${x}");
+        addToDataModel("x", 1);
+        assertOutput("${'${x}'}", "1");
+        assertOutput("${'#{x}'}", "1");
+        assertOutput("${'a${x}b${x*2}c'}", "a1b2c");
+        assertOutput("${'a#{x}b#{x*2}c'}", "a1b2c");
+        assertOutput("${'${x} ${x}'}", "1 1");
+        assertOutput("${'$\\{x}'}", "${x}");
+        assertOutput("${'$\\{x} $\\{x}'}", "${x} ${x}");
+        assertOutput("${'<#-- not a comment -->${x}'}", "<#-- not a comment -->1");
+        assertOutput("${'<#-- not a comment -->$\\{x}'}", "<#-- not a comment -->${x}");
+        assertOutput("${'<#assign x = 2> ${x} <#assign x = 2>'}", "<#assign
x = 2> 1 <#assign x = 2>");
+        assertOutput("${'<#assign x = 2> $\\{x} <#assign x = 2>'}", "<#assign
x = 2> ${x} <#assign x = 2>");
+        assertOutput("${'<@x/>${x}<@x/>'}", "<@x/>1<@x/>");
+        assertOutput("${'<@x/>$\\{x}<@x/>'}", "<@x/>${x}<@x/>");
+        assertOutput("${'<@ ${x}<@'}", "<@ 1<@");
+        assertOutput("${'<@ $\\{x}<@'}", "<@ ${x}<@");
+        assertOutput("${'</@x>${x}'}", "</@x>1");
+        assertOutput("${'</@x>$\\{x}'}", "</@x>${x}");
+        assertOutput("${'</@ ${x}</@'}", "</@ 1</@");
+        assertOutput("${'</@ $\\{x}</@'}", "</@ ${x}</@");
+        assertOutput("${'[@ ${x}'}", "[@ 1");
+        assertOutput("${'[@ $\\{x}'}", "[@ ${x}");
     }
 
     /**
@@ -55,8 +60,18 @@ public class StringLiteralInterpolationTest extends TemplateTest {
      */
     @Test
     public void legacyEscapingBugStillPresent() throws IOException, TemplateException {
-        assertOutput("<#assign x = 1>${'$\\{x} ${x}'}", "1 1");
-        assertOutput("<#assign x = 1>${'${x} $\\{x}'}", "1 1");
+        addToDataModel("x", 1);
+        assertOutput("${'$\\{x} ${x}'}", "1 1");
+        assertOutput("${'${x} $\\{x}'}", "1 1");
+    }
+    
+    @Test
+    public void legacyLengthGlitch() throws IOException, TemplateException {
+        assertOutput("${'${'}", "${");
+        assertOutput("${'${1'}", "${1");
+        assertOutput("${'${}'}", "${}");
+        assertOutput("${'${1}'}", "1");
+        assertErrorContains("${'${  '}", "");
     }
 
     @Test


Mime
View raw message