velocity-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cbris...@apache.org
Subject svn commit: r1870626 - in /velocity/engine/trunk/velocity-engine-core/src: main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java main/parser/Parser.jjt test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
Date Sat, 30 Nov 2019 13:47:08 GMT
Author: cbrisson
Date: Sat Nov 30 13:47:07 2019
New Revision: 1870626

URL: http://svn.apache.org/viewvc?rev=1870626&view=rev
Log:
[core] Use devoted File Separator control character for parser end of template processing

Modified:
    velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
    velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt
    velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java

Modified: velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
URL: http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
--- velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
(original)
+++ velocity/engine/trunk/velocity-engine-core/src/main/java/org/apache/velocity/runtime/parser/VelocityCharStream.java
Sat Nov 30 13:47:07 2019
@@ -61,9 +61,11 @@ implements CharStream
     private int inBuf = 0;
 
     /* CB - to properly handle EOF *inside* javacc lexer,
-     * we send a 'zero-width whitespace' *just before* EOF
+     * we send a 'file separator' ascii char *just before* EOF
+     * (see https://en.wikipedia.org/wiki/Delimiter#ASCII_delimited_text)
      */
     private boolean beforeEOF = false;
+    private static char END_OF_FILE = '\u001C';
 
     private void ExpandBuff(boolean wrapAround)
     {
@@ -161,7 +163,7 @@ implements CharStream
                     inputStream.close();
                     throw new java.io.IOException();
                 }
-                buffer[maxNextCharInd++] = '\u200B';
+                buffer[maxNextCharInd++] = END_OF_FILE;
                 beforeEOF = true;
             }
             else

Modified: velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt
URL: http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
--- velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt (original)
+++ velocity/engine/trunk/velocity-engine-core/src/main/parser/Parser.jjt Sat Nov 30 13:47:07
2019
@@ -749,7 +749,7 @@ TOKEN_MGR_DECLS:
 <PRE_DIRECTIVE,PRE_REFERENCE,PRE_OLD_REFERENCE>
 TOKEN :
 {
-    <LONE_SYMBOL: "\u200B" >
+    <LONE_SYMBOL: "\u001C" >
     {
         stateStackPop();
     }
@@ -759,7 +759,7 @@ TOKEN :
 <REFERENCE,REFMODIFIER,OLD_REFMODIFIER,REFMOD3,REFINDEX,DIRECTIVE,REFMOD2,DEFAULT,REFMOD,IN_TEXTBLOCK,IN_MULTILINE_COMMENT,IN_FORMAL_COMMENT,IN_SINGLE_LINE_COMMENT>
 TOKEN :
 {
-    <ZERO_WIDTH_WHITESPACE: "\u200B">
+    <ZERO_WIDTH_WHITESPACE: "\u001C">
 }
 
 <REFERENCE, REFMODIFIER, OLD_REFMODIFIER, REFMOD3>
@@ -1031,7 +1031,7 @@ MORE :
        }
     }
 
-|   <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}" ~["${parser.char.hash}","\u200B"]>
+|   <"${parser.char.hash}${parser.char.asterisk}${parser.char.asterisk}" ~["${parser.char.hash}","\u001C"]>
     {
     	if (!inComment)
     	{
@@ -1179,7 +1179,7 @@ SKIP :
 <IN_TEXTBLOCK>
 MORE :
 {
-  < ~["\u200B"] >
+  < ~["\u001C"] >
 }
 
 /* -----------------------------------------------------------------------
@@ -1219,7 +1219,7 @@ TOKEN :
 //   <STRING_LITERAL: ( "\"" ( ~["\"","\n","\r"] )* "\"" ) | ( "'" ( ~["'","\n","\r"]
)* "'" ) >
   < STRING_LITERAL:
       ("\""
-        (   (~["\"","\u200B"])
+        (   (~["\"","\u001C"])
           | ("\\"
               ( ["n","t","b","r","f"]
               | ["0"-"7"] ( ["0"-"7"] )?
@@ -1234,7 +1234,7 @@ TOKEN :
       )
     |
     ("\'"
-        (   (~["\'","\u200B"])
+        (   (~["\'","\u001C"])
           | ("''")
           | ( "\\" (" ")* "\n")
         )*
@@ -1526,12 +1526,12 @@ TOKEN :
 {
     <DOUBLE_ESCAPE : "\\\\">
 |   <ESCAPE: "\\" >
-|   <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])*
(~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n", " ", "\t","\u200B"])+
(~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE>
((~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])* <NEWLINE>)*
>
+|   <TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])*
(~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n", " ", "\t","\u001C"])+
(~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])* <NEWLINE>
((~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])* <NEWLINE>)*
>
 }
 
 TOKEN :
 {
-    <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u200B"])+
>
+    <INLINE_TEXT: (~["${parser.char.dollar}", "${parser.char.hash}", "\\", "\r", "\n","\u001C"])+
>
 }
 
 /**

Modified: velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
URL: http://svn.apache.org/viewvc/velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java?rev=1870626&r1=1870625&r2=1870626&view=diff
==============================================================================
--- velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
(original)
+++ velocity/engine/trunk/velocity-engine-core/src/test/java/org/apache/velocity/test/issues/Velocity919TestCase.java
Sat Nov 30 13:47:07 2019
@@ -28,4 +28,10 @@ public class Velocity919TestCase extends
         assertEvalEquals("before\u200Bafter", "before\u200Bafter");
     }
 
+    public void testUserFileSeparator() throws Exception
+    {
+        assertEvalEquals("before\u001Cafter", "before\u001Cafter");
+    }
+
 }
+



Mime
View raw message