Mailing-List: contact notifications-help@freemarker.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@freemarker.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
From: ddekany@apache.org
To: notifications@freemarker.incubator.apache.org
Message-Id: <6637bf8852674f898bac03ff0218993d@git.apache.org>
Subject: incubator-freemarker git commit: Continued working on FM2 to FM3
 converter...
Date: Thu, 13 Jul 2017 18:45:39 +0000 (UTC)
archived-at: Thu, 13 Jul 2017 18:45:48 -0000

Repository: incubator-freemarker
Updated Branches:
  refs/heads/3 7991d3e54 -> 15c6d3817


Continued working on FM2 to FM3 converter...


Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/15c6d381
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/15c6d381
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/15c6d381

Branch: refs/heads/3
Commit: 15c6d381736657b0b64792dae745881f05d9e6a4
Parents: 7991d3e
Author: ddekany <ddekany@apache.org>
Authored: Thu Jul 13 20:45:20 2017 +0200
Committer: ddekany <ddekany@apache.org>
Committed: Thu Jul 13 20:45:20 2017 +0200

----------------------------------------------------------------------
 .../core/FM2ASTToFM3SourceConverter.java        | 142 ++++++++++++++-----
 .../apache/freemarker/converter/Converter.java  |  17 ++-
 .../freemarker/converter/FM2ToFM3Converter.java |  47 +++++-
 .../converter/FM2ToFM3ConverterTest.java        |  44 ++++--
 .../apache/freemarker/core/util/FTLUtil.java    |  27 +++-
 .../jsp/webapps/errors/failing-parsetime.ftlnv  |   2 +-
 6 files changed, 220 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
index d8f5225..e4a3e12 100644
--- a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
+++ b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
@@ -19,6 +19,9 @@
 
 package freemarker.core;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -28,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.freemarker.converter.ConversionMarkers;
 import org.apache.freemarker.converter.ConverterException;
 import org.apache.freemarker.converter.ConverterUtils;
@@ -40,6 +44,8 @@ import org.apache.freemarker.core.util._StringUtil;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
+import freemarker.cache.StringTemplateLoader;
+import freemarker.cache.TemplateLoader;
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template.utility.StringUtil;
@@ -75,6 +81,7 @@ import freemarker.template.utility.StringUtil;
 public class FM2ASTToFM3SourceConverter {
 
     private final Template template;
+    private final StringTemplateLoader overlayTemplateLoader;
     private final String src;
     private final ConversionMarkers markers;
 
@@ -94,9 +101,10 @@ public class FM2ASTToFM3SourceConverter {
      *         {@code false}.
      */
     public static Result convert(
-            String templateName, String src, Configuration fm2Cfg, ConversionMarkers warnReceiver)
+            Template template, Configuration fm2Cfg, StringTemplateLoader overlayTemplateLoader,
+            ConversionMarkers warnReceiver)
             throws ConverterException {
-        return new FM2ASTToFM3SourceConverter(templateName, src, fm2Cfg, warnReceiver).convert();
+        return new FM2ASTToFM3SourceConverter(template, fm2Cfg, overlayTemplateLoader, warnReceiver).convert();
     }
 
     private Result convert() throws ConverterException {
@@ -106,16 +114,30 @@ public class FM2ASTToFM3SourceConverter {
     }
 
     private FM2ASTToFM3SourceConverter(
-            String templateName, String src, Configuration fm2Cfg, ConversionMarkers warnReceiver)
+            Template template, Configuration fm2Cfg, StringTemplateLoader overlayTemplateLoader,
+            ConversionMarkers warnReceiver)
             throws ConverterException {
-        template = createTemplate(templateName, src, fm2Cfg);
-        if (template.getParserConfiguration().getWhitespaceStripping()) {
-            throw new IllegalArgumentException("The Template must have been parsed with whitespaceStripping false.");
-        }
+        this.template = template;
+        this.overlayTemplateLoader = overlayTemplateLoader;
 
-        _NullArgumentException.check("src", src);
+        try {
+            TemplateLoader templateLoader = fm2Cfg.getTemplateLoader();
+            Object templateSource = templateLoader.findTemplateSource(template.getName());
+            if (templateSource == null) {
+                throw new FileNotFoundException("Template not found: " + template.getName());
+            }
 
-        this.src = src;
+            Reader reader = templateLoader.getReader(templateSource, template.getEncoding());
+            try {
+                this.src = IOUtils.toString(reader);
+            } finally {
+                reader.close();
+            }
+        } catch (IOException e) {
+            throw new ConverterException("Failed to load template source", e);
+        } finally {
+            fm2Cfg.clearTemplateCache();
+        }
 
         this.markers = warnReceiver;
 
@@ -129,23 +151,12 @@ public class FM2ASTToFM3SourceConverter {
         }
     }
 
-    /**
-     * Converts a {@link String} to a {@link Template}.
-     */
-    private static Template createTemplate(String templateName, String src, Configuration fm2Cfg) throws
-            ConverterException {
-        if (fm2Cfg.getWhitespaceStripping()) {
-            throw new IllegalArgumentException("Configuration.whitespaceStripping must be false");
-        }
-        try {
-            return new Template(templateName, src, fm2Cfg);
-        } catch (Exception e) {
-            throw new ConverterException("Failed to load FreeMarker 2.3.x template", e);
-        }
-    }
-
     // The FTL tag is not part of the AST tree, so we have to treat it differently
     private void printDirFtl() throws ConverterException {
+        if (printNextCustomDirAsFtlDir) {
+            return;
+        }
+
         int pos = getPositionAfterWSAndExpComments(0);
         if (src.length() > pos + 1 && src.charAt(pos) == tagBeginChar && src.startsWith("#ftl", pos + 1)) {
             printWithConvertedExpComments(src.substring(0, pos));
@@ -186,14 +197,30 @@ public class FM2ASTToFM3SourceConverter {
             boolean hasSlash = src.charAt(tagEnd - 1) == '/';
 
             // We need the Expression-s parsed, but they aren't part of the AST. So, we parse a template that contains
-            // a similar custom "ftl" directive, and the converter to print it as an #ftl directive.
-            FM2ASTToFM3SourceConverter customFtlDirSrcConverter = new FM2ASTToFM3SourceConverter(
-                    template.getName(),
-                    tagBeginChar + "@ftl" + src.substring(pos, tagEnd) + (hasSlash ? "" : "/") + tagEndChar,
-                    template.getConfiguration(), markers
-            );
-            customFtlDirSrcConverter.printNextCustomDirAsFtlDir = true;
-            String fm3Content = customFtlDirSrcConverter.convert().fm3Content;
+            // a similar custom "ftl" directive, and set up the converter to print it as an #ftl directive.
+            String fm3Content;
+            {
+                Configuration fm2Cfg = template.getConfiguration();
+                fm2Cfg.clearTemplateCache();
+                FM2ASTToFM3SourceConverter customFtlDirSrcConverter;
+                overlayTemplateLoader.putTemplate(
+                        template.getName(),
+                        tagBeginChar + "#ftl" + tagEndChar
+                        + tagBeginChar + "@ftl" + src.substring(pos, tagEnd) + (hasSlash ? "" : "/") + tagEndChar);
+                try {
+                    customFtlDirSrcConverter = new FM2ASTToFM3SourceConverter(
+                            fm2Cfg.getTemplate(template.getName()),
+                            fm2Cfg, overlayTemplateLoader, markers
+                    );
+                } catch (IOException e) {
+                    throw new ConverterException("Failed load template made for #ftl parsing", e);
+                } finally {
+                    overlayTemplateLoader.removeTemplate(template.getName());
+                    fm2Cfg.clearTemplateCache();
+                }
+                customFtlDirSrcConverter.printNextCustomDirAsFtlDir = true;
+                fm3Content = customFtlDirSrcConverter.convert().fm3Content;
+            }
             print(hasSlash
                     ? fm3Content
                     : fm3Content.substring(0, fm3Content.length() - 2) + tagEndChar);
@@ -621,7 +648,7 @@ public class FM2ASTToFM3SourceConverter {
                 printOptionalSeparatorAndWSAndExpComments(pos, ",");
             }
         }
-        printDirStartTagEnd(node, pos, true);
+        printDirStartTagEnd(node, pos, false);
     }
 
     private void printDirBreak(BreakInstruction node) throws ConverterException {
@@ -1695,7 +1722,35 @@ public class FM2ASTToFM3SourceConverter {
         }
     }
 
+    private int stringLiteralNestingLevel;
+
     private void printExpStringLiteral(StringLiteral node) throws ConverterException {
+        boolean escapeAmp, escapeLT, escapeGT;
+        if (stringLiteralNestingLevel == 0) {
+            // We check if the source code has avoided '&', '<', and '>'. If it did, we will escape them in the output.
+
+            escapeAmp = true;
+            escapeLT = true;
+            escapeGT = true;
+
+            int endPos = getEndPositionInclusive(node);
+            for (int idx = getStartPosition(node) + 1; idx < endPos; idx++) {
+                char c = src.charAt(idx);
+                if (c == '&') {
+                    escapeAmp = false;
+                } else if (c == '<') {
+                    escapeLT = false;
+                } else if (c == '>') {
+                    escapeGT = false;
+                }
+            }
+        } else {
+            // Don't escape in nested literals (like ${'${"nested"}'}), as the outer literal takes care of that.
+            escapeAmp = false;
+            escapeLT = false;
+            escapeGT = false;
+        }
+
         boolean rawString = false;
         char quote;
         {
@@ -1721,7 +1776,7 @@ public class FM2ASTToFM3SourceConverter {
         int parameterCount = node.getParameterCount();
         if (parameterCount == 0) {
             if (!rawString) {
-                print(FTLUtil.escapeStringLiteralPart(node.getAsString(), quote));
+                print(FTLUtil.escapeStringLiteralPart(node.getAsString(), quote, escapeAmp, escapeLT, escapeGT));
             } else {
                 print(node.getAsString());
             }
@@ -1730,14 +1785,21 @@ public class FM2ASTToFM3SourceConverter {
             for (int paramIdx = 0; paramIdx < parameterCount; paramIdx++) {
                 Object param = getParam(node, paramIdx, ParameterRole.VALUE_PART, Object.class);
                 if (param instanceof String) {
-                    print(FTLUtil.escapeStringLiteralPart((String) param));
+                    print(FTLUtil.escapeStringLiteralPart((String) param, quote, escapeAmp, escapeLT, escapeGT));
                 } else {
                     assertNodeContent(param instanceof Interpolation, node,
                             "Unexpected parameter type: {}", param.getClass().getName());
 
                     // We print the interpolation, the cut it out from the output, then put it back escaped:
                     int interpStartPos = out.length();
-                    printNode((TemplateElement) param);
+
+                    try {
+                        stringLiteralNestingLevel++;
+                        printNode((TemplateElement) param);
+                    } finally {
+                        stringLiteralNestingLevel--;
+                    }
+
                     int interpEndPos = out.length();
                     String interp = out.substring(interpStartPos, interpEndPos);
                     out.setLength(interpStartPos + 2); // +2 to keep the "${"
@@ -1880,6 +1942,14 @@ public class FM2ASTToFM3SourceConverter {
             throw new UnexpectedNodeContentException(node, "Unexpected end tag name: {}", srcTagName);
         }
 
+        char slash = src.charAt(pos - 1);
+        if (pos < 1 || slash != '/') {
+            if (optional) {
+                return;
+            }
+            throw new UnexpectedNodeContentException(node, "'/' expected, but found {}", slash);
+        }
+
         print(tagBeginChar);
         print("/#");
 

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
index 514ffe0..c0527f9 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
@@ -283,6 +283,12 @@ public abstract class Converter {
     }
 
     private boolean isToBeProcessed(File src) {
+        String relSrcPath = getRelativeSourcePathWithSlashes(src);
+        return (include == null || include.matcher(relSrcPath).matches())
+                && (exclude == null || !exclude.matcher(relSrcPath).matches());
+    }
+
+    private String getRelativeSourcePathWithSlashes(File src) {
         String relSrcPath;
         File source = getSource();
         if (source.isFile()) {
@@ -290,9 +296,7 @@ public abstract class Converter {
         } else {
             relSrcPath = pathToStringWithSlashes(source.toPath().relativize(src.toPath()).normalize());
         }
-
-        return (include == null || include.matcher(relSrcPath).matches())
-                && (exclude == null || !exclude.matcher(relSrcPath).matches());
+        return relSrcPath;
     }
 
     private String pathToStringWithSlashes(Path path) {
@@ -404,6 +408,13 @@ public abstract class Converter {
             return sourceFile;
         }
 
+        /**
+         * Returns the path of the source file relatively to the source directory, using slash to separate directories.
+         */
+        public String getRelativeSourcePathWithSlashes() {
+            return Converter.this.getRelativeSourcePathWithSlashes(sourceFile);
+        }
+
         public String getSourceFileName() {
             return sourceFile.getName();
         }

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
index 9525eb1..e233ccd 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
@@ -20,17 +20,21 @@
 package org.apache.freemarker.converter;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.Properties;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.freemarker.core.NamingConvention;
 import org.apache.freemarker.core.util._NullArgumentException;
 
 import com.google.common.collect.ImmutableMap;
 
+import freemarker.cache.FileTemplateLoader;
+import freemarker.cache.MultiTemplateLoader;
+import freemarker.cache.StringTemplateLoader;
+import freemarker.cache.StrongCacheStorage;
+import freemarker.cache.TemplateLoader;
+import freemarker.cache.TemplateLookupStrategy;
 import freemarker.core.CSSOutputFormat;
 import freemarker.core.FM2ASTToFM3SourceConverter;
 import freemarker.core.HTMLOutputFormat;
@@ -80,6 +84,7 @@ public class FM2ToFM3Converter extends Converter {
     private Map<String, String> fileExtensionSubtitutions = DEFAULT_FILE_EXTENSION_SUBSTITUTIONS;
     private Properties freeMarker2Settings;
     private Configuration fm2Cfg;
+    private StringTemplateLoader stringTemplateLoader;
 
     @Override
     protected Pattern getDefaultInclude() {
@@ -90,10 +95,7 @@ public class FM2ToFM3Converter extends Converter {
     protected void prepare() throws ConverterException {
         super.prepare();
         fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial unknown tags */);
-        fm2Cfg.setWhitespaceStripping(false);
-        fm2Cfg.setTabSize(1);
         fm2Cfg.setRecognizeStandardFileExtensions(true);
-        _TemplateAPI.setPreventStrippings(fm2Cfg, true);
         if (freeMarker2Settings != null) {
             try {
                 fm2Cfg.setSettings(freeMarker2Settings);
@@ -101,10 +103,34 @@ public class FM2ToFM3Converter extends Converter {
                 throw new ConverterException("Error while configuring FreeMarker 2", e);
             }
         }
+
+        // From now on we will overwrite settings that the user has set with freeMarker2Settings.
+
+        fm2Cfg.setTabSize(1);
+        _TemplateAPI.setPreventStrippings(fm2Cfg, true);
+        fm2Cfg.setTemplateLookupStrategy(TemplateLookupStrategy.DEFAULT_2_3_0);
+        fm2Cfg.setLocalizedLookup(false);
+        fm2Cfg.setCacheStorage(new StrongCacheStorage());
+
+        stringTemplateLoader = new StringTemplateLoader();
+        try {
+            fm2Cfg.setTemplateLoader(new MultiTemplateLoader(new TemplateLoader[] {
+                    stringTemplateLoader,
+                    new FileTemplateLoader(getSource().isDirectory() ? getSource() : getSource().getParentFile())
+                }
+            ));
+        } catch (IOException e) {
+            throw new ConverterException("Failed to create template loader", e);
+        }
     }
 
     private String getDestinationFileName(Template template) throws ConverterException {
         String srcFileName = template.getName();
+        int lastSlashIdx = srcFileName.lastIndexOf('/');
+        if (lastSlashIdx != -1) {
+            srcFileName = srcFileName.substring(lastSlashIdx + 1);
+        }
+
         int lastDotIdx = srcFileName.lastIndexOf('.');
         if (lastDotIdx == -1) {
             return srcFileName;
@@ -129,9 +155,16 @@ public class FM2ToFM3Converter extends Converter {
     
     @Override
     protected void convertFile(FileConversionContext fileTransCtx) throws ConverterException, IOException {
-        String src = IOUtils.toString(fileTransCtx.getSourceStream(), StandardCharsets.UTF_8);
+        Template template = null;
+        try {
+            template = fm2Cfg.getTemplate(fileTransCtx.getRelativeSourcePathWithSlashes());
+        } catch (Exception e) {
+            throw new ConverterException("Failed to load FreeMarker 2.3.x template", e);
+        }
+        fm2Cfg.clearTemplateCache();
+
         FM2ASTToFM3SourceConverter.Result result = FM2ASTToFM3SourceConverter.convert(
-                fileTransCtx.getSourceFile().getName(), src, fm2Cfg, fileTransCtx.getConversionMarkers()
+                template, fm2Cfg, stringTemplateLoader, fileTransCtx.getConversionMarkers()
         );
         fileTransCtx.setDestinationFileName(getDestinationFileName(result.getFM2Template()));
         fileTransCtx.getDestinationStream().write(

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
index 751fe76..b4bb719 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
@@ -26,6 +26,8 @@ import static org.junit.Assert.*;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Properties;
 import java.util.Set;
 
@@ -62,6 +64,9 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         assertConvertedSame("${'1${x + 1 + \\'s\\'}2'}");
         assertConvertedSame("${\"s ${'x $\\{\\\"y\\\"}'}\"}");
         assertConvertedSame("${'${1}${2}'}");
+        assertConvertedSame("<@m x='${e1 + \"a\\'b$\\{x}\"}' y='$\\{e2}' />");
+        assertConvertedSame("${\"&<>\\\"'{}\\\\a/\"}");
+        assertConvertedSame("${\"${x}&<>\\\"'{}${x}\\\\a/${x}\"}");
 
         assertConvertedSame("${r'${1}'}");
 
@@ -204,7 +209,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         // [FM3] Will be different (comma)
         assertConvertedSame("<#macro m><#nested x + 1 2 3></#macro>");
         assertConvertedSame("<#macro m><#nested <#--1--> x + 1 <#--2--> 2 <#--3-->></#macro>");
-        assertConverted("<#macro m><#nested x></#macro>", "<#macro m><#nested x /></#macro>");
+        assertConvertedSame("<#macro m><#nested x /></#macro>");
         assertConvertedSame("<#macro m><#return><#return ></#macro>");
 
         assertConvertedSame("<#assign x = 1>");
@@ -237,6 +242,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         assertConverted("<#attempt >a<#recover  >r</#attempt   >", "<#attempt >a<#recover  >r</#recover   >");
 
         assertConvertedSame("<#ftl>");
+        assertConvertedSame("[#ftl]"); // To test when the tag syntax is overridden by #ftl
         assertConvertedSame("<#ftl>x");
         assertConvertedSame("<#ftl>x${x}");
         assertConvertedSame("<#ftl>\nx${x}");
@@ -298,6 +304,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
 
         assertConvertedSame("<#list xs as x>${x}<#sep>, </#list>");
         assertConvertedSame("<#list xs as x>${x}<#sep>, </#sep></#list>");
+        assertConvertedSame("<#list xs as x><#sep></#list>");
 
         assertConvertedSame("<#list xs as x>${x}<#else>-</#list>");
         assertConvertedSame("<#list xs as x>${x}<#else >-</#list >");
@@ -422,14 +429,6 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
     }
 
     @Test
-    public void testTagEndCharGlitch() throws IOException, ConverterException {
-        assertConverted("<#assign x = 1>x", "<#assign x = 1]x");
-        assertConverted("<#if x[0] == 1>x<#else>y</#if>", "<#if x[0] == 1]x<#else]y</#if]");
-        assertConverted("<@m x[0]>x</@m>", "<@m x[0]]x</@m]");
-        assertConverted("<#ftl customSettings={'a': []}>x", "<#ftl attributes={'a': []}]x");
-    }
-
-    @Test
     public void testBuiltInExpressions() throws IOException, ConverterException {
         assertConverted("${s?upperCase} ${s?leftPad(123)}", "${s?upper_case} ${s?left_pad(123)}");
         assertConverted("${s?html}", "${s?web_safe}");
@@ -443,6 +442,14 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         assertConvertedSame("${s?then <#--1--> ( <#--2--> 1 <#--3-->, <#--5--> 2 <#--6--> )}");
     }
 
+    @Test
+    public void testTagEndCharGlitch() throws IOException, ConverterException {
+        assertConverted("<#assign x = 1>x", "<#assign x = 1]x");
+        assertConverted("<#if x[0] == 1>x<#else>y</#if>", "<#if x[0] == 1]x<#else]y</#if]");
+        assertConverted("<@m x[0]>x</@m>", "<@m x[0]]x</@m]");
+        assertConverted("<#ftl customSettings={'a': []}>x", "<#ftl attributes={'a': []}]x");
+    }
+
     /** Tests if the names of all current FM2 built-ins can be converted to FM3 names. */
     @Test
     public void testBuiltInNameConversion()
@@ -538,6 +545,25 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
             converter.execute();
     }
 
+    @Test
+    public void testCharset() throws IOException, ConverterException {
+        FileUtils.write(new File(srcDir, "t1.ftl"),
+                "<#ftl encoding='ISO-8859-1'>béka",
+                StandardCharsets.ISO_8859_1);
+        FileUtils.write(new File(srcDir, "t2.ftl"),
+                "béka", Charset.forName("UTF-8"));
+
+        FM2ToFM3Converter converter = new FM2ToFM3Converter();
+        converter.setSource(srcDir);
+        converter.setDestinationDirectory(dstDir);
+        converter.execute();
+
+        assertThat(FileUtils.readFileToString(new File(dstDir, "t1.fm3"), StandardCharsets.ISO_8859_1),
+                containsString("béka"));
+        assertThat(FileUtils.readFileToString(new File(dstDir, "t2.fm3"), UTF_8),
+                containsString("béka"));
+    }
+
     private static final Set<String> LEGACY_ESCAPING_BUTILT_INS = ImmutableSet.of(
             "html", "xml", "xhtml", "rtf", "web_safe");
 

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-core/src/main/java/org/apache/freemarker/core/util/FTLUtil.java
----------------------------------------------------------------------
diff --git a/freemarker-core/src/main/java/org/apache/freemarker/core/util/FTLUtil.java b/freemarker-core/src/main/java/org/apache/freemarker/core/util/FTLUtil.java
index 29ff468..2f94c85 100644
--- a/freemarker-core/src/main/java/org/apache/freemarker/core/util/FTLUtil.java
+++ b/freemarker-core/src/main/java/org/apache/freemarker/core/util/FTLUtil.java
@@ -78,14 +78,29 @@ public final class FTLUtil {
 
     /**
      * Escapes a string according the FTL string literal escaping rules, assuming the literal is quoted with
-     * {@code quotation}; it doesn't add the quotation marks themselves.
+     * {@code quotation}; it doesn't add the quotation marks themselves. {@code '&'}, {@code '<'}, and {@code '>'}
+     * characters will be escaped.
      *
      * @param quotation Either {@code '"'} or {@code '\''}. It's assumed that the string literal whose part we calculate is
      *                  enclosed within this kind of quotation mark. Thus, the other kind of quotation character will not be
      *                  escaped in the result.
      */
     public static String escapeStringLiteralPart(String s, char quotation) {
-        return escapeStringLiteralPart(s, quotation, false);
+        return escapeStringLiteralPart(s, quotation, false, true, true, true);
+    }
+
+    /**
+     * Escapes a string according the FTL string literal escaping rules, assuming the literal is quoted with
+     * {@code quotation}; it doesn't add the quotation marks themselves.
+     *
+     * @param quotation See in {@link #escapeStringLiteralPart(String, char)}
+     * @param escapeAmp Whether to escape {@code '&'}
+     * @param escapeLT Whether to escape {@code '<'}
+     * @param escapeGT Whether to escape {@code '>'}
+     */
+    public static String escapeStringLiteralPart(String s, char quotation,
+            boolean escapeAmp, boolean escapeLT, boolean escapeGT) {
+        return escapeStringLiteralPart(s, quotation, false, escapeAmp, escapeLT, escapeGT);
     }
 
     /**
@@ -100,6 +115,11 @@ public final class FTLUtil {
     }
 
     private static String escapeStringLiteralPart(String s, char quotation, boolean addQuotation) {
+        return escapeStringLiteralPart(s, quotation, addQuotation, true, true, true);
+    }
+
+    private static String escapeStringLiteralPart(String s, char quotation, boolean addQuotation,
+            boolean escapeAmp, boolean escapeLT, boolean escapeGT) {
         final int ln = s.length();
 
         final char otherQuotation;
@@ -121,7 +141,8 @@ public final class FTLUtil {
                     c < escLn ? ESCAPES[c] :
                             c == '{' && i > 0 && isInterpolationStart(s.charAt(i - 1)) ? '{' :
                                     0;
-            if (escape == 0 || escape == otherQuotation) {
+            if (escape == 0 || escape == otherQuotation
+                    || c == '&' && !escapeAmp || c == '<' && !escapeLT || c == '>' && !escapeGT) {
                 if (buf != null) {
                     buf.append(c);
                 }

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/15c6d381/freemarker-servlet/src/test/resources/org/apache/freemarker/servlet/jsp/webapps/errors/failing-parsetime.ftlnv
----------------------------------------------------------------------
diff --git a/freemarker-servlet/src/test/resources/org/apache/freemarker/servlet/jsp/webapps/errors/failing-parsetime.ftlnv b/freemarker-servlet/src/test/resources/org/apache/freemarker/servlet/jsp/webapps/errors/failing-parsetime.ftlnv
index a83c41c..d0ecbe4 100644
--- a/freemarker-servlet/src/test/resources/org/apache/freemarker/servlet/jsp/webapps/errors/failing-parsetime.ftlnv
+++ b/freemarker-servlet/src/test/resources/org/apache/freemarker/servlet/jsp/webapps/errors/failing-parsetime.ftlnv
@@ -17,4 +17,4 @@
   under the License.
 -->
 
-${'x'?no_such_builtin}
\ No newline at end of file
+${'x'?noSuchBuiltin}
\ No newline at end of file