freemarker-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ddek...@apache.org
Subject incubator-freemarker git commit: Continued working on FM2 to FM3 converter...
Date Wed, 12 Jul 2017 22:52:18 GMT
Repository: incubator-freemarker
Updated Branches:
  refs/heads/3 ea7a08903 -> 7991d3e54


Continued working on FM2 to FM3 converter...


Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/7991d3e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/7991d3e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/7991d3e5

Branch: refs/heads/3
Commit: 7991d3e5417269afa2be7b03a52f275c860ec671
Parents: ea7a089
Author: ddekany <ddekany@apache.org>
Authored: Thu Jul 13 00:51:48 2017 +0200
Committer: ddekany <ddekany@apache.org>
Committed: Thu Jul 13 00:51:48 2017 +0200

----------------------------------------------------------------------
 .../core/FM2ASTToFM3SourceConverter.java        | 137 ++++++++-----------
 .../freemarker/converter/ConverterUtils.java    |  20 +++
 .../freemarker/converter/FM2ToFM3Converter.java |  96 ++++++++++---
 .../converter/FM2ToFM3ConverterCLI.java         |  38 ++++-
 .../converter/FM2ToFM3ConverterCLITest.java     |  40 +++++-
 .../converter/FM2ToFM3ConverterTest.java        |  25 ++++
 6 files changed, 250 insertions(+), 106 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
index 09fa90b..d8f5225 100644
--- a/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
+++ b/freemarker-converter/src/main/java/freemarker/core/FM2ASTToFM3SourceConverter.java
@@ -32,7 +32,6 @@ import org.apache.freemarker.converter.ConversionMarkers;
 import org.apache.freemarker.converter.ConverterException;
 import org.apache.freemarker.converter.ConverterUtils;
 import org.apache.freemarker.converter.UnconvertableLegacyFeatureException;
-import org.apache.freemarker.core.NamingConvention;
 import org.apache.freemarker.core.util.FTLUtil;
 import org.apache.freemarker.core.util._ClassUtil;
 import org.apache.freemarker.core.util._NullArgumentException;
@@ -83,11 +82,9 @@ public class FM2ASTToFM3SourceConverter {
     private List<Integer> rowStartPositions;
     private final char tagBeginChar;
     private final char tagEndChar;
-    private final org.apache.freemarker.core.Configuration fm3Config = new org.apache.freemarker.core.Configuration
+    private final Set<String> fm3BuiltInNames = new org.apache.freemarker.core.Configuration
             .Builder(org.apache.freemarker.core.Configuration.getVersion() /* highest possible
by design */)
-            .namingConvention(NamingConvention.CAMEL_CASE)
-            .build();
-    private final Set<String> fm3BuiltInNames = fm3Config.getSupportedBuiltInNames();
+            .build().getSupportedBuiltInNames();
 
     private boolean printNextCustomDirAsFtlDir;
 
@@ -105,16 +102,7 @@ public class FM2ASTToFM3SourceConverter {
     private Result convert() throws ConverterException {
         printDirFtl();
         printNode(template.getRootTreeNode());
-
-        String outAsString = out.toString();
-        try {
-            new org.apache.freemarker.core.Template(null, outAsString, fm3Config);
-        } catch (Exception e) {
-            throw new ConverterException(
-                    "The result of the conversion wasn't valid FreeMarker 3 template; see
cause exception", e);
-        }
-
-        return new Result(template, outAsString);
+        return new Result(template, out.toString());
     }
 
     private FM2ASTToFM3SourceConverter(
@@ -182,7 +170,7 @@ public class FM2ASTToFM3SourceConverter {
                 }
 
                 tagEnd = firstNodePos - 1;
-                while (tagEnd >= 0 && src.charAt(tagEnd) != tagEndChar) {
+                while (tagEnd >= 0 && !isTagEndChar(src.charAt(tagEnd))) {
                     if (!Character.isWhitespace(src.charAt(tagEnd))) {
                         throw new ConverterException("Non-WS character while backtracking
to #ftl tag end character.");
                     }
@@ -482,22 +470,22 @@ public class FM2ASTToFM3SourceConverter {
     private void printDirVisitLike(TemplateElement node, String tagName) throws ConverterException
{
         assertParamCount(node, 2);
 
-        printDirStartTagPartBeforeParams(node, tagName);
-
-        Expression lastParam;
+        int pos = printDirStartTagPartBeforeParams(node, tagName);
 
         Expression nodeExp = getParam(node, 0, ParameterRole.NODE, Expression.class);
-        printExp(nodeExp);
-        lastParam = nodeExp;
+        if (nodeExp != null) {
+            printExp(nodeExp);
+            pos = getEndPositionExclusive(nodeExp);
+        }
 
         Expression ns = getParam(node, 1, ParameterRole.NAMESPACE, Expression.class);
         if (ns != null) {
-            printSeparatorAndWSAndExpComments(getEndPositionExclusive(lastParam), "using");
+            printSeparatorAndWSAndExpComments(pos, "using");
             printExp(ns);
-            lastParam = ns;
+            pos = getEndPositionExclusive(ns);
         }
 
-        printDirStartTagEnd(node, lastParam, false);
+        printDirStartTagEnd(node, pos, false);
     }
 
     private void printDirCase(Case node) throws ConverterException {
@@ -751,11 +739,11 @@ public class FM2ASTToFM3SourceConverter {
             String postVar2WSAndComment = readWSAndExpComments(getEndPositionExclusive(listSource));
 
             printExp(listSource);
-            printWithConvertedExpComments(rightTrim(postVar2WSAndComment));
+            printWithConvertedExpComments(ConverterUtils.rightTrim(postVar2WSAndComment));
             print(" as ");
             print(FTLUtil.escapeIdentifier(loopVal1));
-            printWithConvertedExpComments(rightTrim(postVar1WSAndComment));
-            printWithConvertedExpComments(rightTrim(postInWSAndComment));
+            printWithConvertedExpComments(ConverterUtils.rightTrim(postVar1WSAndComment));
+            printWithConvertedExpComments(ConverterUtils.rightTrim(postInWSAndComment));
         } else {
             throw new UnexpectedNodeContentException(node, "Expected #list or #foreach as
node symbol", null);
         }
@@ -810,7 +798,7 @@ public class FM2ASTToFM3SourceConverter {
             // We only have removed thing after in the src => no need for spacing after
us
             int commentPos = postNameWSOrComment.indexOf("--") - 1;
             if (commentPos >= 0) {
-                printWithConvertedExpComments(rightTrim(postNameWSOrComment));
+                printWithConvertedExpComments(ConverterUtils.rightTrim(postNameWSOrComment));
             }
         }
 
@@ -830,7 +818,7 @@ public class FM2ASTToFM3SourceConverter {
                 } else {
                     int commentPos = postParamWSOrComment.indexOf("--") - 1;
                     if (commentPos >= 0) {
-                        printWithConvertedExpComments(rightTrim(postParamWSOrComment));
+                        printWithConvertedExpComments(ConverterUtils.rightTrim(postParamWSOrComment));
                     }
                 }
             }
@@ -1108,7 +1096,7 @@ public class FM2ASTToFM3SourceConverter {
             {
                 char c = src.charAt(pos);
                 assertNodeContent(
-                        c == ',' || c == ')' || c == tagEndChar
+                        c == ',' || c == ')' || isTagEndChar(c)
                                 || c == '\\' || StringUtil.isFTLIdentifierStart(c),
                         node,
                         "Unexpected character: {}", c);
@@ -1145,7 +1133,7 @@ public class FM2ASTToFM3SourceConverter {
                 "Expected AST parameter at index {} to be the last one", paramIdx);
 
         pos = printOptionalSeparatorAndWSAndExpComments(pos, ")");
-        assertNodeContent(src.charAt(pos) == tagEndChar, node, "Tag end not found");
+        assertNodeContent(isTagEndChar(src.charAt(pos)), node, "Tag end not found");
         print(tagEndChar);
 
         printChildElements(node);
@@ -1250,7 +1238,7 @@ public class FM2ASTToFM3SourceConverter {
         int elementEndPos = getEndPositionInclusive(node);
         {
             char c = src.charAt(elementEndPos);
-            assertNodeContent(c == tagEndChar, node,
+            assertNodeContent(isTagEndChar(c), node,
                     "tagEndChar expected, found {}", c);
         }
         if (startTagEndPos != elementEndPos) { // We have an end-tag
@@ -1510,7 +1498,7 @@ public class FM2ASTToFM3SourceConverter {
         String rho = getParam(node, 1, ParameterRole.RIGHT_HAND_OPERAND, String.class);
         printNode(lho);
         printSeparatorAndWSAndExpComments(getEndPositionExclusive(lho), ".");
-        print(FTLUtil.escapeIdentifier(rho));
+        print(rho.startsWith("*") ? rho : FTLUtil.escapeIdentifier(rho));
     }
 
     private void printExpDynamicKeyName(DynamicKeyName node) throws ConverterException {
@@ -1695,9 +1683,13 @@ public class FM2ASTToFM3SourceConverter {
                     printSeparatorAndWSAndExpComments(pos, ",");
                 }
             }
-            pos = printSeparatorAndWSAndExpComments(pos, ")");
-            assertNodeContent(pos == getEndPositionExclusive(node), node,
+            pos = printWSAndExpComments(pos);
+            boolean endChar = src.charAt(pos) == ')';
+            assertNodeContent(pos == getEndPositionInclusive(node), node,
                     "Actual end position doesn't match node end position.");
+            assertNodeContent(endChar, node,
+                    "Expected ')' but found {}.");
+            print(')');
         } else {
             assertParamCount(node, 2);
         }
@@ -1856,7 +1848,7 @@ public class FM2ASTToFM3SourceConverter {
         int tagEndPos = getEndPositionInclusive(node);
         {
             char c = src.charAt(tagEndPos);
-            if (c != tagEndChar) {
+            if (!isTagEndChar(c)) {
                 if (optional) {
                     return;
                 }
@@ -1985,7 +1977,7 @@ public class FM2ASTToFM3SourceConverter {
         }
 
         char c = src.charAt(pos);
-        if (c == '/' && pos + 1 < src.length() && src.charAt(pos + 1)
== tagEndChar) {
+        if (c == '/' && pos + 1 < src.length() && isTagEndChar(src.charAt(pos
+ 1))) {
             printWithConvertedExpComments(src.substring(startPos, pos));
 
             if (removeSlash) {
@@ -1998,7 +1990,7 @@ public class FM2ASTToFM3SourceConverter {
             }
             print(tagEndChar);
             return pos + 1;
-        } else if (c == tagEndChar) {
+        } else if (isTagEndChar(c)) {
             printWithConvertedExpComments(src.substring(startPos, pos));
             print(tagEndChar);
             return pos;
@@ -2256,7 +2248,7 @@ public class FM2ASTToFM3SourceConverter {
             raw = false;
         }
         char quotationC = c;
-        if (!isQuotationChar(quotationC)) {
+        if (!ConverterUtils.isQuotationChar(quotationC)) {
             throw new IllegalArgumentException("The specifies position is not the beginning
of a string literal");
         }
 
@@ -2279,11 +2271,8 @@ public class FM2ASTToFM3SourceConverter {
             return false;
         }
         char c = src.charAt(pos);
-        return (isQuotationChar(c) || c == 'r' && pos < src.length() + 1 &&
isQuotationChar(src.charAt(pos + 1)));
-    }
-
-    private boolean isQuotationChar(char q) {
-        return q == '\'' || q == '\"';
+        return (ConverterUtils.isQuotationChar(c) || c == 'r' && pos < src.length()
+ 1 && ConverterUtils
+                .isQuotationChar(src.charAt(pos + 1)));
     }
 
     private String readIdentifier(int startPos) throws ConverterException {
@@ -2298,6 +2287,14 @@ public class FM2ASTToFM3SourceConverter {
         return src.substring(startPos, pos);
     }
 
+    /**
+     * Because FM2 has this glitch where tags starting wit {@code <} can be closed with
an unparied {@code ]}, we
+     * have to do this more complicated check.
+     */
+    private boolean isTagEndChar(char c) {
+        return c == tagEndChar || c == ']';
+    }
+
     private boolean isExpCommentStart(int pos) {
         char c = src.charAt(pos);
         return (c == '<' || c == '[')
@@ -2306,6 +2303,24 @@ public class FM2ASTToFM3SourceConverter {
 
     }
 
+    private HashMap<String, Boolean> containsUpperCaseLetterResults = new HashMap<>();
+
+    private boolean containsUpperCaseLetter(String s) {
+        Boolean result = containsUpperCaseLetterResults.get(s);
+        if (result != null) {
+            return result;
+        }
+
+        int i = 0;
+        while (i < s.length() && !ConverterUtils.isUpperCaseLetter(s.charAt(i)))
{
+            i++;
+        }
+        result = i < s.length();
+
+        containsUpperCaseLetterResults.put(s, result);
+        return result;
+    }
+
     public static class Result {
         private final Template fm2Template;
         private final String fm3Content;
@@ -2324,38 +2339,4 @@ public class FM2ASTToFM3SourceConverter {
         }
     }
 
-    private String rightTrim(String s) {
-        if (s == null) {
-            return null;
-        }
-
-        int i = s.length() - 1;
-        while (i >= 0 && Character.isWhitespace(s.charAt(i))) {
-            i--;
-        }
-        return i != -1 ? s.substring(0, i + 1) : "";
-    }
-
-    private boolean isUpperCaseLetter(char c) {
-        return Character.isUpperCase(c) && Character.isLetter(c);
-    }
-
-    private HashMap<String, Boolean> containsUpperCaseLetterResults = new HashMap<>();
-
-    private boolean containsUpperCaseLetter(String s) {
-        Boolean result = containsUpperCaseLetterResults.get(s);
-        if (result != null) {
-            return result;
-        }
-
-        int i = 0;
-        while (i < s.length() && !isUpperCaseLetter(s.charAt(i))) {
-            i++;
-        }
-        result = i < s.length();
-
-        containsUpperCaseLetterResults.put(s, result);
-        return result;
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConverterUtils.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConverterUtils.java
b/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConverterUtils.java
index d4a64aa..aa837b6 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConverterUtils.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/ConverterUtils.java
@@ -52,4 +52,24 @@ public final class ConverterUtils {
         } while (wordStartIdx < s.length());
         return sb.toString();
     }
+
+    public static boolean isUpperCaseLetter(char c) {
+        return Character.isUpperCase(c) && Character.isLetter(c);
+    }
+
+    public static String rightTrim(String s) {
+        if (s == null) {
+            return null;
+        }
+
+        int i = s.length() - 1;
+        while (i >= 0 && Character.isWhitespace(s.charAt(i))) {
+            i--;
+        }
+        return i != -1 ? s.substring(0, i + 1) : "";
+    }
+
+    public static boolean isQuotationChar(char q) {
+        return q == '\'' || q == '\"';
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
index 041258b..9525eb1 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
@@ -21,15 +21,28 @@ package org.apache.freemarker.converter;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.freemarker.core.NamingConvention;
 import org.apache.freemarker.core.util._NullArgumentException;
 
+import com.google.common.collect.ImmutableMap;
+
+import freemarker.core.CSSOutputFormat;
 import freemarker.core.FM2ASTToFM3SourceConverter;
+import freemarker.core.HTMLOutputFormat;
+import freemarker.core.JSONOutputFormat;
+import freemarker.core.JavaScriptOutputFormat;
+import freemarker.core.MarkupOutputFormat;
+import freemarker.core.OutputFormat;
+import freemarker.core.PlainTextOutputFormat;
+import freemarker.core.RTFOutputFormat;
+import freemarker.core.UndefinedOutputFormat;
+import freemarker.core.XHTMLOutputFormat;
+import freemarker.core.XMLOutputFormat;
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template._TemplateAPI;
@@ -54,20 +67,17 @@ import freemarker.template._TemplateAPI;
  */
 public class FM2ToFM3Converter extends Converter {
 
-    private static final Pattern DEFAULT_INCLUDE = Pattern.compile("(?i).*\\.(fm|ftl(x|h)?)");
-
-    private static final Map<String, String> DEFAULT_REPLACED_FILE_EXTENSIONS;
-    static {
-        DEFAULT_REPLACED_FILE_EXTENSIONS = new HashMap<>();
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("ftl", "fm3");
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("fm", "fm3");
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("ftlh", "fm3h");
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("fmh", "fm3h");
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("ftlx", "fm3x");
-        DEFAULT_REPLACED_FILE_EXTENSIONS.put("fmx", "fm3x");
-    }
+    public static final Pattern DEFAULT_INCLUDE = Pattern.compile("(?i).*\\.(fm|ftl(x|h)?)");
+
+    public static final Map<String, String> DEFAULT_FILE_EXTENSION_SUBSTITUTIONS
+            = new ImmutableMap.Builder<String,String>()
+                    .put("ftl", "fm3")
+                    .put("ftlh", "fm3h")
+                    .put("ftlx", "fm3x")
+                    .put("fm", "fm3")
+                    .build();
 
-    private Map<String, String> outputFileExtensions = DEFAULT_REPLACED_FILE_EXTENSIONS;
+    private Map<String, String> fileExtensionSubtitutions = DEFAULT_FILE_EXTENSION_SUBSTITUTIONS;
     private Properties freeMarker2Settings;
     private Configuration fm2Cfg;
 
@@ -82,6 +92,7 @@ public class FM2ToFM3Converter extends Converter {
         fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial
unknown tags */);
         fm2Cfg.setWhitespaceStripping(false);
         fm2Cfg.setTabSize(1);
+        fm2Cfg.setRecognizeStandardFileExtensions(true);
         _TemplateAPI.setPreventStrippings(fm2Cfg, true);
         if (freeMarker2Settings != null) {
             try {
@@ -101,9 +112,9 @@ public class FM2ToFM3Converter extends Converter {
 
         String ext = srcFileName.substring(lastDotIdx + 1);
 
-        String replacementExt = getOutputFileExtensions().get(ext);
+        String replacementExt = getFileExtensionSubtitutions().get(ext);
         if (replacementExt == null) {
-            replacementExt = getOutputFileExtensions().get(ext.toLowerCase());
+            replacementExt = getFileExtensionSubtitutions().get(ext.toLowerCase());
         }
         if (replacementExt == null) {
             return srcFileName;
@@ -125,6 +136,49 @@ public class FM2ToFM3Converter extends Converter {
         fileTransCtx.setDestinationFileName(getDestinationFileName(result.getFM2Template()));
         fileTransCtx.getDestinationStream().write(
                 result.getFM3Content().getBytes(getTemplateEncoding(result.getFM2Template())));
+
+        try {
+            org.apache.freemarker.core.Configuration fm3Config = new org.apache.freemarker.core.Configuration
+                    .Builder(org.apache.freemarker.core.Configuration.getVersion() /* highest
possible by design */)
+                    .namingConvention(NamingConvention.CAMEL_CASE)
+                    .outputFormat(converOutputFormat(result.getFM2Template().getOutputFormat()))
+                    .build();
+            new org.apache.freemarker.core.Template(null, result.getFM3Content(), fm3Config);
+        } catch (Exception e) {
+            throw new ConverterException(
+                    "The result of the conversion wasn't valid FreeMarker 3 template; see
cause exception and "
+                    + fileTransCtx.getDestinationFile(), e);
+        }
+    }
+
+    private org.apache.freemarker.core.outputformat.OutputFormat converOutputFormat(OutputFormat
outputFormat) {
+        return outputFormat == HTMLOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.HTMLOutputFormat.INSTANCE
+                : outputFormat == XHTMLOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.XHTMLOutputFormat.INSTANCE
+                : outputFormat == XMLOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.XMLOutputFormat.INSTANCE
+                : outputFormat == RTFOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.RTFOutputFormat.INSTANCE
+                : outputFormat == PlainTextOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.PlainTextOutputFormat.INSTANCE
+                : outputFormat == UndefinedOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.UndefinedOutputFormat.INSTANCE
+                : outputFormat == JavaScriptOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.JavaScriptOutputFormat.INSTANCE
+                : outputFormat == JSONOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.JSONOutputFormat.INSTANCE
+                : outputFormat == CSSOutputFormat.INSTANCE
+                        ? org.apache.freemarker.core.outputformat.impl.CSSOutputFormat.INSTANCE
+                : getSimilarOutputFormat(outputFormat);
+    }
+
+    private org.apache.freemarker.core.outputformat.OutputFormat getSimilarOutputFormat(OutputFormat
outputFormat) {
+        if (outputFormat instanceof MarkupOutputFormat) {
+            return org.apache.freemarker.core.outputformat.impl.HTMLOutputFormat.INSTANCE;
+        } else {
+            return org.apache.freemarker.core.outputformat.impl.PlainTextOutputFormat.INSTANCE;
+        }
     }
 
     private String getTemplateEncoding(Template template) {
@@ -132,13 +186,13 @@ public class FM2ToFM3Converter extends Converter {
         return encoding != null ? encoding : fm2Cfg.getEncoding(template.getLocale());
     }
 
-    public Map<String, String> getOutputFileExtensions() {
-        return outputFileExtensions;
+    public Map<String, String> getFileExtensionSubtitutions() {
+        return fileExtensionSubtitutions;
     }
 
-    public void setOutputFileExtensions(Map<String, String> outputFileExtensions) {
-        _NullArgumentException.check("outputFileExtensions", outputFileExtensions);
-        this.outputFileExtensions = outputFileExtensions;
+    public void setFileExtensionSubtitutions(Map<String, String> fileExtensionSubtitutions)
{
+        _NullArgumentException.check("fileExtensionSubtitutions", fileExtensionSubtitutions);
+        this.fileExtensionSubtitutions = fileExtensionSubtitutions;
     }
 
     public Properties getFreeMarker2Settings() {

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3ConverterCLI.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3ConverterCLI.java
b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3ConverterCLI.java
index 208a727..3d26ad8 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3ConverterCLI.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3ConverterCLI.java
@@ -22,7 +22,10 @@ package org.apache.freemarker.converter;
 import java.io.File;
 import java.io.PrintWriter;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
@@ -45,7 +48,9 @@ public class FM2ToFM3ConverterCLI {
     private static final String CREATE_DESTINATION_OPTION = "create-destination";
     private static final String INCLUDE_OPTION = "include";
     private static final String EXCLUDE_OPTION = "exclude";
-    private static final String FREEMARKER_2_SETTING_OPTION_SHORT = "S";
+    private static final String FILE_EXTENSION_SUBSTITUTION = "file-ext-subst";
+    private static final String NO_PREDEFINED_FILE_EXTENSION_SUBSTITUTIONS = "no-predef-file-ext-substs";
+    private static final String FREEMARKER_2_SETTING_OPTION = "fm2-setting";
     private static final String HELP_OPTION = "help";
     private static final String HELP_OPTION_SHORT = "h";
 
@@ -71,8 +76,8 @@ public class FM2ToFM3ConverterCLI {
                             + "the files already matched by the \"include\" option. The default
matches nothing "
                             + "(nothing is excluded). See the \"include\" option about the
matched path.")
                     .build())
-            .addOption(Option.builder(FREEMARKER_2_SETTING_OPTION_SHORT)
-                    .hasArg().argName("name=value")
+            .addOption(Option.builder("S").longOpt(FREEMARKER_2_SETTING_OPTION)
+                    .hasArgs().argName("name=value").valueSeparator()
                     .desc("FreeMarker 2 configuration settings, to influence the parsing
of the source. You can have "
                             + "multiple instances of this option, to set multiple settings.
For the possible names "
                             + "and values see the FreeMarker 2 documentation, especially
"
@@ -80,6 +85,16 @@ public class FM2ToFM3ConverterCLI {
                             + ".lang.String-java.lang.String-"
                             + ".")
                     .build())
+            .addOption(Option.builder("E").longOpt(FILE_EXTENSION_SUBSTITUTION)
+                    .hasArgs().argName("old=new").valueSeparator()
+                    .desc("File extensions that will be substituted (replaced). If predefined
substitutions are "
+                            + "allowed (by default they are), then this substitution is added
to those "
+                            + "(maybe replacing one).")
+                    .build())
+            .addOption(Option.builder(null).longOpt(NO_PREDEFINED_FILE_EXTENSION_SUBSTITUTIONS)
+                    .desc("Disables the predefined file extension substitutions (i.e, \"ftl\",
\"ftlh\", "
+                            + "\"ftlx\" and \"fm\" are replaced with the corresponding FreeMarker
3 file extensions).")
+                    .build())
             .addOption(Option.builder(HELP_OPTION_SHORT).longOpt(HELP_OPTION)
                     .desc("Prints command-line help.")
                     .build());
@@ -123,18 +138,33 @@ public class FM2ToFM3ConverterCLI {
                 }
 
                 FM2ToFM3Converter converter = new FM2ToFM3Converter();
+
                 converter.setSource(new File(unparsedArgs.get(0)));
+
                 converter.setDestinationDirectory(new File(cl.getOptionValue(DESTINATION_OPTION)));
+
                 if (cl.hasOption(CREATE_DESTINATION_OPTION)) {
                     converter.setCreateDestinationDirectory(true);
                 }
+
                 if (cl.hasOption(INCLUDE_OPTION)) {
                     converter.setInclude(getRegexpOption(cl, INCLUDE_OPTION));
                 }
+
                 if (cl.hasOption(EXCLUDE_OPTION)) {
                     converter.setExclude(getRegexpOption(cl, EXCLUDE_OPTION));
                 }
-                converter.setFreeMarker2Settings(cl.getOptionProperties(FREEMARKER_2_SETTING_OPTION_SHORT));
+
+                Map<String, String> fileExtensionSubtitutions = cl.hasOption(NO_PREDEFINED_FILE_EXTENSION_SUBSTITUTIONS)
+                        ? new HashMap<String, String>()
+                        : new HashMap<String, String>(FM2ToFM3Converter.DEFAULT_FILE_EXTENSION_SUBSTITUTIONS);
+                for (Map.Entry<Object, Object> entry
+                        : cl.getOptionProperties(FILE_EXTENSION_SUBSTITUTION).entrySet())
{
+                    fileExtensionSubtitutions.put((String) entry.getKey(), (String) entry.getValue());
+                }
+                converter.setFileExtensionSubtitutions(Collections.unmodifiableMap(fileExtensionSubtitutions));
+
+                converter.setFreeMarker2Settings(cl.getOptionProperties(FREEMARKER_2_SETTING_OPTION));
                 try {
                     converter.execute();
 

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterCLITest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterCLITest.java
b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterCLITest.java
index 2d794d6..866c8b6 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterCLITest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterCLITest.java
@@ -81,17 +81,51 @@ public class FM2ToFM3ConverterCLITest extends ConverterTest {
         assertTrue(new File(dstDir, "3.txt").exists());
     }
 
+    @Test
+    public void testExtensionCustomization1() {
+        assertCLIResult(SUCCESS_EXIT_STATUS, null, null,
+                srcDir.toString(), "-d", dstDir.toString(),
+                "--include", ".*", "-Etxt=txt3");
+        assertTrue(new File(dstDir, "1.fm3").exists());
+        assertTrue(new File(dstDir, "3.txt3").exists());
+    }
+
+    @Test
+    public void testExtensionCustomization2() {
+        assertCLIResult(SUCCESS_EXIT_STATUS, null, null,
+                srcDir.toString(), "-d", dstDir.toString(),
+                "--include", ".*", "-Eftl=foo");
+        assertTrue(new File(dstDir, "1.foo").exists());
+        assertTrue(new File(dstDir, "3.txt").exists());
+    }
+
+    @Test
+    public void testExtensionCustomization3() {
+        assertCLIResult(SUCCESS_EXIT_STATUS, null, null,
+                srcDir.toString(), "-d", dstDir.toString(),
+                "--include", ".*",
+                "--no-predef-file-ext-substs",
+                "--file-ext-subst", "txt=txt3",
+                "--no-predef-file-ext-substs"
+        );
+        assertTrue(new File(dstDir, "1.ftl").exists());
+        assertTrue(new File(dstDir, "3.txt3").exists());
+    }
+
     public void assertCLIResult(int expectedExitStatus, String stdoutContains, String stdoutNotContains,
String...
             args) {
         StringWriter sw = new StringWriter();
         PrintWriter out = new PrintWriter(sw);
         int actualExitStatus = FM2ToFM3ConverterCLI.execute(out, args);
-        assertEquals(actualExitStatus, expectedExitStatus);
+        String stdout = sw.toString();
+        if (actualExitStatus != expectedExitStatus) {
+            assertEquals("Exit status mismatch. The output was:\n" + stdout, actualExitStatus,
expectedExitStatus);
+        }
         if (stdoutContains != null) {
-            assertThat(sw.toString(), containsString(stdoutContains));
+            assertThat(stdout, containsString(stdoutContains));
         }
         if (stdoutNotContains != null) {
-            assertThat(sw.toString(), not(containsString(stdoutNotContains)));
+            assertThat(stdout, not(containsString(stdoutNotContains)));
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/7991d3e5/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
index 4852604..751fe76 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
@@ -108,6 +108,9 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
 
         assertConvertedSame("${m.key}");
         assertConvertedSame("${m <#--1--> . <#--3--> key}");
+        assertConvertedSame("${m.@key}");
+        assertConvertedSame("${m.*}");
+        assertConvertedSame("${m.**}");
 
         assertConvertedSame("${.outputFormat}");
         assertConvertedSame("${. <#-- C --> outputFormat}");
@@ -366,6 +369,9 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         assertConvertedSame("<#recurse <#--1--> node <#--2-->>");
         assertConvertedSame("<#recurse node using ns>");
         assertConvertedSame("<#recurse node <#--1--> using <#--2--> ns <#--3-->>");
+        assertConvertedSame("<#recurse>");
+        assertConvertedSame("<#recurse <#--1-->>");
+        assertConvertedSame("<#recurse <#--1--> using <#--2--> ns>");
         assertConvertedSame("<#macro m><#fallback></#macro>");
         assertConvertedSame("<#macro m><#fallback /></#macro>");
 
@@ -416,6 +422,14 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
     }
 
     @Test
+    public void testTagEndCharGlitch() throws IOException, ConverterException {
+        assertConverted("<#assign x = 1>x", "<#assign x = 1]x");
+        assertConverted("<#if x[0] == 1>x<#else>y</#if>", "<#if x[0]
== 1]x<#else]y</#if]");
+        assertConverted("<@m x[0]>x</@m>", "<@m x[0]]x</@m]");
+        assertConverted("<#ftl customSettings={'a': []}>x", "<#ftl attributes={'a':
[]}]x");
+    }
+
+    @Test
     public void testBuiltInExpressions() throws IOException, ConverterException {
         assertConverted("${s?upperCase} ${s?leftPad(123)}", "${s?upper_case} ${s?left_pad(123)}");
         assertConverted("${s?html}", "${s?web_safe}");
@@ -513,6 +527,17 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         assertTrue(new File(dstDir, "t10.Foo3").exists());
     }
 
+    @Test
+    public void testOutputFormatSet() throws IOException, ConverterException {
+            File srcFile = new File(srcDir, "t.ftlh");
+            FileUtils.write(srcFile, "${x?esc}", UTF_8);
+
+            FM2ToFM3Converter converter = new FM2ToFM3Converter();
+            converter.setSource(srcFile);
+            converter.setDestinationDirectory(dstDir);
+            converter.execute();
+    }
+
     private static final Set<String> LEGACY_ESCAPING_BUTILT_INS = ImmutableSet.of(
             "html", "xml", "xhtml", "rtf", "web_safe");
 


Mime
View raw message