freemarker-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ddek...@apache.org
Subject incubator-freemarker git commit: Continued working on FM2 to FM3 converter...
Date Sun, 09 Jul 2017 23:07:35 GMT
Repository: incubator-freemarker
Updated Branches:
  refs/heads/3 80e4c3424 -> 4763581a7


Continued working on FM2 to FM3 converter...


Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/4763581a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/4763581a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/4763581a

Branch: refs/heads/3
Commit: 4763581a7c9b42058ca604efef6117639b8e32bd
Parents: 80e4c34
Author: ddekany <ddekany@apache.org>
Authored: Mon Jul 10 01:07:20 2017 +0200
Committer: ddekany <ddekany@apache.org>
Committed: Mon Jul 10 01:07:20 2017 +0200

----------------------------------------------------------------------
 .../apache/freemarker/converter/Converter.java  | 116 +++++++++++++++++++
 .../freemarker/converter/FM2ToFM3Converter.java |  38 ++++--
 .../converter/FM2ToFM3ConverterTest.java        |  28 ++++-
 .../converter/GenericConverterTest.java         |  39 +++++++
 4 files changed, 209 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
index 2b3d8eb..2c4fa0b 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
@@ -27,7 +27,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Writer;
+import java.nio.file.Path;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -36,16 +38,26 @@ import org.apache.freemarker.core.util._StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Abstract superclass for converters. A converter converts the files in the source directory
(and in its
+ * subdirectories, recursively), or of a single source file, and writes the converted files
into the the destination
+ * directory.
+ */
 public abstract class Converter {
 
+    public static final String PROPERTY_NAME_INCLUDE = "include";
+    public static final String PROPERTY_NAME_EXCLUDE = "exclude";
     public static final String PROPERTY_NAME_SOURCE = "source";
     public static final String PROPERTY_NAME_DESTINATION_DIRECTORY = "destinationDirectory";
+
     public static final String CONVERSION_MARKERS_FILE_NAME = "__conversion-markers.txt";
 
     public static ThreadLocal<FileConversionContext> FILE_CONVERSION_CONTEXT_TLS =
new ThreadLocal<>();
 
     private static final Logger LOG = LoggerFactory.getLogger(Converter.class);
 
+    private Pattern include;
+    private Pattern exclude;
     private File source;
     private File destinationDirectory;
     private boolean createDestinationDirectory;
@@ -54,30 +66,106 @@ public abstract class Converter {
     private Set<File> directoriesKnownToExist = new HashSet<>();
     private Writer conversionMarkersWriter;
 
+    public Converter() {
+        include = getDefaultInclude();
+    }
+
+    /**
+     * Getter pair of {@link #setSource(File)}.
+     */
     public File getSource() {
         return source;
     }
 
+    /**
+     * Sets the directory that contains the files to be converted (subdirectories will be
also searched, and so on
+     * recursively), or the single file that should be converted. Can't be {@code null}.
+     */
     public void setSource(File source) {
+        _NullArgumentException.check("source", source);
         this.source = source != null ? source.getAbsoluteFile() : null;
     }
 
+    /**
+     * Getter pair of {@link #setDestinationDirectory(File)}.
+     */
     public File getDestinationDirectory() {
         return destinationDirectory;
     }
 
+    /**
+     * Sets the directory into which the converted files will be written. Unless
+     * {@linkplain #setCreateDestinationDirectory(boolean) createDestinationDirectory property}
was set to {@code true},
+     * the directory must already exits.
+     * <p>
+     * Already existing files will be overwritten without warning. Already existing files
and subdirectories for which
+     * there's no clashing output will be left as is.
+     */
     public void setDestinationDirectory(File destinationDirectory) {
         this.destinationDirectory = destinationDirectory != null ? destinationDirectory.getAbsoluteFile()
: null;
     }
 
+    /**
+     * Getter pair of {@link #setCreateDestinationDirectory(boolean)}.
+     */
     public boolean isCreateDestinationDirectory() {
         return createDestinationDirectory;
     }
 
+    /**
+     * Sets whether the destination directory should be created if it doesn't exist. Defaults
to {@code false}.
+     * Subdirectories inside the destination directory will be always automatically created,
regardless of this
+     * property.
+     */
     public void setCreateDestinationDirectory(boolean createDestinationDirectory) {
         this.createDestinationDirectory = createDestinationDirectory;
     }
 
+    /**
+     * Getter pair of {@link #setInclude(Pattern)}.
+     */
+    public Pattern getInclude() {
+        return include;
+    }
+
+    /**
+     * Sets what subset of the files selected by the {@linkplain #setSource(File) source
property} will be
+     * processed (unless they are excluded by the {@linkplain #setExclude(Pattern) exclude}
property.)
+     * Default is {@link #getDefaultInclude()}.
+     *
+     * @param include Matched against the source directory relative path. In the matched
path, backslashes are
+     *                replaced with slash. If {@code null}, then matches everything.
+     */
+    public void setInclude(Pattern include) {
+        this.include = include;
+    }
+
+    /**
+     * Returns the default value of the {@linkplain #setInclude(Pattern) include property}.
+     */
+    protected abstract Pattern getDefaultInclude();
+
+    /**
+     * Getter pair of {@link #setExclude(Pattern)}.
+     */
+    public Pattern getExclude() {
+        return exclude;
+    }
+
+    /**
+     * Sets what subset of the files selected by the {@linkplain #setSource(File) source
property} will not be
+     * processed. Default is {@code null}.
+     *
+     * @param exclude Matched against the source directory relative path. In the matched
path, backslashes are
+     *                replaced with slash. If {@code null}, then matches noting (nothing
will be excluded).
+     */
+    public void setExclude(Pattern exclude) {
+        this.exclude = exclude;
+    }
+
+    /**
+     * Executes the file conversions.
+     */
     public final void execute() throws ConverterException {
         if (executed) {
             throw new IllegalStateException("This converted was already invoked once.");
@@ -143,6 +231,10 @@ public abstract class Converter {
     }
 
     private void convertFile(File src, File dstDir) throws ConverterException {
+        if (!isToBeProcessed(src)) {
+            return;
+        }
+
         InputStream srcStream;
         try {
             srcStream = new FileInputStream(src);
@@ -181,6 +273,30 @@ public abstract class Converter {
         }
     }
 
+    private boolean isToBeProcessed(File src) {
+        String relSrcPath;
+        File source = getSource();
+        if (source.isFile()) {
+            relSrcPath = source.getName();
+        } else {
+            relSrcPath = pathToStringWithSlashes(source.toPath().relativize(src.toPath()).normalize());
+        }
+
+        return (include == null || include.matcher(relSrcPath).matches())
+                && (exclude == null || !exclude.matcher(relSrcPath).matches());
+    }
+
+    private String pathToStringWithSlashes(Path path) {
+        StringBuilder sb = new StringBuilder();
+        for (Iterator<Path> it = path.iterator(); it.hasNext();) {
+            if (sb.length() != 0) {
+                sb.append('/');
+            }
+            sb.append(it.next());
+        }
+        return sb.toString();
+    }
+
     private void storeConversionMarkers(ConversionMarkers conversionMarkers, FileConversionContext
ctx)
             throws ConverterException {
         if (conversionMarkersWriter == null) {

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
index 1e024e5..0e2e114 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
@@ -24,20 +24,37 @@ import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.freemarker.core.util._NullArgumentException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import freemarker.core.FM2ASTToFM3SourceConverter;
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template._TemplateAPI;
 
+/**
+ * Converts FreeMarker 2 templates to FreeMarker 3 templates, as far as it's possible automatically.
While the output
+ * will contain syntactically correct FreeMarker 3 templates, the templates will have to
be reviewed by humans, as
+ * due to the semantic differences (such as a different treatment of {@code null}).
+ * <p>
+ * This is work in progress... new conversion are mostly only added when the syntactical
change was
+ * already implemented. Current conversions:
+ * <ul>
+ *     <li>All FreeMarker defined names are converted camel case
+ *     <li>Renamed setting names in the {@code ftl} heading and in {@code #setting}
tags are replaced with the new name.
+ *     <li>Renamed built-in variables are replaced with the new name
+ *     <li>From {@code #else}, {@code #elseif} and {@code #recover} tags that end with
{@code "/>"} or {@code "/]"} the
+ *         "/" characters is removed (as it's now illegal)
+ *     <li>{@code #else}, {@code #elseif} and {@code #recover} tags that end with {@code
"/>"} or {@code "/]"} the
+ *         "/" characters is removed (as it's now illegal)
+ *     <li>The last tag in {@code <#attempt>...<#recover>...</#recover>}
is replaced with {@code </#attempt>}
+ * </ul>
+ */
 public class FM2ToFM3Converter extends Converter {
 
-    public static final Logger LOG = LoggerFactory.getLogger(Converter.class);
+    private static final Pattern DEFAULT_INCLUDE = Pattern.compile("(?i).*\\.(fm|ftl(x|h)?)");
 
     private static final Map<String, String> DEFAULT_REPLACED_FILE_EXTENSIONS;
     static {
@@ -55,16 +72,23 @@ public class FM2ToFM3Converter extends Converter {
     private Configuration fm2Cfg;
 
     @Override
+    protected Pattern getDefaultInclude() {
+        return DEFAULT_INCLUDE;
+    }
+
+    @Override
     protected void prepare() throws ConverterException {
         super.prepare();
         fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial
unknown tags */);
         fm2Cfg.setWhitespaceStripping(false);
         fm2Cfg.setTabSize(1);
         _TemplateAPI.setPreventStrippings(fm2Cfg, true);
-        try {
-            fm2Cfg.setSettings(freeMarker2Settings);
-        } catch (Exception e) {
-            throw new ConverterException("Error while configuring FreeMarker 2", e);
+        if (freeMarker2Settings != null) {
+            try {
+                fm2Cfg.setSettings(freeMarker2Settings);
+            } catch (Exception e) {
+                throw new ConverterException("Error while configuring FreeMarker 2", e);
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
index 7d691c9..07af58b 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
@@ -426,7 +426,27 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
     }
 
     @Test
-    public void testFileExtensions() throws IOException, ConverterException {
+    public void testDefaultIncludes() throws IOException, ConverterException {
+        FileUtils.write(new File(srcDir, "t.txt"), "x", UTF_8);
+        FileUtils.write(new File(srcDir, "t.fm"), "x", UTF_8);
+        FileUtils.write(new File(srcDir, "t.ftl"), "x", UTF_8);
+        FileUtils.write(new File(srcDir, "t.ftlfoo"), "x", UTF_8);
+        FileUtils.write(new File(srcDir, "U.FTLH"), "x", UTF_8);
+
+        FM2ToFM3Converter converter = new FM2ToFM3Converter();
+        converter.setSource(srcDir);
+        converter.setDestinationDirectory(dstDir);
+        converter.execute();
+
+        assertFalse(new File(dstDir, "t.txt").exists());
+        assertTrue(new File(dstDir, "t.fm3").exists());
+        assertTrue(new File(dstDir, "t.fm3").exists());
+        assertFalse(new File(dstDir, "t.ftlfoo").exists());
+        assertTrue(new File(dstDir, "U.fm3h").exists());
+    }
+
+    @Test
+    public void testFileExtensionConversion() throws IOException, ConverterException {
         FileUtils.write(new File(srcDir, "t1"), "x", UTF_8);
         FileUtils.write(new File(srcDir, "t2.foo"), "x", UTF_8);
         FileUtils.write(new File(srcDir, "t3.ftl"), "x", UTF_8);
@@ -441,10 +461,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         FM2ToFM3Converter converter = new FM2ToFM3Converter();
         converter.setSource(srcDir);
         converter.setDestinationDirectory(dstDir);
-        Properties properties = new Properties();
-        properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name());
-        converter.setFreeMarker2Settings(properties);
-
+        converter.setInclude(null);
         converter.execute();
 
         assertTrue(new File(dstDir, "t1").exists());
@@ -496,6 +513,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
         FM2ToFM3Converter converter = new FM2ToFM3Converter();
         converter.setSource(srcFile);
         converter.setDestinationDirectory(dstDir);
+        converter.setInclude(null);
         Properties properties = new Properties();
         properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name());
         if (squareBracketTagSyntax) {

http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
index 7fcee1a..0873133 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.*;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.freemarker.converter.ConversionMarkers;
@@ -212,9 +213,47 @@ public class GenericConverterTest extends ConverterTest {
         }
     }
 
+    @Test
+    public void testIncludeAndExclude() throws IOException, ConverterException {
+        write(new File(srcDir, "included.txt"), "x", UTF_8);
+        write(new File(srcDir, "not-included.doc"), "x", UTF_8);
+        write(new File(srcDir, "~excluded.txt"), "x", UTF_8);
+
+        ToUpperCaseConverter converter = new ToUpperCaseConverter();
+        converter.setSource(srcDir);
+        converter.setDestinationDirectory(dstDir);
+        converter.setInclude(Pattern.compile("(?i).*\\.txt"));
+        converter.setExclude(Pattern.compile("(.*/)?~[^/]*"));
+        converter.execute();
+
+        assertTrue(new File(dstDir, "included.txt.uc").exists());
+        assertFalse(new File(dstDir, "not-included.doc.uc").exists());
+        assertFalse(new File(dstDir, "~excluded.txt.uc").exists());
+    }
+
+    @Test
+    public void testIncludeAndExclude2() throws IOException, ConverterException {
+        write(new File(srcDir, "included.txt"), "x", UTF_8);
+        write(new File(srcDir, "included2.doc"), "x", UTF_8);
+
+        ToUpperCaseConverter converter = new ToUpperCaseConverter();
+        converter.setSource(srcDir);
+        converter.setDestinationDirectory(dstDir);
+        converter.setInclude(null);
+        converter.execute();
+
+        assertTrue(new File(dstDir, "included.txt.uc").exists());
+        assertTrue(new File(dstDir, "included2.doc.uc").exists());
+    }
+
     public static class ToUpperCaseConverter extends Converter {
 
         @Override
+        protected Pattern getDefaultInclude() {
+            return Pattern.compile("(?i).*\\.txt");
+        }
+
+        @Override
         protected void convertFile(FileConversionContext ctx) throws ConverterException,
IOException {
             String content = IOUtils.toString(ctx.getSourceStream(), StandardCharsets.UTF_8);
             ctx.setDestinationFileName(ctx.getSourceFileName() + ".uc");


Mime
View raw message