flex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aha...@apache.org
Subject git commit: [flex-falcon] [refs/heads/develop] - FLEX-35183 FLEX-35184 fix unterminated CDATA and Comments in MXML and E4X
Date Wed, 23 Nov 2016 23:29:32 GMT
Repository: flex-falcon
Updated Branches:
  refs/heads/develop e28b138d4 -> c040b5c88


FLEX-35183 FLEX-35184 fix unterminated CDATA and Comments in MXML and E4X


Project: http://git-wip-us.apache.org/repos/asf/flex-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/flex-falcon/commit/c040b5c8
Tree: http://git-wip-us.apache.org/repos/asf/flex-falcon/tree/c040b5c8
Diff: http://git-wip-us.apache.org/repos/asf/flex-falcon/diff/c040b5c8

Branch: refs/heads/develop
Commit: c040b5c88980130cd43fb903d52f9cdef59a7101
Parents: e28b138
Author: Alex Harui <aharui@apache.org>
Authored: Wed Nov 23 15:29:10 2016 -0800
Committer: Alex Harui <aharui@apache.org>
Committed: Wed Nov 23 15:29:25 2016 -0800

----------------------------------------------------------------------
 .../compiler/internal/caches/MXMLDataCache.java |  2 +
 .../internal/parsing/as/BaseRawASTokenizer.java | 22 +++++++
 .../parsing/mxml/BaseRawMXMLTokenizer.java      | 41 +++++++++++++
 .../internal/parsing/mxml/MXMLScopeBuilder.java |  1 +
 .../internal/parsing/mxml/MXMLTokenizer.java    |  2 +
 .../problems/CDataNotClosedProblem.java         | 41 +++++++++++++
 .../problems/CommentNotClosedProblem.java       | 41 +++++++++++++
 .../problems/MXMLUnclosedTagProblem.java        |  2 +-
 .../internal/parsing/as/RawASTokenizer.lex      | 48 +++------------
 .../internal/parsing/mxml/RawMXMLTokenizer.lex  | 64 ++++++--------------
 10 files changed, 179 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
index 7f06a2d..af7f4a2 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
@@ -86,6 +86,8 @@ public class MXMLDataCache extends ConcurrentCacheStoreBase<MXMLData>
             
             // Build tags and attributes from the tokens.
             final MXMLData mxmlData = new MXMLData(tokens, tokenizer.getPrefixMap(), fileSpec);
+            if (tokenizer.hasTokenizationProblems())
+            	mxmlData.getProblems().addAll(tokenizer.getTokenizationProblems());
             return mxmlData;
         }
         catch (FileNotFoundException e)

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
index cf22ce6..3a80b66 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
@@ -21,6 +21,8 @@ package org.apache.flex.compiler.internal.parsing.as;
 
 import org.apache.flex.compiler.common.ISourceLocation;
 import org.apache.flex.compiler.problems.ASDocNotClosedProblem;
+import org.apache.flex.compiler.problems.CDataNotClosedProblem;
+import org.apache.flex.compiler.problems.CommentNotClosedProblem;
 import org.apache.flex.compiler.problems.ICompilerProblem;
 import org.apache.flex.compiler.problems.StringLiteralMustBeTerminatedBeforeLineBreakProblem;
 import org.apache.flex.compiler.problems.StringLiteralNotClosedProblem;
@@ -340,6 +342,26 @@ public abstract class BaseRawASTokenizer extends BaseRawTokenizer<ASToken>
     }
 
     /**
+     * Report syntax error: input ended before Comment is closed.
+     */
+    protected final void reportUnclosedComment()
+    {
+        final ISourceLocation location = getCurrentSourceLocation(0);
+        final ICompilerProblem problem = new CommentNotClosedProblem(location);
+        getProblems().add(problem);
+    }
+
+    /**
+     * Report syntax error: input ended before CDATA is closed.
+     */
+    protected final void reportUnclosedCDATA()
+    {
+        final ISourceLocation location = getCurrentSourceLocation(0);
+        final ICompilerProblem problem = new CDataNotClosedProblem(location);
+        getProblems().add(problem);
+    }
+
+    /**
      * Convert escaped unicode sequence such as {@code \u00FF} and {@code \xFF}
      * to unicode code point.
      * 

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
index c6709c8..09277cf 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
@@ -24,7 +24,10 @@ import java.util.List;
 
 import org.apache.flex.compiler.internal.parsing.as.ASToken;
 import org.apache.flex.compiler.parsing.ICMToken;
+import org.apache.flex.compiler.problems.ASDocNotClosedProblem;
+import org.apache.flex.compiler.problems.CommentNotClosedProblem;
 import org.apache.flex.compiler.problems.ICompilerProblem;
+import org.apache.flex.compiler.problems.MXMLUnclosedTagProblem;
 
 import antlr.CommonToken;
 import antlr.Token;
@@ -59,6 +62,8 @@ public abstract class BaseRawMXMLTokenizer
 
     private Token lastToken = null;
 
+    protected String sourcePath = null;
+
     public BaseRawMXMLTokenizer()
     {
         // no-arg constructor
@@ -106,6 +111,7 @@ public abstract class BaseRawMXMLTokenizer
         CommonToken token = new CommonToken(type, text);
         token.setLine(line);
         token.setColumn(column);
+        token.setFilename(sourcePath);
         lastToken = token;
         return token;
     }
@@ -159,6 +165,11 @@ public abstract class BaseRawMXMLTokenizer
         return lastToken != null ? lastToken.getText() : "";
     }
 
+    public void setSourcePath(String sourcePath)
+    {
+        this.sourcePath = sourcePath;
+    }
+
     protected void setLastToken(Token token)
     {
         lastToken = token;
@@ -241,6 +252,36 @@ public abstract class BaseRawMXMLTokenizer
         return yytext() + " (" + line + ")";
     }
 
+    /**
+     * Report an unclosed entity problem
+     * 
+     * @param The token
+     */
+    protected void reportUnclosedCDATA(MXMLToken token)
+    {
+        getProblems().add(new MXMLUnclosedTagProblem((ASToken)token, "CDATA"));
+    }
+    
+    /**
+     * Report an unclosed entity problem
+     * 
+     * @param The token
+     */
+    protected void reportUnclosedComment(MXMLToken token)
+    {
+        getProblems().add(new CommentNotClosedProblem((ASToken)token));
+    }
+    
+    /**
+     * Report an unclosed entity problem
+     * 
+     * @param The token
+     */
+    protected void reportUnclosedASDocComment(MXMLToken token)
+    {
+        getProblems().add(new ASDocNotClosedProblem((ASToken)token));
+    }
+    
     protected List<ICompilerProblem> problems = null;
 
     /**

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
index 79e59c2..bd778e7 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
@@ -106,6 +106,7 @@ public class MXMLScopeBuilder
         fileScope.addDefinition(packageDefinition);
         
         problems = new LinkedList<ICompilerProblem>();
+        problems.addAll(mxmlData.getProblems());
 
         includeHandler = new IncludeHandler(fileSpecGetter);
         includeHandler.setProjectAndCompilationUnit(project, compilationUnit);

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
index 49a3fea..51718ca 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
@@ -89,6 +89,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable
 	public MXMLTokenizer(String path)
 	{
 	    tokenizer = new RawMXMLTokenizer();
+	    tokenizer.setSourcePath(path);
         problems = new ArrayList<ICompilerProblem>();
         rootPrefixMap = new MutablePrefixMap();
         this.path = path;
@@ -116,6 +117,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable
 	
 	public void setPath(String path) {
 	    this.path = path;
+	    tokenizer.setSourcePath(path);
 	}
 	
 	public void setReader(Reader reader) {

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
new file mode 100644
index 0000000..e542f3d
--- /dev/null
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
@@ -0,0 +1,41 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.flex.compiler.problems;
+
+import org.apache.flex.compiler.common.ISourceLocation;
+import org.apache.flex.compiler.problems.annotations.ProblemClassification;
+
+/**
+ * Syntax error: input ended before ASDoc is closed.
+ */
+@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR)
+public final class CDataNotClosedProblem extends ParserProblem
+{
+    public static final String DESCRIPTION =
+            "input ended before CDATA is closed.";
+
+    public static final int errorCode = 1549;
+
+    public CDataNotClosedProblem(ISourceLocation location)
+    {
+        super(location);
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
new file mode 100644
index 0000000..4c5954c
--- /dev/null
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
@@ -0,0 +1,41 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.flex.compiler.problems;
+
+import org.apache.flex.compiler.common.ISourceLocation;
+import org.apache.flex.compiler.problems.annotations.ProblemClassification;
+
+/**
+ * Syntax error: input ended before ASDoc is closed.
+ */
+@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR)
+public final class CommentNotClosedProblem extends ParserProblem
+{
+    public static final String DESCRIPTION =
+            "input ended before comment is closed.";
+
+    public static final int errorCode = 1549;
+
+    public CommentNotClosedProblem(ISourceLocation location)
+    {
+        super(location);
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
index dab73ad..b668100 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
@@ -30,7 +30,7 @@ import org.apache.flex.compiler.problems.annotations.ProblemClassification;
 public final class MXMLUnclosedTagProblem extends ParserProblem
 {
     public static final String DESCRIPTION =
-        "'${tokenText}' tag (or tag before this tag) is unclosed or tag not allowed here.";
+        "'${tokenText}' tag (or tag before this tag, or non-tag inside this tag) is unclosed
or tag not allowed here.";
 
     public static final int errorCode = 1552;
     

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
----------------------------------------------------------------------
diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
index f0fd6b6..5c93547 100644
--- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
@@ -988,27 +988,12 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
 	yybegin(MARKUP_IGNORE);
 }
 
-<CDATA> ([^\]])*
+<CDATA> [^]
 {
 	continueAggregate();
 }
 
-<CDATA> ("]"+[^\]>])
-{
-	continueAggregate();
-}
-
-<CDATA> {WHITE_SPACE_CHAR}+
-{
-	continueAggregate();
-}
-
-<CDATA> .
-{
-	continueAggregate();
-}
-
-<CDATA> ("]""]"+">")
+<CDATA> ~("]""]"+">")
 {
 	continueAggregate();
 	yybegin(E4X);
@@ -1017,32 +1002,16 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
 
 <CDATA><<EOF>>
 {
-	continueAggregate();
-	yybegin(E4X);
-	return buildAggregateToken(TOKEN_E4X_CDATA);
-}
-
-<E4XCOMMENT> ([^-])*
-{
-	continueAggregate();
-}
-
-<E4XCOMMENT> ("-"+[^->])
-{
-	continueAggregate();
+	reportUnclosedCDATA();
+	return null;
 }
 
-<E4XCOMMENT> {WHITE_SPACE_CHAR}+
+<E4XCOMMENT> [^]
 {
 	continueAggregate();
 }
 
-<E4XCOMMENT> .
-{
-	continueAggregate();
-}
-
-<E4XCOMMENT> ("-""-"+">")
+<E4XCOMMENT> ~("-""-"+">")
 {
 	continueAggregate();
 	yybegin(E4X);
@@ -1051,9 +1020,8 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
 
 <E4XCOMMENT><<EOF>>
 {
-	continueAggregate();
-	yybegin(E4X);
-	return buildAggregateToken(TOKEN_E4X_COMMENT);
+        reportUnclosedComment();
+	return null;
 }
 
 <DIRECTIVE> ([^?])*

http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
----------------------------------------------------------------------
diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
index 4c3a6f6..4447086 100644
--- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
@@ -81,7 +81,9 @@ public RawMXMLTokenizer()
 
 protected final Token buildToken(int type, int start, int end, int line, int column, String
text)
 {
-	return new MXMLToken(type, start, end, line, column, text);
+	MXMLToken token = new MXMLToken(type, start, end, line, column, text);
+	token.setSourcePath(sourcePath);
+	return token;
 }
 
 public void reset()
@@ -260,52 +262,39 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012]
 // before returning to the initial state.
 //
 
-<COMMENT,ASDOC_COMMENT> ([^-])*
-{
-	continueAggregate();
-}
-
-<COMMENT,ASDOC_COMMENT> ("-"+[^->])
-{
-	continueAggregate();
-}
-
-<COMMENT,ASDOC_COMMENT> {WHITE_SPACE_CHAR}+
+<COMMENT,ASDOC_COMMENT> [^]
 {
 	continueAggregate();
 }
 
-<COMMENT> ("-""-"+">")
+<COMMENT> ~("-""-"+">")
 {
 	continueAggregate();
 	yybegin(YYINITIAL);
 	return buildAggregateToken(TOKEN_COMMENT);
 }
 
-<ASDOC_COMMENT> ("-""-"+">")
+<ASDOC_COMMENT> ~("-""-"+">")
 {
 	continueAggregate();
 	yybegin(YYINITIAL);
 	return buildAggregateToken(TOKEN_ASDOC_COMMENT);
 }
 
-<COMMENT,ASDOC_COMMENT> .
-{
-	continueAggregate();
-}
-
 <COMMENT><<EOF>>
 {
-	continueAggregate();
-	yybegin(YYINITIAL);
-	return buildAggregateToken(TOKEN_COMMENT);
+	MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_COMMENT);
+	if (token != null)
+            reportUnclosedComment(token);
+	return token;
 }
 
 <ASDOC_COMMENT><<EOF>>
 {
-	continueAggregate();
-	yybegin(YYINITIAL);
-	return buildAggregateToken(TOKEN_ASDOC_COMMENT);
+	MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_ASDOC_COMMENT);
+        if (token != null)
+	    reportUnclosedASDocComment(token);
+	return token;
 }
 
 //
@@ -517,38 +506,25 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012]
 // before returning to the initial state.
 //
 
-<CDATA> ([^\]])*
+<CDATA> [^]
 {
 	continueAggregate();
 }
 
-<CDATA> ("]"+[^\]>])
-{
-	continueAggregate();
-}
-
-<CDATA> {WHITE_SPACE_CHAR}+
-{
-	continueAggregate();
-}
 
-<CDATA> ("]""]"+">")
+<CDATA> ~("]""]"+">")
 {
 	continueAggregate();
 	yybegin(YYINITIAL);
 	return buildAggregateToken(TOKEN_CDATA);
 }
 
-<CDATA> .
-{
-	continueAggregate();
-}
-
 <CDATA><<EOF>>
 {
-	continueAggregate();
-	yybegin(YYINITIAL);
-	return buildAggregateToken(TOKEN_CDATA);
+	MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_CDATA);
+        if (token != null)
+	    reportUnclosedCDATA(token);
+	return token;
 }
 
 //


Mime
View raw message