incubator-flex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gordonsm...@apache.org
Subject svn commit: r1391636 - in /incubator/flex/falcon/trunk/compiler.tests: ./ unit-tests/ unit-tests/org/ unit-tests/org/apache/ unit-tests/org/apache/flex/ unit-tests/org/apache/flex/compiler/ unit-tests/org/apache/flex/compiler/internal/ unit-tests/org/a...
Date Fri, 28 Sep 2012 20:23:42 GMT
Author: gordonsmith
Date: Fri Sep 28 20:23:42 2012
New Revision: 1391636

URL: http://svn.apache.org/viewvc?rev=1391636&view=rev
Log:
Falcon: Added first JUnit test file (for MXMLTokenizer). Added compiler.tests project for
debugging unit tests in Eclipse. The ability to run unit tests in Ant will come next.

Added:
    incubator/flex/falcon/trunk/compiler.tests/.classpath   (with props)
    incubator/flex/falcon/trunk/compiler.tests/.project   (with props)
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/
    incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java
  (with props)

Added: incubator/flex/falcon/trunk/compiler.tests/.classpath
URL: http://svn.apache.org/viewvc/incubator/flex/falcon/trunk/compiler.tests/.classpath?rev=1391636&view=auto
==============================================================================
--- incubator/flex/falcon/trunk/compiler.tests/.classpath (added)
+++ incubator/flex/falcon/trunk/compiler.tests/.classpath Fri Sep 28 20:23:42 2012
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="unit-tests"/>
+	<classpathentry kind="lib" path="lib/junit-4.10.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="lib" path="/compiler/lib/commons-io.jar"/>
+	<classpathentry kind="src" path="/compiler"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

Propchange: incubator/flex/falcon/trunk/compiler.tests/.classpath
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/flex/falcon/trunk/compiler.tests/.project
URL: http://svn.apache.org/viewvc/incubator/flex/falcon/trunk/compiler.tests/.project?rev=1391636&view=auto
==============================================================================
--- incubator/flex/falcon/trunk/compiler.tests/.project (added)
+++ incubator/flex/falcon/trunk/compiler.tests/.project Fri Sep 28 20:23:42 2012
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>compiler.tests</name>
+	<comment></comment>
+	<projects>
+		<project>compiler</project>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

Propchange: incubator/flex/falcon/trunk/compiler.tests/.project
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java
URL: http://svn.apache.org/viewvc/incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java?rev=1391636&view=auto
==============================================================================
--- incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java
(added)
+++ incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java
Fri Sep 28 20:23:42 2012
@@ -0,0 +1,799 @@
+package org.apache.flex.compiler.internal.parsing.mxml;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.core.Is.is;
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.List;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.flex.compiler.parsing.MXMLTokenTypes;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class MXMLTokenizerTests
+{
+	/**
+	 * Lexes MXML code into MXML tokens.
+	 */
+	private MXMLToken[] lex(String code)
+	{
+		Reader reader = new StringReader(code);
+		MXMLTokenizer tokenizer = new MXMLTokenizer();
+		List<MXMLToken> tokens = tokenizer.parseTokens(reader);
+		IOUtils.closeQuietly(tokenizer);
+		return tokens.toArray(new MXMLToken[0]);
+	}
+	
+	/*
+	 * Smallest processing instruction.
+	 */
+	@Test
+	public void processingInstruction1()
+	{
+		String code = "<?xml?>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_PROCESSING_INSTRUCTION));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * An entire processing instruction is lexed as a single token.
+	 */
+	@Test
+	public void processingInstruction2()
+	{
+		String code = "<?xml version='1.0' encoding=\"utf-8\"?>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_PROCESSING_INSTRUCTION));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Smallest normal comment.
+	 * This is getting incorrectly lexed as an ASDoc comment.
+	 */
+	@Ignore
+	@Test
+	public void comment1()
+	{
+		String code = "<!---->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Normal comment with some whitespace.
+	 */
+	@Test
+	public void comment2()
+	{
+		String code = "<!-- -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Normal comment with some text, including whitespace.
+	 */
+	@Test
+	public void comment3()
+	{
+		String code = "<!-- xxx \t\r\nxxx -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Normal comment with a tag inside.
+	 */
+	@Test
+	public void comment4()
+	{
+		String code = "<!-- <a/> -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Normal comment with an entity inside.
+	 */
+	@Test
+	public void comment5()
+	{
+		String code = "<!-- &#65; -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Normal comment with a comment "nested" inside.
+	 * Note that the they don't actually nest.
+	 */
+	@Test
+	public void nestedComment()
+	{
+		String code = "<!-- <!-- --> -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is("<!-- <!-- -->"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("1 text", tokens[1].getText(), is(" -->"));
+	}
+	
+	/*
+	 * Smallest ASDoc comment.
+	 */
+	@Test
+	public void asdoc1()
+	{
+		String code = "<!----->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * ASDoc comment with some whitespace.
+	 */
+	@Test
+	public void asdoc2()
+	{
+		String code = "<!--- -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * ASDoc comment with some text.
+	 */
+	@Test
+	public void asdoc3()
+	{
+		String code = "<!--- xxx \t\r\nxxx -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * ASDoc comment with a tag inside.
+	 */
+	@Test
+	public void asdoc4()
+	{
+		String code = "<!--- <a/> -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * ASDoc comment with an entity inside.
+	 */
+	@Test
+	public void asdoc5()
+	{
+		String code = "<!---&#65; -->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * ASDoc comment with an ASDoc comment "nested" inside.
+	 * Note that they don't actually nest.
+	 */
+	@Test
+	public void nestedASDoc()
+	{
+		String code = "<!--- <!--- ---> --->";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_ASDOC_COMMENT));
+		assertThat("0 text", tokens[0].getText(), is("<!--- <!--- --->"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("1 text", tokens[1].getText(), is(" --->"));
+	}
+	
+	/*
+	 * Pure whitespace of various kinds. TODO: What are all the kinds?
+	 */
+	@Test
+	public void whitespace()
+	{
+		String code = " \t\r\n";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_WHITESPACE));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Letters with surrounding and contained whitespace are just text.
+	 */
+	@Test
+	public void text1()
+	{
+		String code = " xxx yyy ";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Digits with surrounding and contained whitespace are just text.
+	 */
+	@Test
+	public void text2()
+	{
+		String code = " 123 456 ";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * Punctuation characters other than < are just text.
+	 */
+	@Test
+	public void text3()
+	{
+		String code = "`~!@#$%^&*()-_=+[{]};:'\",./?\\|>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(1));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("0 text", tokens[0].getText(), is(code));
+	}
+	
+	/*
+	 * An empty tag has a start token and and end token.
+	 */
+	@Test
+	public void emptyTag1()
+	{
+		String code = "<a/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is("/>"));
+	}
+	
+	/*
+	 * Whitespace between the start and end is ignored.
+	 */
+	@Test
+	public void emptyTag2()
+	{
+		String code = "<a />";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is("/>"));
+	}
+	
+	/*
+	 * A single underscore is a legal tag name, at least in XML.
+	 */
+	@Test
+	public void emptyTagName1()
+	{
+		String code = "<_/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<_"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is("/>"));
+	}
+	
+	/*
+	 * A single colon is a legal tag name, at least in XML.
+	 */
+	@Test
+	public void emptyTagName2()
+	{
+		String code = "<:/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<:"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is("/>"));
+	}
+	
+	/*
+	 * The non-first character of a tag name can be letter, digit, underscore, colon, period,
or hyphen.
+	 * Hyphen doesn't currently work.
+	 * TODO: What other Unicode characters are allowed?
+	 */
+	@Ignore
+	@Test
+	public void emptyTagName3()
+	{
+		String code = "<azAZ19_:.-/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(2));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<azAZ19_:.-"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is("/>"));
+	}
+	
+	/*
+	 * Each attribute produces three tokens.
+	 * Note that the text of the attribute value includes the single or double quotes.
+	 */
+	@Test
+	public void emptyTagWithAttributes()
+	{
+		String code = "<a b='c' d=\"f\"/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(8));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("b"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'c'"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("4 text", tokens[4].getText(), is("d"));
+		
+		assertThat("5 type", tokens[5].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("5 text", tokens[5].getText(), is("="));
+		
+		assertThat("6 type", tokens[6].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("6 text", tokens[6].getText(), is("\"f\""));
+		
+		assertThat("7 type", tokens[7].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("7 text", tokens[7].getText(), is("/>"));
+	}
+	
+	/*
+	 * A single underscore is a legal attribute name, at least in XML.
+	 */
+	@Test
+	public void attributeName1()
+	{
+		String code = "<a _='c'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("_"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'c'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * A single colon is a legal attribute name, at least in XML.
+	 */
+	@Test
+	public void attributeName2()
+	{
+		String code = "<a :='c'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is(":"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'c'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * Attribute names have the same rules as tag names.
+	 */
+	@Ignore
+	@Test
+	public void attributeName3()
+	{
+		String code = "<a azAZ19_:.-='c'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("azAZ19_:.-"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'c'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+
+	/*
+	 * Whitespace as attribute value.
+	 */
+	@Test
+	public void attributeValue1()
+	{
+		String code = "<a b=' \t\r\n'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("b"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("' \t\r\n'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * > as attribute value.
+	 */
+	@Test
+	public void attributeValue2()
+	{
+		String code = "<a b='>'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("b"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'>'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * Entity as attribute value.
+	 */
+	@Test
+	public void attributeValue3()
+	{
+		String code = "<a b='&#65;'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("b"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'&#65;'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * Punctuation as attribute value. Note: < and the delimiter are not allowed.
+	 */
+	@Test
+	public void attributeValue4()
+	{
+		String code = "<a b='`~!@#$%^*()-_=+{}[];:\",./?\\|'/>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_NAME));
+	    assertThat("1 text", tokens[1].getText(), is("b"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_EQUALS));
+	    assertThat("2 text", tokens[2].getText(), is("="));
+		
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_STRING));
+	    assertThat("3 text", tokens[3].getText(), is("'`~!@#$%^*()-_=+{}[];:\",./?\\|'"));
+				
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is("/>"));
+	}
+	
+	/*
+	 * Open/close tags.
+	 */
+	@Test
+	public void openTag_closeTag()
+	{
+		String code = "<a></a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(4));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+		assertThat("2 text", tokens[2].getText(), is("</a"));
+		
+		assertThat("3 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("3 text", tokens[1].getText(), is(">"));
+	}
+	
+	/*
+	 * Open/close tags with whitespace content.
+	 */
+	@Test
+	public void openTag_ws_closeTag()
+	{
+		String code = "<a> \t\r\n</a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_WHITESPACE));
+		assertThat("2 text", tokens[2].getText(), is(" \t\r\n"));
+	    
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+		assertThat("3 text", tokens[3].getText(), is("</a"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is(">"));
+	}
+	
+	/*
+	 * Open/close tags with text content.
+	 */
+	@Test
+	public void openTag_text_closeTag1()
+	{
+		String code = "<a>xxx</a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("2 text", tokens[2].getText(), is("xxx"));
+	    
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+		assertThat("3 text", tokens[3].getText(), is("</a"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is(">"));
+	}
+	
+	/*
+	 * Open/close tags with text content that includes whitespace.
+	 */
+	@Test
+	public void openTag_text_closeTag2()
+	{
+		String code = "<a> \t\r\nxxx \t\r\n</a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(5));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_TEXT));
+		assertThat("2 text", tokens[2].getText(), is(" \t\r\nxxx \t\r\n"));
+	    
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+		assertThat("3 text", tokens[3].getText(), is("</a"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("4 text", tokens[4].getText(), is(">"));
+	}
+	
+	/*
+	 * Open/close tags with an empty tag as content.
+	 */
+	@Test
+	public void openTag_emptyTag_closeTag()
+	{
+		String code = "<a><b/></a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(6));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("2 text", tokens[2].getText(), is("<b"));
+	    
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_EMPTY_TAG_END));
+		assertThat("3 text", tokens[3].getText(), is("/>"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+	    assertThat("4 text", tokens[4].getText(), is("</a"));
+		
+	    assertThat("5 type", tokens[5].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("5 text", tokens[5].getText(), is(">"));
+	}
+	
+	/*
+	 * Open/close tags with open/close tags nested inside.
+	 */
+	@Test
+	public void openTag_openTag_closeTag_closeTag()
+	{
+		String code = "<a><b></b></a>";
+		MXMLToken[] tokens = lex(code);
+		
+		assertThat("count", tokens.length, is(8));
+		
+		assertThat("0 type", tokens[0].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("0 text", tokens[0].getText(), is("<a"));
+		
+		assertThat("1 type", tokens[1].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("1 text", tokens[1].getText(), is(">"));
+		
+	    assertThat("2 type", tokens[2].getType(), is(MXMLTokenTypes.TOKEN_OPEN_TAG_START));
+		assertThat("2 text", tokens[2].getText(), is("<b"));
+	    
+		assertThat("3 type", tokens[3].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+		assertThat("3 text", tokens[3].getText(), is(">"));
+		
+		assertThat("4 type", tokens[4].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+	    assertThat("4 text", tokens[4].getText(), is("</b"));
+		
+	    assertThat("5 type", tokens[5].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("5 text", tokens[5].getText(), is(">"));
+	   
+	    assertThat("6 type", tokens[6].getType(), is(MXMLTokenTypes.TOKEN_CLOSE_TAG_START));
+	    assertThat("6 text", tokens[6].getText(), is("</a"));
+	    
+	    assertThat("7 type", tokens[7].getType(), is(MXMLTokenTypes.TOKEN_TAG_END));
+	    assertThat("7 text", tokens[7].getText(), is(">"));
+	}
+}

Propchange: incubator/flex/falcon/trunk/compiler.tests/unit-tests/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizerTests.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message