incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1157646 - in /incubator/lcf/trunk/framework: ./ scriptengine/ scriptengine/src/ scriptengine/src/main/ scriptengine/src/main/java/ scriptengine/src/main/java/org/ scriptengine/src/main/java/org/apache/ scriptengine/src/main/java/org/apache...
Date Sun, 14 Aug 2011 20:21:23 GMT
Author: kwright
Date: Sun Aug 14 20:21:23 2011
New Revision: 1157646

URL: http://svn.apache.org/viewvc?rev=1157646&view=rev
Log:
Begin work on script engine.  This commit represents the lexical analysis part.

Added:
    incubator/lcf/trunk/framework/scriptengine/
    incubator/lcf/trunk/framework/scriptengine/src/
    incubator/lcf/trunk/framework/scriptengine/src/main/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
  (with props)
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
  (with props)
    incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
  (with props)
Modified:
    incubator/lcf/trunk/framework/build.xml

Modified: incubator/lcf/trunk/framework/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/build.xml?rev=1157646&r1=1157645&r2=1157646&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/build.xml (original)
+++ incubator/lcf/trunk/framework/build.xml Sun Aug 14 20:21:23 2011
@@ -102,6 +102,18 @@
         </javac>
     </target>
 
+    <target name="compile-script-engine" depends="compile-core">
+        <mkdir dir="build/script-engine/classes"/>
+        <javac srcdir="scriptengine/src/main/java" destdir="build/script-engine/classes"
target="1.5" source="1.5" debug="true" debuglevel="lines,vars,source">
+            <classpath>
+                 <fileset dir="lib"> 
+                    <include name="*.jar"/> 
+                </fileset>
+                <pathelement location="build/core/classes"/>
+            </classpath>
+        </javac>
+    </target>
+
     <target name="compile-authority-servlet" depends="compile-core,compile-agents,compile-pull-agent">
         <mkdir dir="build/authority-servlet/classes"/>
         <javac srcdir="authority-servlet/src/main/java" destdir="build/authority-servlet/classes"
target="1.5" source="1.5" debug="true" debuglevel="lines,vars,source">
@@ -193,6 +205,11 @@
         <jar destfile="build/jar/mcf-jetty-runner.jar" basedir="build/jetty-runner/classes"/>
     </target>
 
+    <target name="jar-script-engine" depends="compile-script-engine">
+        <mkdir dir="build/jar"/>
+        <jar destfile="build/jar/mcf-script-engine.jar" basedir="build/script-engine/classes"/>
+    </target>
+
     <target name="webapp-authority-service" depends="jar-core,jar-agents,jar-pull-agent,compile-authority-servlet">
         <mkdir dir="build/webapp/authority-service/WEB-INF/lib"/>
         <copy todir="build/webapp/authority-service/WEB-INF/lib">
@@ -367,6 +384,24 @@
         </copy>
     </target>
 
+    <target name="script-engine" depends="jar-script-engine,jar-core">
+        <mkdir dir="dist/script-engine/lib"/>
+        <copy todir="dist/script-engine/lib">
+            <fileset dir="lib">
+                <include name="commons-codec*.jar"/>
+                <include name="commons-collections*.jar"/>
+                <include name="commons-httpclient-mcf.jar"/>
+                <include name="commons-io*.jar"/>
+                <include name="commons-logging*.jar"/>
+            </fileset>
+        </copy>
+        <copy todir="dist/example/lib">
+            <fileset dir="build/jar">
+                <include name="mcf-core.jar"/>
+            </fileset>
+        </copy>
+    </target>
+    
     <target name="example" depends="war-crawler-ui,war-api-service,war-authority-service,jar-jetty-runner,jar-core,jar-agents,jar-pull-agent">
         <mkdir dir="dist/example/lib"/>
         <copy todir="dist/example/lib">
@@ -555,7 +590,7 @@
         </junit>
     </target>
 
-    <target name="build" depends="processes,war-authority-service,war-crawler-ui,war-api-service,example"/>
+    <target name="build" depends="processes,war-authority-service,war-crawler-ui,war-api-service,example,script-engine"/>
     
     <target name="all" depends="build,doc,build-tests,run-tests,run-tests-postgresql"/>
     

Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
(added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
Sun Aug 14 20:21:23 2011
@@ -0,0 +1,44 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/** This class tracks the position of a script execution, and allows
+* access to the characters in a nested file setup.
+*/
+public class Position
+{
+  protected TokenStream tokenStream;
+  protected int characterPosition;
+  
+  public Position(TokenStream s, int characterPosition)
+  {
+    this.tokenStream = s;
+    this.characterPosition = characterPosition;
+  }
+  
+  public TokenStream getTokenStream()
+  {
+    return tokenStream;
+  }
+  
+  public int getCharacterPosition()
+  {
+    return characterPosition;
+  }
+  
+}
\ No newline at end of file

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Position.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
(added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
Sun Aug 14 20:21:23 2011
@@ -0,0 +1,48 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+public class Token
+{
+  public static final int TOKEN_PUNCTUATION = 0;
+  public static final int TOKEN_STRING = 1;
+  
+  protected int tokenType;
+  protected String tokenValue;
+  
+  public Token(int type, String value)
+  {
+    this.tokenType = type;
+    this.tokenValue = tokenValue;
+  }
+  
+  public String getPunctuation()
+  {
+    if (tokenType == TOKEN_PUNCTUATION)
+      return tokenValue;
+    return null;
+  }
+  
+  public String getString()
+  {
+    if (tokenType == TOKEN_STRING)
+      return tokenValue;
+    return null;
+  }
+  
+}
\ No newline at end of file

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/Token.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java?rev=1157646&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
(added)
+++ incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
Sun Aug 14 20:21:23 2011
@@ -0,0 +1,170 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/** Convert a script into a stream of tokens.
+*/
+public class TokenStream
+{
+  protected String fileName;
+  protected String code;
+  protected int characterPosition = 0;
+  protected Token currentToken = null;
+  
+  public TokenStream(String fileName, String code)
+  {
+    this.fileName = fileName;
+    this.code = code;
+  }
+  
+  public int getCharacterPosition()
+  {
+    return characterPosition;
+  }
+  
+  public void setCharacterPosition(int characterPosition)
+  {
+    this.characterPosition = characterPosition;
+    currentToken = null;
+  }
+  
+  public Token peek()
+  {
+    if (currentToken == null)
+      currentToken = parseNextToken();
+    return currentToken;
+  }
+  
+  public void skip()
+  {
+    currentToken = null;
+  }
+  
+  public String getFileName()
+  {
+    return fileName;
+  }
+
+  // Protected methods
+  
+  protected Token parseNextToken()
+  {
+    // Skip to the start of the next token (or exit if no token start can be found)
+    while (true)
+    {
+      // Skip white space
+      while (characterPosition < code.length())
+      {
+        char x = code.charAt(characterPosition);
+        if (x > ' ')
+          break;
+        characterPosition++;
+      }
+      
+      // End of text?
+      if (characterPosition == code.length())
+        return null;
+      
+      // Is it a comment?
+      char y = code.charAt(characterPosition);
+      if (y == '#')
+      {
+        // Skip to the end of the line, or the end of the text.
+        while (characterPosition < code.length())
+        {
+          char x = code.charAt(characterPosition++);
+          if (x == '\n')
+            break;
+        }
+        continue;
+      }
+      break;
+    }
+
+    // At the start of the next token.
+    // Figure out what kind of token it is.  Currently we only care about punctuation
+    // and strings.  Furthermore, no distinction is made between quoted and unquoted
+    // strings.
+    char z = code.charAt(characterPosition);
+    if (isLegalTokenCharacter(z))
+    {
+      // Legal token character
+      StringBuilder tokenBuffer = new StringBuilder();
+      int quoteMark = -1;
+      while (characterPosition < code.length())
+      {
+        char x = code.charAt(characterPosition);
+        if (quoteMark == -1)
+        {
+          // Not yet in a quote
+          if (x <= ' ' || x == '#')
+            break;
+          if (x == '\'' || x == '"')
+          {
+            characterPosition++;
+            quoteMark = (int)x;
+          }
+          else if (isLegalTokenCharacter(x))
+          {
+            characterPosition++;
+            tokenBuffer.append(x);
+          }
+          else
+            break;
+        }
+        else
+        {
+          characterPosition++;
+          // In a quote.
+          if (((int)x) == quoteMark)
+            // End of the quotation
+            break;
+          if (x == '\\')
+          {
+            // Escape character
+            if (characterPosition < code.length())
+              tokenBuffer.append(code.charAt(characterPosition++));
+          }
+          else
+            tokenBuffer.append(x);
+        }
+      }
+      return new Token(Token.TOKEN_STRING,tokenBuffer.toString());
+    }
+    else
+    {
+      // Set a punctuation token
+      characterPosition++;
+      return new Token(Token.TOKEN_PUNCTUATION,new StringBuilder(z).toString());
+    }
+  }
+  
+  protected static boolean isLegalTokenCharacter(char x)
+  {
+    if (x >= '0' && x <= '9')
+      return true;
+    if (x >= 'a' && x <= 'z')
+      return true;
+    if (x >= 'A' && x <='Z')
+      return true;
+    if (x == '_' || x == '$' || x == '@')
+      return true;
+    return false;
+  }
+  
+}

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/scriptengine/src/main/java/org/apache/manifoldcf/scriptengine/TokenStream.java
------------------------------------------------------------------------------
    svn:keywords = Id



Mime
View raw message