ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vjapa...@apache.org
Subject svn commit: r1555844 - /ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
Date Mon, 06 Jan 2014 15:03:39 GMT
Author: vjapache
Date: Mon Jan  6 15:03:38 2014
New Revision: 1555844

URL: http://svn.apache.org/r1555844
Log:
remove newlines within sentences from the token list

Modified:
    ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java

Modified: ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
URL: http://svn.apache.org/viewvc/ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java?rev=1555844&r1=1555843&r2=1555844&view=diff
==============================================================================
--- ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
(original)
+++ ctakes/branches/ytex/ctakes-dependency-parser/src/main/java/org/apache/ctakes/dependency/parser/ae/ClearNLPSemanticRoleLabelerAE.java
Mon Jan  6 15:03:38 2014
@@ -30,6 +30,7 @@ import org.apache.ctakes.core.resource.F
 import org.apache.ctakes.core.util.ListFactory;
 import org.apache.ctakes.typesystem.type.syntax.BaseToken;
 import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
+import org.apache.ctakes.typesystem.type.syntax.NewlineToken;
 import org.apache.ctakes.typesystem.type.textsem.Predicate;
 import org.apache.ctakes.typesystem.type.textsem.SemanticArgument;
 import org.apache.ctakes.typesystem.type.textsem.SemanticRoleRelation;
@@ -169,30 +170,36 @@ final String language = AbstractReader.L
       int[] headIDs = new int[tokens.size()];
       String[] deprels = new String[tokens.size()];
 
-      // Initialize Token / Sentence info for the ClearNLP Semantic Role Labeler
+      // Initialize Token / Sentence info for the ClearNLP Semantic Role Labeler
+      // we are filtering out newline tokens
+      // use idIter as the non-newline token index counter 
+      int idIter = 0;
       for (int i = 0; i < tokens.size(); i++) {
-        BaseToken token = tokens.get(i);
-
-        // Determine HeadId
-        List<ConllDependencyNode> casDepNodes = JCasUtil.selectCovered(jCas, ConllDependencyNode.class,
token);
-        
-        ConllDependencyNode casDepNode = casDepNodes.get(0);
-        if(casDepNode.getId() == 0) casDepNode = casDepNodes.get(1);
-
-        deprels[i] = casDepNode.getDeprel();
-        ConllDependencyNode head = casDepNode.getHead();
-
-        // If there is no head, this is the head node, set node to 0
-        headIDs[i] = (head == null) ? 0 : depNodeToID.get(head);
-
-        // Populate Dependency Node / Tree information
-        int id = i + 1;
-        String form = casDepNode.getForm();
-        String pos = casDepNode.getPostag();
-        String lemma = casDepNode.getLemma();
-        
-        DEPNode node = new DEPNode(id, form, lemma, pos, new DEPFeat());
-        tree.add(node);
+        BaseToken token = tokens.get(i);
+        // ignore newline tokens within a sentence - newline = whitespace = non-token
+        if(!(token instanceof NewlineToken)) {
+	        // Determine HeadId
+	        List<ConllDependencyNode> casDepNodes = JCasUtil.selectCovered(jCas, ConllDependencyNode.class,
token);
+	        
+	        ConllDependencyNode casDepNode = casDepNodes.get(0);
+	        if(casDepNode.getId() == 0) casDepNode = casDepNodes.get(1);
+	
+	        deprels[i] = casDepNode.getDeprel();
+	        ConllDependencyNode head = casDepNode.getHead();
+	
+	        // If there is no head, this is the head node, set node to 0
+	        headIDs[i] = (head == null) ? 0 : depNodeToID.get(head);
+	
+	        // Populate Dependency Node / Tree information
+	        int id = idIter + 1;
+	        String form = casDepNode.getForm();
+	        String pos = casDepNode.getPostag();
+	        String lemma = casDepNode.getLemma();
+	        
+	        DEPNode node = new DEPNode(id, form, lemma, pos, new DEPFeat());
+	        tree.add(node);
+	        idIter++;
+        }
       }
 
       for (int i=1; i<tree.size(); i++)



Mime
View raw message