ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tm...@apache.org
Subject svn commit: r1611024 - in /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal: ae/feature/coreference/ utils/
Date Wed, 16 Jul 2014 14:35:11 GMT
Author: tmill
Date: Wed Jul 16 14:35:11 2014
New Revision: 1611024

URL: http://svn.apache.org/r1611024
Log:
CTAKES-199: Moved some coref classes into temporal for event coreference.

Added:
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistSemFeatureExtractor.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistanceFeatureExtractor.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/StringMatchingFeatureExtractor.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/TokenFeatureExtractor.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/UMLSFeatureExtractor.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/utils/CorefConst.java

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistSemFeatureExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistSemFeatureExtractor.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistSemFeatureExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistSemFeatureExtractor.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,105 @@
+package org.apache.ctakes.temporal.ae.feature.coreference;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.ctakes.core.resource.FileLocator;
+import org.apache.ctakes.dependency.parser.util.DependencyUtility;
+import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
+import org.apache.ctakes.typesystem.type.syntax.BaseToken;
+import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.ctakes.utils.distsem.WordEmbeddings;
+import org.apache.ctakes.utils.distsem.WordVector;
+import org.apache.ctakes.utils.distsem.WordVectorReader;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.cleartk.classifier.Feature;
+import org.uimafit.util.JCasUtil;
+
+public class DistSemFeatureExtractor implements RelationFeaturesExtractor {
+
+  // default value is 0.5 (rather than 0.0) because we don't want to assume OOV words are
dissimilar
+  public static final double DEFAULT_SIM = 0.5;  
+  
+  private WordEmbeddings words = null;
+  
+  public DistSemFeatureExtractor() throws FileNotFoundException, IOException{
+    words = WordVectorReader.getEmbeddings(FileLocator.getAsStream("org/apache/ctakes/coreference/distsem/mimic_vectors.txt"));
+  }
+  
+  @Override
+  public List<Feature> extract(JCas jCas, IdentifiedAnnotation arg1,
+      IdentifiedAnnotation arg2) throws AnalysisEngineProcessException {
+    List<Feature> feats = new ArrayList<>();
+    
+    double sim = 0.0;
+//    double[] a1vec = getArgVector(arg1);
+//    double[] a2vec = getArgVector(arg2);
+//    
+//    if(a1vec != null && a2vec != null){
+//      for(int i = 0; i < a1vec.length; i++){
+//        sim += a1vec[i] * a2vec[i];
+//      }
+//    }else{
+//      sim = DEFAULT_SIM;
+//    }
+//    
+//    assert !Double.isNaN(sim);
+//    
+//    feats.add(new Feature("ARG_SIMILARITY_WORD2VEC", sim));
+    
+    ConllDependencyNode node1 = DependencyUtility.getNominalHeadNode(jCas, arg1);
+    ConllDependencyNode node2 = DependencyUtility.getNominalHeadNode(jCas, arg2);
+    String head1 = node1 != null ? node1.getCoveredText().toLowerCase() : null;
+    String head2 = node2 != null ? node2.getCoveredText().toLowerCase() : null;
+    if(head1 != null && head2 != null && words.containsKey(head1) &&
words.containsKey(head2)){
+      sim = words.getSimilarity(head1, head2);
+    }else{
+      sim = DEFAULT_SIM;
+    }
+    feats.add(new Feature("HEAD_SIMILARITY_WORD2VEC", sim));
+    
+    return feats;
+  }
+
+  
+  private double[] getArgVector(IdentifiedAnnotation arg){
+    double[] vec = null;
+    
+    Collection<BaseToken> tokens = JCasUtil.selectCovered(BaseToken.class, arg);
+    
+    for(BaseToken token : tokens){
+      WordVector wv = words.getVector(token.getCoveredText());
+      if(wv == null){
+        wv = words.getVector(token.getCoveredText().toLowerCase());
+      }
+      if(wv != null){
+        if(vec == null){
+          vec = new double[wv.size()];
+          Arrays.fill(vec, 0.0);
+        }
+        for(int i = 0; i < vec.length; i++){
+          vec[i] += wv.getValue(i);
+        }
+      }
+    }
+    
+    if(vec != null){
+      double len = 0.0;
+      for(int i = 0; i < vec.length; i++){
+        len += vec[i]*vec[i];
+      }
+      len = Math.sqrt(len);
+      assert !Double.isNaN(len);
+      for(int i = 0; i < vec.length; i++){
+        vec[i] /= len;
+      }
+    }
+    return vec;
+  }
+}

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistanceFeatureExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistanceFeatureExtractor.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistanceFeatureExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/DistanceFeatureExtractor.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,29 @@
+package org.apache.ctakes.temporal.ae.feature.coreference;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
+import org.apache.ctakes.temporal.utils.CorefConst;
+import org.apache.ctakes.typesystem.type.syntax.BaseToken;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.ctakes.typesystem.type.textspan.Sentence;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.cleartk.classifier.Feature;
+import org.uimafit.util.JCasUtil;
+
+public class DistanceFeatureExtractor implements RelationFeaturesExtractor {
+
+	@Override
+	public List<Feature> extract(JCas jCas, IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2) throws AnalysisEngineProcessException {
+		List<Feature> feats = new ArrayList<>();
+		feats.add(new Feature("TOK_DIST",
+				  JCasUtil.selectCovered(jCas, BaseToken.class, arg1.getBegin(), arg2.getEnd()).size()
/ (double)CorefConst.TOKEN_DIST));
+		feats.add(new Feature("SENT_DIST",
+				JCasUtil.selectCovered(jCas, Sentence.class, arg1.getBegin(), arg2.getEnd()).size() /
(double) CorefConst.NE_DIST));
+		return feats;
+	}
+
+}

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/StringMatchingFeatureExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/StringMatchingFeatureExtractor.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/StringMatchingFeatureExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/StringMatchingFeatureExtractor.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,98 @@
+package org.apache.ctakes.temporal.ae.feature.coreference;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
+import org.apache.ctakes.typesystem.type.syntax.BaseToken;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.tcas.Annotation;
+import org.cleartk.classifier.Feature;
+import org.uimafit.util.JCasUtil;
+
+public class StringMatchingFeatureExtractor implements
+		RelationFeaturesExtractor {
+
+	@Override
+	public List<Feature> extract(JCas jCas, IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2) throws AnalysisEngineProcessException {
+		List<Feature> feats = new ArrayList<>();
+		
+		String s1 = arg1.getCoveredText();
+		String s2 = arg2.getCoveredText();
+		Set<String> words1 = contentWords(arg1);
+		Set<String> words2 = contentWords(arg2);
+		
+		feats.add(new Feature("MATCH_EXACT",
+				s1.equalsIgnoreCase(s2)));
+		feats.add(new Feature("MATCH_START",
+				startMatch(s1,s2)));
+		feats.add(new Feature("MATCH_END",
+				endMatch(s1,s2)));
+		feats.add(new Feature("MATCH_SOON",
+				soonMatch(s1,s2)));
+		feats.add(new Feature("MATCH_OVERLAP",
+				wordOverlap(words1, words2)));
+		feats.add(new Feature("MATCH_SUBSTRING",
+				wordSubstring(words1, words2)));
+		return feats;
+	}
+
+	public static boolean startMatch (String a, String b) {
+		int ia = a.indexOf(" ");
+		int ib = b.indexOf(" ");
+		String aa = a.substring(0, ia==-1?(a.length()>5?5:a.length()):ia);
+		String bb = b.substring(0, ib==-1?(b.length()>5?5:b.length()):ib);
+		return aa.equalsIgnoreCase(bb);
+	}
+
+	public static boolean endMatch (String a, String b) {
+		int ia = a.lastIndexOf(" ");
+		int ib = b.lastIndexOf(" ");
+		String aa = a.substring(ia==-1?(a.length()>5?a.length()-5:0):ia);
+		String bb = b.substring(ib==-1?(b.length()>5?b.length()-5:0):ib);
+		return aa.equalsIgnoreCase(bb);
+	}
+
+	public static boolean soonMatch (String s1, String s2) {
+		String sl1 = nonDetSubstr(s1.toLowerCase());
+		String sl2 = nonDetSubstr(s2.toLowerCase());
+		return sl1.equals(sl2);
+	}
+
+	public static String nonDetSubstr (String s) {
+		if(s.startsWith("the ")) return s.substring(4);
+		if(s.startsWith("a ")) return s.substring(2);
+		if(s.startsWith("this ")) return s.substring(5);
+		if(s.startsWith("that ")) return s.substring(5);
+		if(s.startsWith("these ")) return s.substring(6);
+		if(s.startsWith("those ")) return s.substring(6);
+		return s;
+	}
+
+	public static boolean wordOverlap(Set<String> t1, Set<String> t2) {
+		for (String s : t2){
+			if (t1.contains(s)){
+				return true;
+			}
+		}
+		return false;
+	}
+
+	public static boolean wordSubstring(Set<String> t1, Set<String> t2){
+		// TODO
+		return false;
+	}
+	
+	public static Set<String> contentWords(Annotation a1){
+		Set<String> words = new HashSet<>();
+		for(BaseToken tok : JCasUtil.selectCovered(BaseToken.class, a1)){
+			words.add(tok.getCoveredText().toLowerCase());
+		}
+		return words;
+	}
+}

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/TokenFeatureExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/TokenFeatureExtractor.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/TokenFeatureExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/TokenFeatureExtractor.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,99 @@
+package org.apache.ctakes.temporal.ae.feature.coreference;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
+import org.apache.ctakes.typesystem.type.syntax.BaseToken;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.cleartk.classifier.Feature;
+import org.uimafit.util.JCasUtil;
+
+public class TokenFeatureExtractor implements RelationFeaturesExtractor {
+
+	@Override
+	public List<Feature> extract(JCas jCas, IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2) throws AnalysisEngineProcessException {
+		List<Feature> feats = new ArrayList<>();
+		
+		String s1 = arg1.getCoveredText().toLowerCase();
+		String s2 = arg2.getCoveredText().toLowerCase();
+		
+		boolean dem1 = isDemonstrative(s1);
+		boolean dem2 = isDemonstrative(s2);
+		
+		feats.add(new Feature("TOKEN_DEM1", dem1));
+		feats.add(new Feature("TOKEN_DEM2", dem2));
+		feats.add(new Feature("TOKEN_DEF1", isDefinite(s1)));
+		feats.add(new Feature("TOKEN_DEF2", isDefinite(s2)));
+		feats.add(new Feature("TOKEN_NUMAGREE",
+				numberSingular(arg1) == numberSingular(arg2)));
+		
+		String gen1 = getGender(s1);
+		String gen2 = getGender(s2);
+		feats.add(new Feature("TOKEN_GEN1", gen1));
+		feats.add(new Feature("TOKEN_GEN2", gen2));
+		feats.add(new Feature("TOKEN_GENAGREE", gen1.equals(gen2)));
+		
+		String p1 = getPerson(s1);
+		String p2 = getPerson(s2);
+		feats.add(new Feature("TOKEN_PERSON1", p1));
+		feats.add(new Feature("TOKEN_PERSON2", p2));
+		feats.add(new Feature("TOKEN_PERSONAGREE", p1.equals(p2)));
+		return feats;
+	}
+	
+	public static boolean isDemonstrative (String s) {
+		if (s.startsWith("this") ||
+				s.startsWith("that") ||
+				s.startsWith("these") ||
+				s.startsWith("those")){
+				return true;
+		}
+		return false;
+	}
+	
+	public static boolean isDefinite (String s) {
+		return s.startsWith("the ");
+	}
+
+	// FYI - old code used treebanknode types and found head using head rules filled in by the
parser
+	// not sure if there is an appreciable difference...
+	public static boolean numberSingular(IdentifiedAnnotation arg){
+		List<BaseToken> tokens = new ArrayList<>(JCasUtil.selectCovered(BaseToken.class,
arg));
+		for (int i = tokens.size()-1; i >=0; i--){
+			BaseToken t = tokens.get(i);
+			String pos = t.getPartOfSpeech();
+			if ("NN".equals(pos) || "NNP".equals(pos)){
+				return true;
+			}else if ("NNS".equals(pos) || "NNPS".equals(pos)){
+				return false;
+			}else if(t.getCoveredText().toLowerCase().equals("we")){
+			  return true;
+			}
+		}
+		return true;
+	}
+	
+	public static String getGender(String s1){
+	  if(s1.equals("he") || s1.equals("his") || s1.startsWith("mr.")) return "MALE";
+	  else if(s1.equals("she") || s1.equals("her") || s1.startsWith("mrs.") || s1.startsWith("ms."))
return "FEMALE";
+	  else return "NEUTER";
+	}
+	
+	public static String getPerson(String s1){
+	  if(s1.equals("i")) return "FIRST";
+	  else if(s1.equals("he") || s1.equals("she") || s1.equals("his") || s1.equals("her") ||
s1.equals("hers")){
+	    return "THIRD";
+	  }else if(s1.equals("you") || s1.equals("your")) return "SECOND";
+	  else if(s1.equals("we")) return "FIRST_PLURAL";
+	  else return "NONE";
+	}
+	
+	public static boolean getAnimate(String s1){
+	  if(s1.equals("i")) return true;
+	  return false;
+	}
+}

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/UMLSFeatureExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/UMLSFeatureExtractor.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/UMLSFeatureExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/coreference/UMLSFeatureExtractor.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,73 @@
+package org.apache.ctakes.temporal.ae.feature.coreference;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.ctakes.dependency.parser.util.DependencyUtility;
+import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
+import org.apache.ctakes.typesystem.type.refsem.UmlsConcept;
+import org.apache.ctakes.typesystem.type.syntax.ConllDependencyNode;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.ctakes.typesystem.type.textsem.Markable;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.cas.FSArray;
+import org.cleartk.classifier.Feature;
+import org.uimafit.util.JCasUtil;
+
+public class UMLSFeatureExtractor implements RelationFeaturesExtractor {
+
+	@Override
+	public List<Feature> extract(JCas jCas, IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2) throws AnalysisEngineProcessException {
+		List<Feature> feats = new ArrayList<>();
+		
+		if(arg1 instanceof Markable && arg2 instanceof Markable){
+		  // get the head of each markable
+		  ConllDependencyNode head1 = DependencyUtility.getNominalHeadNode(jCas, arg1);
+		  ConllDependencyNode head2 = DependencyUtility.getNominalHeadNode(jCas, arg2);
+		  
+		  if(head1 != null && head2 != null){
+		    List<IdentifiedAnnotation> ents1 = JCasUtil.selectCovering(jCas, IdentifiedAnnotation.class,
head1.getBegin(), head1.getEnd());
+		    List<IdentifiedAnnotation> ents2 = JCasUtil.selectCovering(jCas, IdentifiedAnnotation.class,
head2.getBegin(), head2.getEnd());
+
+		    for(IdentifiedAnnotation ent1 : ents1){
+		      for(IdentifiedAnnotation ent2 : ents2){
+		        if(alias(ent1, ent2)){
+		          feats.add(new Feature("UMLS_ALIAS", true));
+		          break;
+		        }
+		      }
+		    }
+		  }
+		}
+		return feats;
+	}
+
+	public static boolean alias(IdentifiedAnnotation a1, IdentifiedAnnotation a2){  
+	  if(a1 != null && a2 != null){
+	    FSArray fsa = a1.getOntologyConceptArr();
+	    if(fsa != null){
+	      HashSet<String> cuis = new HashSet<>();
+	      for(int i = 0; i < fsa.size(); i++){
+	        if(fsa.get(i) instanceof UmlsConcept){
+	          cuis.add(((UmlsConcept)fsa.get(i)).getCui());
+	        }
+	      }
+
+	      fsa = a2.getOntologyConceptArr();
+	      if(fsa != null){
+	        for(int i = 0; i < fsa.size(); i++){
+	          if(fsa.get(i) instanceof UmlsConcept){
+	            if(cuis.contains(((UmlsConcept)fsa.get(i)).getCui())){
+	              return true;
+	            }
+	          }
+	        }
+	      }
+	    }
+	  }
+		return false;
+	}
+}

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/utils/CorefConst.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/utils/CorefConst.java?rev=1611024&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/utils/CorefConst.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/utils/CorefConst.java
Wed Jul 16 14:35:11 2014
@@ -0,0 +1,6 @@
+package org.apache.ctakes.temporal.utils;
+
+public class CorefConst {
+	public static final int NE_DIST = 20;
+	public static final int TOKEN_DIST = 600;
+}



Mime
View raw message