ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stevenbeth...@apache.org
Subject svn commit: r1460892 - /incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java
Date Mon, 25 Mar 2013 21:03:26 GMT
Author: stevenbethard
Date: Mon Mar 25 21:03:26 2013
New Revision: 1460892

URL: http://svn.apache.org/r1460892
Log:
Adds logging of errors for event properties classifiers.

Modified:
    incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java

Modified: incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java
URL: http://svn.apache.org/viewvc/incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java?rev=1460892&r1=1460891&r2=1460892&view=diff
==============================================================================
--- incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java
(original)
+++ incubator/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java
Mon Mar 25 21:03:26 2013
@@ -19,11 +19,16 @@
 package org.apache.ctakes.temporal.eval;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.logging.FileHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
 
 import org.apache.ctakes.temporal.ae.DocTimeRelAnnotator;
 import org.apache.ctakes.typesystem.type.refsem.EventProperties;
@@ -44,6 +49,7 @@ import org.uimafit.pipeline.SimplePipeli
 import org.uimafit.util.JCasUtil;
 
 import com.google.common.base.Function;
+import com.google.common.collect.Maps;
 import com.lexicalscope.jewel.cli.CliFactory;
 
 public class EvaluationOfEventProperties extends
@@ -64,6 +70,7 @@ public class EvaluationOfEventProperties
         options.getKnowtatorXMLDirectory(),
         options.getXMIDirectory());
     evaluation.prepareXMIsFor(patientSets);
+    evaluation.logClassificationErrors(new File("target/eval"), "ctakes-event-property-errors");
     Map<String, AnnotationStatistics<String>> stats = evaluation.trainAndTest(trainItems,
devItems);
     for (String name : PROPERTY_NAMES) {
       System.err.println("====================");
@@ -73,12 +80,17 @@ public class EvaluationOfEventProperties
     }
   }
 
+  private Map<String, Logger> loggers = Maps.newHashMap();
+  
   public EvaluationOfEventProperties(
       File baseDirectory,
       File rawTextDirectory,
       File knowtatorXMLDirectory,
       File xmiDirectory) {
     super(baseDirectory, rawTextDirectory, knowtatorXMLDirectory, xmiDirectory);
+    for (String name : PROPERTY_NAMES) {
+      this.loggers.put(name, Logger.getLogger(String.format("%s.%s", this.getClass().getName(),
name)));
+    }
   }
 
   @Override
@@ -113,18 +125,58 @@ public class EvaluationOfEventProperties
     for (JCas jCas : new JCasIterable(collectionReader, aggregateBuilder.createAggregate()))
{
       JCas goldView = jCas.getView(GOLD_VIEW_NAME);
       JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA);
-      Collection<EventMention> goldEvents = selectExact(goldView, EventMention.class);
-      Collection<EventMention> systemEvents = selectExact(systemView, EventMention.class);
+      String text = goldView.getDocumentText();
+      List<EventMention> goldEvents = selectExact(goldView, EventMention.class);
+      List<EventMention> systemEvents = selectExact(systemView, EventMention.class);
       for (String name : PROPERTY_NAMES) {
+        Function<EventMention, String> getProperty = propertyGetters.get(name);
         statsMap.get(name).add(
             goldEvents,
             systemEvents,
             eventMentionToSpan,
-            propertyGetters.get(name));
+            getProperty);
+        for (int i = 0; i < goldEvents.size(); ++i) {
+          String goldOutcome = getProperty.apply(goldEvents.get(i));
+          String systemOutcome = getProperty.apply(systemEvents.get(i));
+          if (!goldOutcome.equals(systemOutcome)) {
+            EventMention event = goldEvents.get(i);
+            int begin = event.getBegin();
+            int end = event.getEnd();
+            int windowBegin = Math.max(0, begin - 50);
+            int windowEnd = Math.min(text.length(), end + 50);
+            this.loggers.get(name).fine(String.format(
+                "%s was %s but should be %s, in  ...%s[!%s!]%s...",
+                name,
+                goldOutcome,
+                systemOutcome,
+                text.substring(windowBegin, begin).replaceAll("[\r\n]", " "),
+                text.substring(begin, end),
+                text.substring(end, windowEnd).replaceAll("[\r\n]", " ")));
+          }
+        }
       }
     }
     return statsMap;
   }
+  
+  public void logClassificationErrors(File outputDir, String outputFilePrefix) throws IOException
{
+    if (!outputDir.exists()) {
+      outputDir.mkdirs();
+    }
+    for (String name : PROPERTY_NAMES) {
+      Logger logger = this.loggers.get(name);
+      logger.setLevel(Level.FINE);
+      File outputFile = new File(outputDir, String.format("%s.%s.log", outputFilePrefix,
name));
+      FileHandler handler = new FileHandler(outputFile.getPath());
+      handler.setFormatter(new Formatter() {
+        @Override
+        public String format(LogRecord record) {
+          return record.getMessage() + '\n';
+        }
+      });
+      logger.addHandler(handler);
+    }
+  }
 
   private static Function<EventMention, String> getPropertyGetter(final String propertyName)
{
     return new Function<EventMention, String>() {



Mime
View raw message