Return-Path: X-Original-To: apmail-ctakes-commits-archive@www.apache.org Delivered-To: apmail-ctakes-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 46EB910899 for ; Fri, 7 Jun 2013 22:12:19 +0000 (UTC) Received: (qmail 88299 invoked by uid 500); 7 Jun 2013 22:12:19 -0000 Delivered-To: apmail-ctakes-commits-archive@ctakes.apache.org Received: (qmail 88275 invoked by uid 500); 7 Jun 2013 22:12:19 -0000 Mailing-List: contact commits-help@ctakes.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ctakes.apache.org Delivered-To: mailing list commits@ctakes.apache.org Received: (qmail 88267 invoked by uid 99); 7 Jun 2013 22:12:19 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Jun 2013 22:12:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Jun 2013 22:12:16 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 52F3623888D2; Fri, 7 Jun 2013 22:11:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1490852 - /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/ Date: Fri, 07 Jun 2013 22:11:56 -0000 To: commits@ctakes.apache.org From: stevenbethard@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130607221157.52F3623888D2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: stevenbethard Date: Fri Jun 7 22:11:56 2013 New Revision: 1490852 URL: http://svn.apache.org/r1490852 Log: Makes ctakes-temporal evaluations Segment-aware. Annotators already were, but were being evaluated on, e.g. TimeMentions from outside of segments. Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfAnnotationSpans_ImplBase.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKEventSpans.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKTimeSpans.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventSpans.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfNEPredicateEventSpans.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfTimeSpans.java ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/Evaluation_ImplBase.java Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfAnnotationSpans_ImplBase.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfAnnotationSpans_ImplBase.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfAnnotationSpans_ImplBase.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfAnnotationSpans_ImplBase.java Fri Jun 7 22:11:56 2013 @@ -31,6 +31,7 @@ import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.cas.CAS; import org.apache.uima.collection.CollectionReader; @@ -42,6 +43,7 @@ import org.cleartk.util.ViewURIUtil; import org.uimafit.factory.AggregateBuilder; import org.uimafit.pipeline.JCasIterable; import org.uimafit.pipeline.SimplePipeline; +import org.uimafit.util.JCasUtil; import com.google.common.base.Function; import com.google.common.collect.Ordering; @@ -104,9 +106,9 @@ protected abstract AnalysisEngineDescrip protected abstract AnalysisEngineDescription getAnnotatorDescription(File directory) throws ResourceInitializationException; - protected abstract Collection getGoldAnnotations(JCas jCas); + protected abstract Collection getGoldAnnotations(JCas jCas, Segment segment); - protected abstract Collection getSystemAnnotations(JCas jCas); + protected abstract Collection getSystemAnnotations(JCas jCas, Segment segment); @Override protected AnnotationStatistics test(CollectionReader collectionReader, File directory) @@ -125,49 +127,53 @@ protected abstract AnalysisEngineDescrip for (JCas jCas : new JCasIterable(collectionReader, aggregateBuilder.createAggregate())) { JCas goldView = jCas.getView(GOLD_VIEW_NAME); JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA); - Collection goldAnnotations = this.getGoldAnnotations(goldView); - Collection systemAnnotations = this.getSystemAnnotations(systemView); - stats.add(goldAnnotations, systemAnnotations); - - Set goldSet = new TreeSet(bySpans); - for (Annotation goldAnnotation : goldAnnotations) { - // TODO: fix data so that this is not necessary - if (goldAnnotation.getBegin() == Integer.MAX_VALUE || goldAnnotation.getEnd() == Integer.MIN_VALUE) { - this.logger.warning("Invalid annotation"); - continue; - } - goldSet.add(goldAnnotation); - } - //goldSet.addAll(goldAnnotations); - Set systemSet = new TreeSet(bySpans); - systemSet.addAll(systemAnnotations); - - Set goldOnly = new TreeSet(bySpans); - goldOnly.addAll(goldSet); - goldOnly.removeAll(systemSet); - - Set systemOnly = new TreeSet(bySpans); - systemOnly.addAll(systemSet); - systemOnly.removeAll(goldSet); - - String text = jCas.getDocumentText().replaceAll("[\r\n]", " "); - if (!goldOnly.isEmpty() || !systemOnly.isEmpty()) { - this.logger.fine("Errors in : " + ViewURIUtil.getURI(jCas).toString()); - Set errors = new TreeSet(bySpans); - errors.addAll(goldOnly); - errors.addAll(systemOnly); - for (Annotation annotation : errors) { - int begin = annotation.getBegin(); - int end = annotation.getEnd(); - int windowBegin = Math.max(0, begin - 50); - int windowEnd = Math.min(text.length(), end + 50); - String label = goldOnly.contains(annotation) ? "DROPPED:" : "ADDED: "; - this.logger.fine(String.format( - "%s ...%s[!%s!]%s...", - label, - text.substring(windowBegin, begin), - text.substring(begin, end), - text.substring(end, windowEnd))); + for (Segment segment : JCasUtil.select(jCas, Segment.class)) { + if (!THYMEData.SEGMENTS_TO_SKIP.contains(segment.getId())) { + Collection goldAnnotations = this.getGoldAnnotations(goldView, segment); + Collection systemAnnotations = this.getSystemAnnotations(systemView, segment); + stats.add(goldAnnotations, systemAnnotations); + + Set goldSet = new TreeSet(bySpans); + for (Annotation goldAnnotation : goldAnnotations) { + // TODO: fix data so that this is not necessary + if (goldAnnotation.getBegin() == Integer.MAX_VALUE || goldAnnotation.getEnd() == Integer.MIN_VALUE) { + this.logger.warning("Invalid annotation"); + continue; + } + goldSet.add(goldAnnotation); + } + //goldSet.addAll(goldAnnotations); + Set systemSet = new TreeSet(bySpans); + systemSet.addAll(systemAnnotations); + + Set goldOnly = new TreeSet(bySpans); + goldOnly.addAll(goldSet); + goldOnly.removeAll(systemSet); + + Set systemOnly = new TreeSet(bySpans); + systemOnly.addAll(systemSet); + systemOnly.removeAll(goldSet); + + String text = jCas.getDocumentText().replaceAll("[\r\n]", " "); + if (!goldOnly.isEmpty() || !systemOnly.isEmpty()) { + this.logger.fine("Errors in : " + ViewURIUtil.getURI(jCas).toString()); + Set errors = new TreeSet(bySpans); + errors.addAll(goldOnly); + errors.addAll(systemOnly); + for (Annotation annotation : errors) { + int begin = annotation.getBegin(); + int end = annotation.getEnd(); + int windowBegin = Math.max(0, begin - 50); + int windowEnd = Math.min(text.length(), end + 50); + String label = goldOnly.contains(annotation) ? "DROPPED:" : "ADDED: "; + this.logger.fine(String.format( + "%s ...%s[!%s!]%s...", + label, + text.substring(windowBegin, begin), + text.substring(begin, end), + text.substring(end, windowEnd))); + } + } } } } Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKEventSpans.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKEventSpans.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKEventSpans.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKEventSpans.java Fri Jun 7 22:11:56 2013 @@ -24,6 +24,7 @@ import java.util.List; import java.util.logging.Level; import org.apache.ctakes.typesystem.type.textsem.EventMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; @@ -95,12 +96,12 @@ public class EvaluationOfClearTKEventSpa } @Override - protected Collection getGoldAnnotations(JCas jCas) { - return selectExact(jCas, EventMention.class); + protected Collection getGoldAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, EventMention.class, segment); } @Override - protected Collection getSystemAnnotations(JCas jCas) { - return JCasUtil.select(jCas, Event.class); + protected Collection getSystemAnnotations(JCas jCas, Segment segment) { + return JCasUtil.selectCovered(jCas, Event.class, segment); } } Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKTimeSpans.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKTimeSpans.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKTimeSpans.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfClearTKTimeSpans.java Fri Jun 7 22:11:56 2013 @@ -24,6 +24,7 @@ import java.util.List; import java.util.logging.Level; import org.apache.ctakes.typesystem.type.textsem.TimeMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; @@ -93,12 +94,12 @@ public class EvaluationOfClearTKTimeSpan } @Override - protected Collection getGoldAnnotations(JCas jCas) { - return JCasUtil.select(jCas, TimeMention.class); + protected Collection getGoldAnnotations(JCas jCas, Segment segment) { + return JCasUtil.selectCovered(jCas, TimeMention.class, segment); } @Override - protected Collection getSystemAnnotations(JCas jCas) { - return JCasUtil.select(jCas, Time.class); + protected Collection getSystemAnnotations(JCas jCas, Segment segment) { + return JCasUtil.selectCovered(jCas, Time.class, segment); } } Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventProperties.java Fri Jun 7 22:11:56 2013 @@ -34,6 +34,7 @@ import org.apache.ctakes.temporal.ae.Doc import org.apache.ctakes.typesystem.type.refsem.EventProperties; import org.apache.ctakes.typesystem.type.textsem.EventMention; import org.apache.ctakes.typesystem.type.textsem.TimeMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineProcessException; import org.apache.uima.cas.CAS; import org.apache.uima.cas.Feature; @@ -130,32 +131,36 @@ public class EvaluationOfEventProperties JCas goldView = jCas.getView(GOLD_VIEW_NAME); JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA); String text = goldView.getDocumentText(); - List goldEvents = selectExact(goldView, EventMention.class); - List systemEvents = selectExact(systemView, EventMention.class); - for (String name : PROPERTY_NAMES) { - Function getProperty = propertyGetters.get(name); - statsMap.get(name).add( - goldEvents, - systemEvents, - eventMentionToSpan, - getProperty); - for (int i = 0; i < goldEvents.size(); ++i) { - String goldOutcome = getProperty.apply(goldEvents.get(i)); - String systemOutcome = getProperty.apply(systemEvents.get(i)); - if (!goldOutcome.equals(systemOutcome)) { - EventMention event = goldEvents.get(i); - int begin = event.getBegin(); - int end = event.getEnd(); - int windowBegin = Math.max(0, begin - 50); - int windowEnd = Math.min(text.length(), end + 50); - this.loggers.get(name).fine(String.format( - "%s was %s but should be %s, in ...%s[!%s!]%s...", - name, - systemOutcome, - goldOutcome, - text.substring(windowBegin, begin).replaceAll("[\r\n]", " "), - text.substring(begin, end), - text.substring(end, windowEnd).replaceAll("[\r\n]", " "))); + for (Segment segment : JCasUtil.select(jCas, Segment.class)) { + if (!THYMEData.SEGMENTS_TO_SKIP.contains(segment.getId())) { + List goldEvents = selectExact(goldView, EventMention.class, segment); + List systemEvents = selectExact(systemView, EventMention.class, segment); + for (String name : PROPERTY_NAMES) { + Function getProperty = propertyGetters.get(name); + statsMap.get(name).add( + goldEvents, + systemEvents, + eventMentionToSpan, + getProperty); + for (int i = 0; i < goldEvents.size(); ++i) { + String goldOutcome = getProperty.apply(goldEvents.get(i)); + String systemOutcome = getProperty.apply(systemEvents.get(i)); + if (!goldOutcome.equals(systemOutcome)) { + EventMention event = goldEvents.get(i); + int begin = event.getBegin(); + int end = event.getEnd(); + int windowBegin = Math.max(0, begin - 50); + int windowEnd = Math.min(text.length(), end + 50); + this.loggers.get(name).fine(String.format( + "%s was %s but should be %s, in ...%s[!%s!]%s...", + name, + systemOutcome, + goldOutcome, + text.substring(windowBegin, begin).replaceAll("[\r\n]", " "), + text.substring(begin, end), + text.substring(end, windowEnd).replaceAll("[\r\n]", " "))); + } + } } } } Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventSpans.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventSpans.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventSpans.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventSpans.java Fri Jun 7 22:11:56 2013 @@ -26,6 +26,7 @@ import java.util.logging.Level; import org.apache.ctakes.temporal.ae.EventAnnotator; import org.apache.ctakes.temporal.ae.feature.selection.FeatureSelection; import org.apache.ctakes.typesystem.type.textsem.EventMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; @@ -134,12 +135,12 @@ public class EvaluationOfEventSpans exte } @Override - protected Collection getGoldAnnotations(JCas jCas) { - return selectExact(jCas, EventMention.class); + protected Collection getGoldAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, EventMention.class, segment); } @Override - protected Collection getSystemAnnotations(JCas jCas) { - return selectExact(jCas, EventMention.class); + protected Collection getSystemAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, EventMention.class, segment); } } Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfNEPredicateEventSpans.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfNEPredicateEventSpans.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfNEPredicateEventSpans.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfNEPredicateEventSpans.java Fri Jun 7 22:11:56 2013 @@ -7,6 +7,7 @@ import java.util.logging.Level; import org.apache.ctakes.temporal.ae.NEPredicateEventAnnotator; import org.apache.ctakes.typesystem.type.textsem.EventMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; @@ -59,13 +60,13 @@ public class EvaluationOfNEPredicateEven } @Override - protected Collection getGoldAnnotations(JCas jCas) { - return selectExact(jCas, EventMention.class); + protected Collection getGoldAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, EventMention.class, segment); } @Override - protected Collection getSystemAnnotations(JCas jCas) { - return selectExact(jCas, EventMention.class); + protected Collection getSystemAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, EventMention.class, segment); } /** Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfTimeSpans.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfTimeSpans.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfTimeSpans.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfTimeSpans.java Fri Jun 7 22:11:56 2013 @@ -27,6 +27,7 @@ import java.util.logging.Level; import org.apache.ctakes.temporal.ae.ConstituencyBasedTimeAnnotator; import org.apache.ctakes.temporal.ae.TimeAnnotator; import org.apache.ctakes.typesystem.type.textsem.TimeMention; +import org.apache.ctakes.typesystem.type.textspan.Segment; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; @@ -144,13 +145,13 @@ public class EvaluationOfTimeSpans exten } @Override - protected Collection getGoldAnnotations(JCas jCas) { - return selectExact(jCas, TimeMention.class); + protected Collection getGoldAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, TimeMention.class, segment); } @Override - protected Collection getSystemAnnotations(JCas jCas) { - return selectExact(jCas, TimeMention.class); + protected Collection getSystemAnnotations(JCas jCas, Segment segment) { + return selectExact(jCas, TimeMention.class, segment); } private File getModelDirectory(File directory) { Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/Evaluation_ImplBase.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/Evaluation_ImplBase.java?rev=1490852&r1=1490851&r2=1490852&view=diff ============================================================================== --- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/Evaluation_ImplBase.java (original) +++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/Evaluation_ImplBase.java Fri Jun 7 22:11:56 2013 @@ -423,9 +423,9 @@ public abstract class Evaluation_ImplBas return aggregateBuilder; } - public static List selectExact(JCas jCas, Class annotationClass) { + public static List selectExact(JCas jCas, Class annotationClass, Segment segment) { List annotations = Lists.newArrayList(); - for (T annotation : JCasUtil.select(jCas, annotationClass)) { + for (T annotation : JCasUtil.selectCovered(jCas, annotationClass, segment)) { if (annotation.getClass().equals(annotationClass)) { annotations.add(annotation); }