Return-Path: X-Original-To: apmail-ctakes-commits-archive@www.apache.org Delivered-To: apmail-ctakes-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7CCFA1056B for ; Wed, 14 Aug 2013 15:29:19 +0000 (UTC) Received: (qmail 23052 invoked by uid 500); 14 Aug 2013 15:29:19 -0000 Delivered-To: apmail-ctakes-commits-archive@ctakes.apache.org Received: (qmail 23013 invoked by uid 500); 14 Aug 2013 15:29:15 -0000 Mailing-List: contact commits-help@ctakes.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ctakes.apache.org Delivered-To: mailing list commits@ctakes.apache.org Received: (qmail 22755 invoked by uid 99); 14 Aug 2013 15:29:13 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Aug 2013 15:29:13 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Aug 2013 15:29:12 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id C0DE22388860; Wed, 14 Aug 2013 15:28:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1513928 - in /ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion: eval/AnnotationStatisticsCompact.java eval/AssertionEvaluation.java train/TestAttributeModels.java train/TrainAttributeModels.java Date: Wed, 14 Aug 2013 15:28:51 -0000 To: commits@ctakes.apache.org From: swu@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130814152851.C0DE22388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: swu Date: Wed Aug 14 15:28:51 2013 New Revision: 1513928 URL: http://svn.apache.org/r1513928 Log: overdue updates for assertion evaluation from shannon. Modified: ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AnnotationStatisticsCompact.java ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AssertionEvaluation.java ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TestAttributeModels.java ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TrainAttributeModels.java Modified: ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AnnotationStatisticsCompact.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AnnotationStatisticsCompact.java?rev=1513928&r1=1513927&r2=1513928&view=diff ============================================================================== --- ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AnnotationStatisticsCompact.java (original) +++ ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AnnotationStatisticsCompact.java Wed Aug 14 15:28:51 2013 @@ -438,7 +438,7 @@ public class AnnotationStatisticsCompact public String toTsv() { StringBuilder result = new StringBuilder(); result.append(String.format( - "%s:%.3f\t%s:%.3f\t%s:%.3f\t", + "%s\t%.3f\t%s\t%.3f\t%s\t%.3f\t", "All",this.precision(), "All",this.recall(), "All",this.f1())); @@ -447,7 +447,7 @@ public class AnnotationStatisticsCompact Collections.sort(outcomes); for (OUTCOME_TYPE outcome : outcomes) { result.append(String.format( - "%s:%.3f\t%s:%.3f\t%s:%.3f\t", + "%s\t%.3f\t%s\t%.3f\t%s\t%.3f\t", outcome,this.precision(outcome), outcome,this.recall(outcome), outcome,this.f1(outcome))); Modified: ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AssertionEvaluation.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AssertionEvaluation.java?rev=1513928&r1=1513927&r2=1513928&view=diff ============================================================================== --- ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AssertionEvaluation.java (original) +++ ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/eval/AssertionEvaluation.java Wed Aug 14 15:28:51 2013 @@ -116,7 +116,7 @@ public class AssertionEvaluation extends private static Logger logger = Logger.getLogger(AssertionEvaluation.class); - private static final String YTEX_NEGATION_DESCRIPTOR = "ytex/uima/NegexAnnotator.xml"; + private static final String YTEX_NEGATION_DESCRIPTOR = "ytex.uima.NegexAnnotator"; public static class Options extends Options_ImplBase { @Option( @@ -256,18 +256,15 @@ private static Logger logger = Logger.ge private String sharpCorpusDirectory; - static public File evaluationLogFile; - static public BufferedWriter evaluationLogFileOut; + static public String evaluationLogFilePath; + static private File evaluationLogFile; + static private BufferedWriter evaluationLogFileOut; static { - try { - evaluationLogFile = new File("eval_"+new Date().toString().replaceAll(" ","_") + ".txt"); - evaluationLogFileOut = new BufferedWriter(new FileWriter(evaluationLogFile), 32768); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + evaluationLogFilePath = "eval_"+new Date().toString().replaceAll(" ","_") + ".txt"; } + static public boolean useEvaluationLogFile = false; + private boolean ignoreAnatomicalSites = false; protected static Options options = new Options(); @@ -278,7 +275,12 @@ private static Logger logger = Logger.ge //Options options = new Options(); resetOptions(); options.parseOptions(args); - + + if (useEvaluationLogFile) { + evaluationLogFile = new File(evaluationLogFilePath); + evaluationLogFileOut = new BufferedWriter(new FileWriter(evaluationLogFile), 32768); + } + // System.err.println("forcing skipping of subject processing!!!"); // options.runSubject = false; // System.err.println("forcing skipping of generic processing!!!"); @@ -399,7 +401,7 @@ private static Logger logger = Logger.ge CollectionReader testCollectionReader = evaluation.getCollectionReader(testFiles); Map stats = evaluation.test(testCollectionReader, modelsDir); - AssertionEvaluation.printScore(stats, modelsDir.getAbsolutePath()); + AssertionEvaluation.printScore(stats, modelsDir!=null? modelsDir.getAbsolutePath() : "no_model"); } } @@ -895,8 +897,10 @@ public static void printScore(Map goldEntitiesAndEvents = new ArrayList(); - Collection goldEntities = JCasUtil.select(goldView, EntityMention.class); - goldEntitiesAndEvents.addAll(goldEntities); + if ( !ignoreAnatomicalSites ) { + Collection goldEntities = JCasUtil.select(goldView, EntityMention.class); + goldEntitiesAndEvents.addAll(goldEntities); + } Collection goldEvents = JCasUtil.select(goldView, EventMention.class); goldEntitiesAndEvents.addAll(goldEvents); // System.out.format("gold entities: %d%ngold events: %d%n%n", goldEntities.size(), goldEvents.size()); @@ -912,8 +916,10 @@ public static void printScore(Map systemEntitiesAndEvents = new ArrayList(); - Collection systemEntities = JCasUtil.select(jCas, EntityMention.class); - systemEntitiesAndEvents.addAll(systemEntities); + if ( !ignoreAnatomicalSites ) { + Collection systemEntities = JCasUtil.select(jCas, EntityMention.class); + systemEntitiesAndEvents.addAll(systemEntities); + } Collection systemEvents = JCasUtil.select(jCas, EventMention.class); systemEntitiesAndEvents.addAll(systemEvents); // System.out.format("system entities: %d%nsystem events: %d%n%n", systemEntities.size(), systemEvents.size()); Modified: ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TestAttributeModels.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TestAttributeModels.java?rev=1513928&r1=1513927&r2=1513928&view=diff ============================================================================== --- ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TestAttributeModels.java (original) +++ ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TestAttributeModels.java Wed Aug 14 15:28:51 2013 @@ -21,14 +21,16 @@ public class TestAttributeModels { for (String attribute : AssertionConst.annotationTypes) { ArrayList params = new ArrayList(); - +// AssertionEvaluation.useEvaluationLogFile = true; + params.add("--test-dir"); params.add(AssertionConst.testDirectories.get(attribute)); - params.add("--models-dir"); params.add(AssertionConst.modelDirectory); +// params.add("--models-dir"); params.add(AssertionConst.modelDirectory); +// params.add("--ytex-negation"); params.add("--evaluation-output-dir"); params.add(AssertionConst.evalOutputDir); params.add("--test-only"); // Build up an "ignore" string - for (String ignoreAttribute : AssertionConst.annotationTypes) { + for (String ignoreAttribute : AssertionConst.allAnnotationTypes) { if (!ignoreAttribute.equals(attribute)) { if (ignoreAttribute.equals("historyOf")) { Modified: ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TrainAttributeModels.java URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TrainAttributeModels.java?rev=1513928&r1=1513927&r2=1513928&view=diff ============================================================================== --- ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TrainAttributeModels.java (original) +++ ctakes/trunk/ctakes-assertion/src/main/java/org/apache/ctakes/assertion/train/TrainAttributeModels.java Wed Aug 14 15:28:51 2013 @@ -25,7 +25,7 @@ public class TrainAttributeModels { params.add("--models-dir"); params.add(AssertionConst.modelDirectory); // params.add("--evaluation-output-dir"); params.add(AssertionConst.evalOutputDir); params.add("--train-only"); - params.add("--feature-selection"); params.add("1.0"); +// params.add("--feature-selection"); params.add("1.0"); // Build up an "ignore" string for (String ignoreAttribute : AssertionConst.allAnnotationTypes) {