ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c...@apache.org
Subject svn commit: r1634983 [2/3] - in /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal: ae/ ae/feature/ eval/ pipelines/
Date Tue, 28 Oct 2014 21:01:07 GMT
Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java?rev=1634983&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java (added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventEventThymeRelations.java Tue Oct 28 21:01:06 2014
@@ -0,0 +1,1057 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ctakes.temporal.eval;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.net.URI;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
+import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
+//import org.apache.ctakes.temporal.ae.EventTimeSyntacticAnnotator;
+//import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
+//import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
+import org.apache.ctakes.temporal.ae.baselines.RecallBaselineEventTimeRelationAnnotator;
+import org.apache.ctakes.temporal.eval.EvaluationOfEventTimeRelations.ParameterSettings;
+import org.apache.ctakes.temporal.eval.EvaluationOfTemporalRelations_ImplBase.RemoveNonContainsRelations.RemoveGoldAttributes;
+//import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.WriteI2B2XML;
+//import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.XMLFormat;
+import org.apache.ctakes.temporal.utils.AnnotationIdCollection;
+import org.apache.ctakes.temporal.utils.TLinkTypeArray2;
+import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
+import org.apache.ctakes.typesystem.type.relation.RelationArgument;
+import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
+import org.apache.ctakes.typesystem.type.textsem.EventMention;
+import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
+import org.apache.ctakes.typesystem.type.textsem.TimeMention;
+import org.apache.ctakes.typesystem.type.textspan.Sentence;
+import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.CASException;
+import org.apache.uima.collection.CollectionReader;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.tcas.Annotation;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.FileUtils;
+import org.cleartk.ml.jar.JarClassifierBuilder;
+import org.cleartk.ml.liblinear.LibLinearStringOutcomeDataWriter;
+//import org.cleartk.ml.libsvm.LIBSVMStringOutcomeDataWriter;
+//import org.cleartk.ml.tksvmlight.TKSVMlightStringOutcomeDataWriter;
+import org.cleartk.ml.tksvmlight.model.CompositeKernel.ComboOperator;
+import org.cleartk.eval.AnnotationStatistics;
+import org.cleartk.util.ViewUriUtil;
+import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
+import org.apache.uima.fit.descriptor.ConfigurationParameter;
+import org.apache.uima.fit.factory.AggregateBuilder;
+import org.apache.uima.fit.factory.AnalysisEngineFactory;
+import org.apache.uima.fit.pipeline.JCasIterator;
+import org.apache.uima.fit.pipeline.SimplePipeline;
+import org.apache.uima.fit.util.JCasUtil;
+
+import com.google.common.base.Function;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.lexicalscope.jewel.cli.CliFactory;
+import com.lexicalscope.jewel.cli.Option;
+
+public class EvaluationOfEventEventThymeRelations extends
+EvaluationOfTemporalRelations_ImplBase{
+	static interface TempRelOptions extends Evaluation_ImplBase.Options{
+		@Option
+		public boolean getPrintFormattedRelations();
+
+		@Option
+		public boolean getBaseline();
+
+		@Option
+		public boolean getClosure();
+
+		@Option
+		public boolean getUseTmp();
+
+		@Option
+		public boolean getUseGoldAttributes();
+
+		@Option
+		public boolean getSkipTrain();
+	}
+
+	//  protected static boolean DEFAULT_BOTH_DIRECTIONS = false;
+	//  protected static float DEFAULT_DOWNSAMPLE = 1.0f;
+	//  private static double DEFAULT_SVM_C = 1.0;
+	//  private static double DEFAULT_SVM_G = 1.0;
+	//  private static double DEFAULT_TK = 0.5;
+	//  private static double DEFAULT_LAMBDA = 0.5;
+
+	//  defaultParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "tk",
+	//  		  DEFAULT_SVM_C, DEFAULT_SVM_G, "polynomial", ComboOperator.SUM, DEFAULT_TK, DEFAULT_LAMBDA);
+	protected static ParameterSettings flatParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "linear",
+			10.0, 1.0, "linear", ComboOperator.VECTOR_ONLY, DEFAULT_TK, DEFAULT_LAMBDA);
+	protected static ParameterSettings allBagsParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "tk", 
+			100.0, 0.1, "radial basis function", ComboOperator.SUM, 0.5, 0.5);
+	protected static ParameterSettings allParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "tk",
+			10.0, 1.0, "polynomial", ComboOperator.SUM, 0.1, 0.5);  // (0.3, 0.4 for tklibsvm)
+	protected static ParameterSettings ftParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "tk", 
+			1.0, 0.1, "radial basis function", ComboOperator.SUM, 0.5, 0.5);
+	private static Boolean recallModeEvaluation = true;
+
+	public static void main(String[] args) throws Exception {
+		TempRelOptions options = CliFactory.parseArguments(TempRelOptions.class, args);
+		List<Integer> trainItems = null;
+		List<Integer> devItems = null;
+		List<Integer> testItems = null;
+
+		List<Integer> patientSets = options.getPatients().getList();
+		if(options.getXMLFormat() == XMLFormat.I2B2){
+			trainItems = I2B2Data.getTrainPatientSets(options.getXMLDirectory());
+			devItems = I2B2Data.getDevPatientSets(options.getXMLDirectory());
+			testItems = I2B2Data.getTestPatientSets(options.getXMLDirectory());
+		}else{
+			trainItems = THYMEData.getPatientSets(patientSets, options.getTrainRemainders().getList());
+			devItems = THYMEData.getPatientSets(patientSets, options.getDevRemainders().getList());
+			testItems = THYMEData.getPatientSets(patientSets, options.getTestRemainders().getList());
+		}
+		ParameterSettings params = allParams;
+
+		//    possibleParams.add(defaultParams);
+
+		//    for(ParameterSettings params : possibleParams){
+		try{
+			File workingDir = new File("target/eval/thyme/");
+			if(!workingDir.exists()) workingDir.mkdirs();
+			if(options.getUseTmp()){
+				File tempModelDir = File.createTempFile("temporal", null, workingDir);
+				tempModelDir.delete();
+				tempModelDir.mkdir();
+				workingDir = tempModelDir;
+			}
+			EvaluationOfEventEventThymeRelations evaluation = new EvaluationOfEventEventThymeRelations(
+					workingDir,
+					options.getRawTextDirectory(),
+					options.getXMLDirectory(),
+					options.getXMLFormat(),
+					options.getXMIDirectory(),
+					options.getTreebankDirectory(),
+					options.getClosure(),
+					options.getPrintErrors(),
+					options.getPrintFormattedRelations(),
+					options.getBaseline(),
+					options.getUseGoldAttributes(),
+					options.getKernelParams(),
+					params);
+			evaluation.prepareXMIsFor(patientSets);
+			if(options.getI2B2Output()!=null) evaluation.setI2B2Output(options.getI2B2Output() + "/temporal-relations/event-event");
+			List<Integer> training = trainItems;
+			List<Integer> testing = null;
+			if(options.getTest()){
+				training.addAll(devItems);
+				testing = testItems;
+			}else{
+				testing = devItems;
+			}
+			//do closure on system, but not on gold, to calculate recall
+			evaluation.skipTrain = options.getSkipTrain();
+			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("No closure on gold::Closure on System::Recall Mode");
+			System.err.println(params.stats);
+
+			//do closure on gold, but not on system, to calculate precision
+			evaluation.skipTrain = true;
+			recallModeEvaluation = false;
+			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("No closure on System::Closure on Gold::Precision Mode");
+			System.err.println(params.stats);
+
+			//do closure on train, but not on test, to calculate plain results
+			evaluation.skipTrain = true;
+			evaluation.useClosure = false;
+			evaluation.printErrors = false;
+			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("Closure on train::No closure on Test::Plain Mode");
+			System.err.println(params.stats);
+
+			if(options.getUseTmp()){
+				// won't work because it's not empty. should we be concerned with this or is it responsibility of 
+				// person invoking the tmp flag?
+				FileUtils.deleteRecursive(workingDir);
+			}
+		}catch(ResourceInitializationException e){
+			System.err.println("Error with parameter settings: " + params);
+			e.printStackTrace();
+		}
+	}
+
+	//  private ParameterSettings params;
+	private boolean baseline;
+	protected boolean useClosure;
+	protected boolean useGoldAttributes;
+	protected boolean skipTrain=false;
+	//  protected boolean printRelations = false;
+
+	public EvaluationOfEventEventThymeRelations(
+			File baseDirectory,
+			File rawTextDirectory,
+			File xmlDirectory,
+			XMLFormat xmlFormat,
+			File xmiDirectory,
+			File treebankDirectory,
+			boolean useClosure,
+			boolean printErrors,
+			boolean printRelations,
+			boolean baseline,
+			boolean useGoldAttributes,
+			String kernelParams,
+			ParameterSettings params){
+		super(
+				baseDirectory,
+				rawTextDirectory,
+				xmlDirectory,
+				xmlFormat,
+				xmiDirectory,
+				treebankDirectory,
+				printErrors,
+				printRelations,
+				params);
+		this.params = params;
+		this.useClosure = useClosure;
+		this.printErrors = printErrors;
+		this.printRelations = printRelations;
+		this.useGoldAttributes = useGoldAttributes;
+		this.baseline = baseline;
+		this.kernelParams = kernelParams == null ? null : kernelParams.split(" ");
+	}
+
+	//  public EvaluationOfTemporalRelations(File baseDirectory, File rawTextDirectory,
+	//      File knowtatorXMLDirectory, File xmiDirectory) {
+	//
+	//    super(baseDirectory, rawTextDirectory, knowtatorXMLDirectory, xmiDirectory, null);
+	//    this.params = defaultParams;
+	//    this.printErrors = false;
+	//  }
+
+	@Override
+	protected void train(CollectionReader collectionReader, File directory) throws Exception {
+		//	  if(this.baseline) return;
+		if(this.skipTrain) return;
+		AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();
+		aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class, BinaryTextRelation.class));
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveCrossSentenceRelations.class));
+		if(!this.useGoldAttributes){
+			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveGoldAttributes.class));
+		}
+		if (this.useClosure) {
+			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddClosure.class));//aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class));
+			//			aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddContain2Overlap.class));
+			//			aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveBeforeAndOnRelations.class));
+		}
+		//		aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveNonContainsRelations.class));
+		//		aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
+
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(PreserveEventEventRelations.class));
+		//		aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveNonUMLSEvents.class));
+		
+		//add unlabeled nearby system events as potential links: 
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddEEPotentialRelations.class));
+				
+		aggregateBuilder.add(EventEventRelationAnnotator.createDataWriterDescription(
+				LibLinearStringOutcomeDataWriter.class,
+				//				LIBSVMStringOutcomeDataWriter.class,
+				//				TKSVMlightStringOutcomeDataWriter.class,
+				//        TKLIBSVMStringOutcomeDataWriter.class,
+				//        SVMlightStringOutcomeDataWriter.class,        
+				new File(directory,"event-event"),
+				params.probabilityOfKeepingANegativeExample));
+		//		aggregateBuilder.add(EventEventRelationAnnotator.createDataWriterDescription(
+		//				LIBSVMStringOutcomeDataWriter.class,
+		//				new File(directory,"event-event"), 
+		//				params.probabilityOfKeepingANegativeExample));
+		SimplePipeline.runPipeline(collectionReader, aggregateBuilder.createAggregate());
+		String[] optArray;
+
+		if(this.kernelParams == null){
+			ArrayList<String> svmOptions = new ArrayList<>();
+			svmOptions.add("-c"); svmOptions.add(""+params.svmCost);        // svm cost
+			svmOptions.add("-t"); svmOptions.add(""+params.svmKernelIndex); // kernel index 
+			svmOptions.add("-d"); svmOptions.add("3");                      // degree parameter for polynomial
+			svmOptions.add("-g"); svmOptions.add(""+params.svmGamma);
+			if(params.svmKernelIndex==ParameterSettings.SVM_KERNELS.indexOf("tk")){
+				svmOptions.add("-S"); svmOptions.add(""+params.secondKernelIndex);   // second kernel index (similar to -t) for composite kernel
+				String comboFlag = (params.comboOperator == ComboOperator.SUM ? "+" : params.comboOperator == ComboOperator.PRODUCT ? "*" : params.comboOperator == ComboOperator.TREE_ONLY ? "T" : "V");
+				svmOptions.add("-C"); svmOptions.add(comboFlag);
+				svmOptions.add("-L"); svmOptions.add(""+params.lambda);
+				svmOptions.add("-T"); svmOptions.add(""+params.tkWeight);
+				svmOptions.add("-N"); svmOptions.add("3");   // normalize trees and features
+			}
+			optArray = svmOptions.toArray(new String[]{});
+		}else{
+			optArray = this.kernelParams;
+			for(int i = 0; i < optArray.length; i+=2){
+				optArray[i] = "-" + optArray[i];
+			}
+		}
+
+		//    HideOutput hider = new HideOutput();
+		JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"),"-c", optArray[1]);//"-w1","0.09","-w2","4","-w3","9","-w5","2","-w6","16","-w7","10","-w8","6", "-w9","45","-w10","30","-c", optArray[1]);//"-c", "0.05");//optArray);
+		//		JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"), "-h","0","-c", "1000");
+		//    hider.restoreOutput();
+		//    hider.close();
+	}
+
+	@Override
+	protected AnnotationStatistics<String> test(CollectionReader collectionReader, File directory)
+			throws Exception {
+		AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();
+		aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class));
+
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(
+				RemoveCrossSentenceRelations.class,
+				RemoveCrossSentenceRelations.PARAM_SENTENCE_VIEW,
+				CAS.NAME_DEFAULT_SOFA,
+				RemoveCrossSentenceRelations.PARAM_RELATION_VIEW,
+				GOLD_VIEW_NAME));
+
+		aggregateBuilder.add(
+				AnalysisEngineFactory.createEngineDescription(PreserveEventEventRelations.class),
+				CAS.NAME_DEFAULT_SOFA,
+				GOLD_VIEW_NAME);
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveNonUMLSEvents.class));
+
+		if (!recallModeEvaluation && this.useClosure) { //closure for gold
+			aggregateBuilder.add(
+					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
+					CAS.NAME_DEFAULT_SOFA,
+					GOLD_VIEW_NAME);
+		}
+
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveRelations.class));
+		aggregateBuilder.add(this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory) :
+			EventEventRelationAnnotator.createAnnotatorDescription(new File(directory,"event-event")));
+
+		if(this.i2b2Output != null){
+			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(WriteI2B2XML.class, WriteI2B2XML.PARAM_OUTPUT_DIR, this.i2b2Output), "TimexView", CAS.NAME_DEFAULT_SOFA);
+		}
+
+		File outf = null;
+		if (recallModeEvaluation && this.useClosure) {//add closure for system output
+			aggregateBuilder.add(
+					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
+					GOLD_VIEW_NAME,
+					CAS.NAME_DEFAULT_SOFA
+					);
+			outf =  new File("target/eval/thyme/SystemError_eventEvent_recall_UMLSsingle_dev.txt");
+		}else if (!recallModeEvaluation && this.useClosure){
+			outf =  new File("target/eval/thyme/SystemError_eventEvent_precision_UMLSsingle_dev.txt");
+		}else{
+			outf =  new File("target/eval/thyme/SystemError_eventEvent_plain_UMLSsingle_dev.txt");
+		}
+
+		PrintWriter outDrop =null;
+		outDrop = new PrintWriter(new BufferedWriter(new FileWriter(outf, false)));
+
+		Function<BinaryTextRelation, ?> getSpan = new Function<BinaryTextRelation, HashableArguments>() {
+			public HashableArguments apply(BinaryTextRelation relation) {
+				return new HashableArguments(relation);
+			}
+		};
+		Function<BinaryTextRelation, String> getOutcome = AnnotationStatistics.annotationToFeatureValue("category");
+
+		AnnotationStatistics<String> stats = new AnnotationStatistics<>();
+		JCasIterator jcasIter =new JCasIterator(collectionReader, aggregateBuilder.createAggregate());
+		JCas jCas = null;
+		while(jcasIter.hasNext()) {
+			jCas = jcasIter.next();
+			JCas goldView = jCas.getView(GOLD_VIEW_NAME);
+			JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA);
+			Collection<BinaryTextRelation> goldRelations = JCasUtil.select(
+					goldView,
+					BinaryTextRelation.class);
+			Collection<BinaryTextRelation> systemRelations = JCasUtil.select(
+					systemView,
+					BinaryTextRelation.class);
+
+			//newly add
+			//			systemRelations = removeNonGoldRelations(systemRelations, goldRelations, getSpan);//for removing non-gold pairs
+			//			systemRelations = correctArgOrder(systemRelations, goldRelations);//change the argument order of "OVERLAP" relation, if the order is flipped
+			//find duplicates in gold relations:
+			//			Collection<BinaryTextRelation> duplicateGoldRelations = getDuplicateRelations(goldRelations, getSpan);
+			//			if(!duplicateGoldRelations.isEmpty()){
+			//				System.err.println("******Duplicate gold relations in : " + ViewURIUtil.getURI(jCas).toString());
+			//				for (BinaryTextRelation rel : duplicateGoldRelations){
+			//					System.err.println("Duplicate : "+ formatRelation(rel));
+			//				}
+			//			}
+			//end newly add
+
+			stats.add(goldRelations, systemRelations, getSpan, getOutcome);
+			if(this.printRelations){
+				URI uri = ViewUriUtil.getURI(jCas);
+				String[] path = uri.getPath().split("/");
+				printRelationAnnotations(path[path.length - 1], systemRelations);
+			}
+			if(this.printErrors){
+				Map<HashableArguments, BinaryTextRelation> goldMap = Maps.newHashMap();
+				for (BinaryTextRelation relation : goldRelations) {
+					goldMap.put(new HashableArguments(relation), relation);
+				}
+				Map<HashableArguments, BinaryTextRelation> systemMap = Maps.newHashMap();
+				for (BinaryTextRelation relation : systemRelations) {
+					systemMap.put(new HashableArguments(relation), relation);
+				}
+				Set<HashableArguments> all = Sets.union(goldMap.keySet(), systemMap.keySet());
+				List<HashableArguments> sorted = Lists.newArrayList(all);
+				Collections.sort(sorted);
+				outDrop.println("Doc id: " + ViewUriUtil.getURI(jCas).toString());
+				for (HashableArguments key : sorted) {
+					BinaryTextRelation goldRelation = goldMap.get(key);
+					BinaryTextRelation systemRelation = systemMap.get(key);
+					if (goldRelation == null) {
+						outDrop.println("System added: " + formatRelation(systemRelation));
+					} else if (systemRelation == null) {
+						outDrop.println("System dropped: " + formatRelation(goldRelation));
+					} else if (!systemRelation.getCategory().equals(goldRelation.getCategory())) {
+						String label = systemRelation.getCategory();
+						outDrop.printf("System labeled %s for %s\n", label, formatRelation(goldRelation));
+					} else{
+						outDrop.println("Nailed it! " + formatRelation(systemRelation));
+					}
+				}
+			}
+		}
+		outDrop.close();
+		return stats;
+	}
+
+	public static class RemoveNonUMLSEvents extends JCasAnnotator_ImplBase {
+		public static final String PARAM_GOLD_VIEW = "GoldView";
+
+		@ConfigurationParameter(name = PARAM_GOLD_VIEW,mandatory=false)
+		private String goldViewName = CAS.NAME_DEFAULT_SOFA;
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+			JCas sysView;
+			JCas goldView;
+			try {
+				sysView = jCas.getView(CAS.NAME_DEFAULT_SOFA);
+				goldView = jCas.getView(PARAM_GOLD_VIEW);
+			} catch (CASException e) {
+				throw new AnalysisEngineProcessException(e);
+			}
+			for(TemporalTextRelation relation : Lists.newArrayList(JCasUtil.select(goldView, TemporalTextRelation.class))){
+				Annotation arg1 = relation.getArg1().getArgument();
+				Annotation arg2 = relation.getArg2().getArgument();
+				boolean arg1Valid = false;
+				boolean arg2Valid = false;
+				for (EventMention event : JCasUtil.selectCovered(sysView, EventMention.class, arg1)){
+					if(!event.getClass().equals(EventMention.class)){
+						arg1Valid = true;
+						break;
+					}
+				}
+				for (EventMention event : JCasUtil.selectCovered(sysView, EventMention.class, arg2)){
+					if(!event.getClass().equals(EventMention.class)){
+						arg2Valid = true;
+						break;
+					}
+				}
+				if(arg1Valid && arg2Valid){
+					// these are the kind we keep.
+					continue;
+				}
+				arg1.removeFromIndexes();
+				arg2.removeFromIndexes();
+				relation.removeFromIndexes();
+			}
+		}   
+	}
+	
+	public static class AddEEPotentialRelations extends JCasAnnotator_ImplBase {
+		public static final String PARAM_RELATION_VIEW = "RelationView";
+		@ConfigurationParameter(name = PARAM_RELATION_VIEW,mandatory=false)
+		private String relationViewName = CAS.NAME_DEFAULT_SOFA;
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+			JCas relationView;
+			try {
+				relationView = jCas.getView(this.relationViewName);
+			} catch (CASException e) {
+				throw new AnalysisEngineProcessException(e);
+			}
+
+			Map<EventMention, Collection<EventMention>> coveringMap =
+					  JCasUtil.indexCovering(relationView, EventMention.class, EventMention.class);
+			for(TemporalTextRelation relation : Lists.newArrayList(JCasUtil.select(relationView, TemporalTextRelation.class))){
+				Annotation arg1 = relation.getArg1().getArgument();
+				Annotation arg2 = relation.getArg2().getArgument();
+				if(arg1 instanceof EventMention && arg2 instanceof EventMention){
+					EventMention event1 = (EventMention) arg1;
+					EventMention event2 = (EventMention) arg2;
+					for(EventMention covEventA : coveringMap.get(event1)){
+						for(EventMention covEventB : coveringMap.get(event2)){
+							createRelation(relationView, covEventA, covEventB, relation.getCategory());
+						}
+					}
+				}
+			}
+
+		}
+
+		private static void createRelation(JCas jCas, Annotation arg1,
+				Annotation arg2, String category) {
+			RelationArgument relArg1 = new RelationArgument(jCas);
+			relArg1.setArgument(arg1);
+			relArg1.setRole("Arg1");
+			relArg1.addToIndexes();
+			RelationArgument relArg2 = new RelationArgument(jCas);
+			relArg2.setArgument(arg2);
+			relArg2.setRole("Arg2");
+			relArg2.addToIndexes();
+			TemporalTextRelation relation = new TemporalTextRelation(jCas);
+			relation.setArg1(relArg1);
+			relation.setArg2(relArg2);
+			relation.setCategory(category);
+			relation.addToIndexes();
+			
+		}
+	}
+
+	/*
+  private static String formatRelation(BinaryTextRelation relation) {
+	  IdentifiedAnnotation arg1 = (IdentifiedAnnotation)relation.getArg1().getArgument();
+	  IdentifiedAnnotation arg2 = (IdentifiedAnnotation)relation.getArg2().getArgument();
+	  String text = arg1.getCAS().getDocumentText();
+	  int begin = Math.min(arg1.getBegin(), arg2.getBegin());
+	  int end = Math.max(arg1.getBegin(), arg2.getBegin());
+	  begin = Math.max(0, begin - 50);
+	  end = Math.min(text.length(), end + 50);
+	  return String.format(
+			  "%s(%s(type=%d), %s(type=%d)) in ...%s...",
+			  relation.getCategory(),
+			  arg1.getCoveredText(),
+			  arg1.getTypeID(),
+			  arg2.getCoveredText(),
+			  arg2.getTypeID(),
+			  text.substring(begin, end).replaceAll("[\r\n]", " "));
+  }
+
+  private static void printRelationAnnotations(String fileName, Collection<BinaryTextRelation> relations) {
+
+	  for(BinaryTextRelation binaryTextRelation : relations) {
+
+		  Annotation arg1 = binaryTextRelation.getArg1().getArgument();
+		  Annotation arg2 = binaryTextRelation.getArg2().getArgument();
+
+		  String arg1Type = arg1.getClass().getSimpleName();
+		  String arg2Type = arg2.getClass().getSimpleName();
+
+		  int arg1Begin = arg1.getBegin();
+		  int arg1End = arg1.getEnd();
+		  int arg2Begin = arg2.getBegin();
+		  int arg2End = arg2.getEnd();
+
+		  String category = binaryTextRelation.getCategory();
+
+		  System.out.format("%s\t%s\t%s\t%d\t%d\t%s\t%d\t%d\n", 
+				  fileName, category, arg1Type, arg1Begin, arg1End, arg2Type, arg2Begin, arg2End);
+	  }
+  }
+	 */
+
+	
+
+	//	@SuppressWarnings("unchecked")
+	//	private static <SPAN> Collection<BinaryTextRelation> getDuplicateRelations(
+	//			Collection<BinaryTextRelation> goldRelations,
+	//			Function<BinaryTextRelation, ?> getSpan) {
+	//		Set<BinaryTextRelation> duplicateRelations = Sets.newHashSet();
+	//		//build a multimap that map gold span to gold relation
+	//		Multimap<SPAN, BinaryTextRelation> spanToRelation = HashMultimap.create();
+	//		for (BinaryTextRelation relation : goldRelations) {
+	//			spanToRelation.put((SPAN) getSpan.apply(relation), relation);			
+	//		}
+	//		for (SPAN span: spanToRelation.keySet()){
+	//			Collection<BinaryTextRelation> relations = spanToRelation.get(span);
+	//			if(relations.size()>1){//if same span maps to multiple relations
+	//				duplicateRelations.addAll(relations);
+	//			}
+	//		}
+	//		return duplicateRelations;
+	//	}
+
+	//	private static Collection<BinaryTextRelation> removeNonGoldRelations(
+	//			Collection<BinaryTextRelation> systemRelations, Collection<BinaryTextRelation> goldRelations) {
+	//		//remove non-gold pairs from system relations:
+	//		Set<BinaryTextRelation> goodSys = Sets.newHashSet();
+	//
+	//		for(BinaryTextRelation sysrel : systemRelations){
+	//			Annotation sysArg1 = sysrel.getArg1().getArgument();
+	//			Annotation sysArg2 = sysrel.getArg2().getArgument();
+	//			for(BinaryTextRelation goldrel : goldRelations){
+	//				Annotation goldArg1 = goldrel.getArg1().getArgument();
+	//				Annotation goldArg2 = goldrel.getArg2().getArgument();
+	//				if(matchSpan(sysArg1, goldArg1) && matchSpan(sysArg2, goldArg2)){
+	//					goodSys.add(sysrel);
+	//					continue;
+	//				}else if (matchSpan(sysArg2, goldArg1) && matchSpan(sysArg1, goldArg2)){//the order of system pair was flipped 
+	//					if(sysrel.getCategory().equals("OVERLAP")){ //if the relation is overlap, and the arg order was flipped, then change back the order
+	//						RelationArgument tempArg = (RelationArgument) sysrel.getArg1().clone();
+	//						sysrel.setArg1((RelationArgument) sysrel.getArg2().clone());
+	//						sysrel.setArg2(tempArg);
+	//					}//for other types of relation, still maintain the type.
+	//					goodSys.add(sysrel);
+	//					continue;
+	//				}
+	//			}
+	//		}
+	//
+	//		return goodSys;
+	//	}
+
+
+
+
+	/*  public static class RemoveNonTLINKRelations extends JCasAnnotator_ImplBase {
+    @Override
+    public void process(JCas jCas) throws AnalysisEngineProcessException {
+      for (BinaryTextRelation relation : Lists.newArrayList(JCasUtil.select(
+          jCas,
+          BinaryTextRelation.class))) {
+        if (!(relation instanceof TemporalTextRelation)) {
+          relation.getArg1().removeFromIndexes();
+          relation.getArg2().removeFromIndexes();
+          relation.removeFromIndexes();
+        }
+      }
+    }
+  }*/
+
+	public static class RemoveCrossSentenceRelations extends JCasAnnotator_ImplBase {
+
+		public static final String PARAM_SENTENCE_VIEW = "SentenceView";
+
+		@ConfigurationParameter(name = PARAM_SENTENCE_VIEW,mandatory=false)
+		private String sentenceViewName = CAS.NAME_DEFAULT_SOFA;
+
+		public static final String PARAM_RELATION_VIEW = "RelationView";
+
+		@ConfigurationParameter(name = PARAM_RELATION_VIEW,mandatory=false)
+		private String relationViewName = CAS.NAME_DEFAULT_SOFA;
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+			JCas sentenceView, relationView;
+			try {
+				sentenceView = jCas.getView(this.sentenceViewName);
+				relationView = jCas.getView(this.relationViewName);
+			} catch (CASException e) {
+				throw new AnalysisEngineProcessException(e);
+			}
+
+			// map events and times to the sentences that contain them
+			Map<IdentifiedAnnotation, Integer> sentenceIndex = Maps.newHashMap();
+			int index = -1;
+			for (Sentence sentence : JCasUtil.select(sentenceView, Sentence.class)) {
+				++index;
+				for (EventMention event : JCasUtil.selectCovered(relationView, EventMention.class, sentence)) {
+					sentenceIndex.put(event, index);
+				}
+				for (TimeMention time : JCasUtil.selectCovered(relationView, TimeMention.class, sentence)) {
+					sentenceIndex.put(time, index);
+				}
+			}
+
+			// remove any relations that are in different sentences.
+			for (BinaryTextRelation relation : Lists.newArrayList(JCasUtil.select(
+					relationView,
+					BinaryTextRelation.class))) {
+				Integer sent1 = sentenceIndex.get(relation.getArg1().getArgument());
+				Integer sent2 = sentenceIndex.get(relation.getArg2().getArgument());
+				if (sent1 == null || sent2 == null || !sent1.equals(sent2)) {
+					relation.getArg1().removeFromIndexes();
+					relation.getArg2().removeFromIndexes();
+					relation.removeFromIndexes();
+				}
+			}
+		}
+	}
+
+
+	public static class RemoveRelations extends JCasAnnotator_ImplBase {
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+			for (BinaryTextRelation relation : Lists.newArrayList(JCasUtil.select(
+					jCas,
+					BinaryTextRelation.class))) {
+				relation.getArg1().removeFromIndexes();
+				relation.getArg2().removeFromIndexes();
+				relation.removeFromIndexes();
+			}
+		}
+	}
+
+
+	public static class AddTransitiveContainsRelations extends JCasAnnotator_ImplBase {
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+
+			// collect many-to-many mappings of containment relations 
+			Multimap<Annotation, Annotation> isContainedIn = HashMultimap.create();
+			Multimap<Annotation, Annotation> contains = HashMultimap.create();
+			Set<BinaryTextRelation> containsRelations = Sets.newHashSet();
+			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
+				if (relation.getCategory().equals("CONTAINS")) {
+					containsRelations.add(relation);
+					Annotation arg1 = relation.getArg1().getArgument();
+					Annotation arg2 = relation.getArg2().getArgument();
+					contains.put(arg1, arg2);
+					isContainedIn.put(arg2, arg1);
+				}
+			}
+
+			// look for X -> Y -> Z containment chains and add X -> Z relations
+			Deque<Annotation> todo = new ArrayDeque<>(isContainedIn.keySet());
+			while (!todo.isEmpty()) {
+				Annotation next = todo.removeFirst();
+				for (Annotation parent : Lists.newArrayList(isContainedIn.get(next))) {
+					for (Annotation grandParent : Lists.newArrayList(isContainedIn.get(parent))) {
+						if (!isContainedIn.containsEntry(next, grandParent)) {
+							isContainedIn.put(next, grandParent);
+							contains.put(grandParent, next);
+
+							// once X -> Z has been added, we need to re-do all W where W -> X
+							for (Annotation child : contains.get(next)) {
+								todo.add(child);
+							}
+						}
+					}
+				}
+			}
+
+			// remove old relations
+			for (BinaryTextRelation relation : containsRelations) {
+				relation.getArg1().removeFromIndexes();
+				relation.getArg2().removeFromIndexes();
+				relation.removeFromIndexes();
+			}
+
+			// add new, transitive relations
+			for (Annotation contained : isContainedIn.keySet()) {
+				for (Annotation container : isContainedIn.get(contained)) {
+					RelationArgument arg1 = new RelationArgument(jCas);
+					arg1.setArgument(container);
+					RelationArgument arg2 = new RelationArgument(jCas);
+					arg2.setArgument(contained);
+					BinaryTextRelation relation = new BinaryTextRelation(jCas);
+					relation.setArg1(arg1);
+					relation.setArg2(arg2);
+					relation.setCategory("CONTAINS");
+					arg1.addToIndexes();
+					arg2.addToIndexes();
+					relation.addToIndexes();
+				}
+			}
+		}
+
+	}
+
+	public static class AddContain2Overlap extends JCasAnnotator_ImplBase {
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+
+			Set<BinaryTextRelation> containsRelations = Sets.newHashSet();
+			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
+				if (relation.getCategory().equals("CONTAINS")) {
+					containsRelations.add(relation);
+				}
+			}
+
+			for (BinaryTextRelation relation : containsRelations) {
+				RelationArgument arg1 = (RelationArgument) relation.getArg1().clone();
+				RelationArgument arg2 = (RelationArgument) relation.getArg2().clone();
+				BinaryTextRelation newrelation = new BinaryTextRelation(jCas);
+				newrelation.setArg1(arg1);
+				newrelation.setArg2(arg2);
+				newrelation.setCategory("OVERLAP");
+				arg1.addToIndexes();
+				arg2.addToIndexes();
+				newrelation.addToIndexes();
+			}
+		}
+	}
+
+	public static class AddFlippedOverlap extends JCasAnnotator_ImplBase {
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+
+			Set<BinaryTextRelation> overlapRelations = Sets.newHashSet();
+			Multimap<Annotation, Annotation> overlaps = HashMultimap.create();
+			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
+				if (relation.getCategory().equals("OVERLAP")) {
+					overlapRelations.add(relation);
+					Annotation arg1 = relation.getArg1().getArgument();
+					Annotation arg2 = relation.getArg2().getArgument();
+					overlaps.put(arg1, arg2);
+				}
+			}
+
+			for (BinaryTextRelation orelation : overlapRelations) {
+				Annotation argA = orelation.getArg1().getArgument();
+				Annotation argB = orelation.getArg2().getArgument();
+				//add overlap 
+				if (!overlaps.containsEntry(argB, argA)) {
+					//create a new flipped relation:
+					RelationArgument arg1 = new RelationArgument(jCas);
+					arg1.setArgument(argB);
+					RelationArgument arg2 = new RelationArgument(jCas);
+					arg2.setArgument(argA);
+					BinaryTextRelation relation = new BinaryTextRelation(jCas);
+					relation.setArg1(arg1);
+					relation.setArg2(arg2);
+					relation.setCategory("OVERLAP");
+					arg1.addToIndexes();
+					arg2.addToIndexes();
+					relation.addToIndexes();
+					overlaps.put(argB, argA);
+				}
+
+			}
+		}
+	}
+
+	public static class AddClosure extends JCasAnnotator_ImplBase {
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+
+			Multimap<List<Annotation>, BinaryTextRelation> annotationsToRelation = HashMultimap.create();
+			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)){
+				String relationType = relation.getCategory();
+				if(validTemporalType(relationType)){
+					Annotation arg1 = relation.getArg1().getArgument();
+					Annotation arg2 = relation.getArg2().getArgument();
+					annotationsToRelation.put(Arrays.asList(arg1, arg2), relation);
+				}
+			}
+			for (List<Annotation> span: Lists.newArrayList(annotationsToRelation.keySet())){
+				Collection<BinaryTextRelation> relations = annotationsToRelation.get(span);
+				if(relations.size()>1){//if same span maps to multiple relations
+					Set<String> types = Sets.newHashSet();
+					for(BinaryTextRelation relation: relations){
+						types.add(relation.getCategory());
+					}
+					if(types.size()>1){
+						for(BinaryTextRelation relation: Lists.newArrayList(relations)){
+							annotationsToRelation.remove(span, relation);
+							relation.getArg1().removeFromIndexes();
+							relation.getArg2().removeFromIndexes();
+							relation.removeFromIndexes();
+						}
+					}else if(types.size()==1){
+						for (int i =1; i< relations.size(); i++){
+							BinaryTextRelation relation = (BinaryTextRelation) relations.toArray()[i];
+							annotationsToRelation.remove(span, relation);
+							relation.getArg1().removeFromIndexes();
+							relation.getArg2().removeFromIndexes();
+							relation.removeFromIndexes();
+						}
+					}
+				}
+			}
+
+			ArrayList<BinaryTextRelation> temporalRelation = new ArrayList<>(annotationsToRelation.values());//new ArrayList<BinaryTextRelation>();
+			//			Map<List<Annotation>, BinaryTextRelation> temporalRelationLookup = new HashMap<List<Annotation>, BinaryTextRelation>();
+			//
+			//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)){
+			//				String relationType = relation.getCategory();
+			//				if(validTemporalType(relationType)){
+			//					Annotation arg1 = relation.getArg1().getArgument();
+			//			        Annotation arg2 = relation.getArg2().getArgument();
+			//			        BinaryTextRelation tempRelation = temporalRelationLookup.get(Arrays.asList(arg1, arg2));
+			//					if( tempRelation == null){
+			//						temporalRelation.add(relation);					
+			//				        temporalRelationLookup.put(Arrays.asList(arg1, arg2), relation);
+			//					}else{//if there is duplicate
+			//						relation.getArg1().removeFromIndexes();
+			//						relation.getArg2().removeFromIndexes();
+			//						relation.removeFromIndexes();
+			//					}
+			//					
+			//				}
+			//			}
+
+			if (!temporalRelation.isEmpty()){
+				TLinkTypeArray2 relationArray = new TLinkTypeArray2(temporalRelation, new AnnotationIdCollection(temporalRelation));
+
+				int addedCount = 0;
+				for (BinaryTextRelation relation : relationArray.getClosedTlinks(jCas)) {
+					RelationArgument arg1 = relation.getArg1();
+					RelationArgument arg2 = relation.getArg2();
+					String relationType = relation.getCategory();
+					if(relationType.equals("CONTAINED-BY")||relationType.equals("AFTER")){//ignore these two categories, because their reciprocal already exist.
+						continue;
+					}
+					//check if the inferred relation new:
+					Collection<BinaryTextRelation> relations = annotationsToRelation.get(Arrays.asList(arg1.getArgument(), arg2.getArgument()));
+					if(relations.isEmpty()){ //if haven't seen this inferred relation before, then add this relation
+						arg1.addToIndexes();
+						arg2.addToIndexes();
+						relation.addToIndexes();
+						addedCount++;
+					}		
+				}
+
+				System.out.println( "**************************************************************");
+				System.out.println( "Finally added closure relations: " + addedCount );
+				System.out.println( "**************************************************************");
+			}			
+
+		}
+
+		private static boolean validTemporalType(String relationType) {
+			if(relationType.equals("CONTAINS")||relationType.equals("OVERLAP")||relationType.equals("BEFORE")||relationType.equals("ENDS-ON")||relationType.equals("BEGINS-ON"))
+				return true;
+			return false;
+		}
+	}
+
+	//	public static class AddTransitiveBeforeAndOnRelations extends JCasAnnotator_ImplBase {
+	//
+	//		@Override
+	//		public void process(JCas jCas) throws AnalysisEngineProcessException {
+	//
+	//			// collect many-to-many mappings of containment relations 
+	//			Multimap<Annotation, Annotation> contains = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> before = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> endson = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> beginson = HashMultimap.create();
+	//			Set<BinaryTextRelation> beforeRel = Sets.newHashSet();
+	//
+	//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
+	//				if (relation.getCategory().equals("CONTAINS")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					contains.put(arg1, arg2);
+	//				}else if (relation.getCategory().equals("BEFORE")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					before.put(arg1, arg2);
+	//					beforeRel.add(relation);
+	//				}else if (relation.getCategory().equals("ENDS-ON")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					endson.put(arg1, arg2);
+	//					if (!endson.containsEntry(arg2, arg1)) {
+	//						endson.put(arg2, arg1);
+	//					}
+	//				}else if (relation.getCategory().equals("BEGINS-ON")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					beginson.put(arg1, arg2);
+	//					if (!beginson.containsEntry(arg2, arg1)) {
+	//						beginson.put(arg2, arg1);
+	//					}
+	//				}
+	//			}
+	//
+	//			// for A BEFORE B, check if A and B Contain anything
+	//			for (BinaryTextRelation brelation : beforeRel) {
+	//				Annotation argA = brelation.getArg1().getArgument();
+	//				Annotation argB = brelation.getArg2().getArgument();
+	//				//add contained before
+	//				for (Annotation childA : contains.get(argA)) {
+	//					for (Annotation childB : contains.get(argB)) {
+	//						if (!before.containsEntry(childA, childB)) {
+	//							//create a new before relation:
+	//							RelationArgument arg1 = new RelationArgument(jCas);
+	//							arg1.setArgument(childA);
+	//							RelationArgument arg2 = new RelationArgument(jCas);
+	//							arg2.setArgument(childB);
+	//							BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//							relation.setArg1(arg1);
+	//							relation.setArg2(arg2);
+	//							relation.setCategory("BEFORE");
+	//							arg1.addToIndexes();
+	//							arg2.addToIndexes();
+	//							relation.addToIndexes();
+	//							before.put(childA, childB);
+	//						}
+	//					}
+	//				}
+	//				//add ends-on A
+	//				for (Annotation endsOnA : endson.get(argA)) {
+	//					if (!before.containsEntry(endsOnA, argB)) {
+	//						//create a new before relation:
+	//						RelationArgument arg1 = new RelationArgument(jCas);
+	//						arg1.setArgument(endsOnA);
+	//						RelationArgument arg2 = new RelationArgument(jCas);
+	//						arg2.setArgument(argB);
+	//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//						relation.setArg1(arg1);
+	//						relation.setArg2(arg2);
+	//						relation.setCategory("BEFORE");
+	//						arg1.addToIndexes();
+	//						arg2.addToIndexes();
+	//						relation.addToIndexes();
+	//						before.put(endsOnA, argB);
+	//					}
+	//				}
+	//				//add begins-on B
+	//				for (Annotation beginsOnB : beginson.get(argB)) {
+	//					if (!before.containsEntry(argA, beginsOnB)) {
+	//						//create a new before relation:
+	//						RelationArgument arg1 = new RelationArgument(jCas);
+	//						arg1.setArgument(argA);
+	//						RelationArgument arg2 = new RelationArgument(jCas);
+	//						arg2.setArgument(beginsOnB);
+	//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//						relation.setArg1(arg1);
+	//						relation.setArg2(arg2);
+	//						relation.setCategory("BEFORE");
+	//						arg1.addToIndexes();
+	//						arg2.addToIndexes();
+	//						relation.addToIndexes();
+	//						before.put(argA, beginsOnB);
+	//					}
+	//				}
+	//			}
+	//		}
+	//
+	//	}
+}

Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java?rev=1634983&r1=1634982&r2=1634983&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java (original)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java Tue Oct 28 21:01:06 2014
@@ -18,7 +18,10 @@
  */
 package org.apache.ctakes.temporal.eval;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
 import java.net.URI;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -26,18 +29,27 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Deque;
+//import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.ctakes.relationextractor.eval.RelationExtractorEvaluation.HashableArguments;
 import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
+import org.apache.ctakes.temporal.ae.EventTimeSelfRelationAnnotator;
+//import org.apache.ctakes.temporal.ae.EventTimeSyntacticAnnotator;
+//import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
+//import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
 import org.apache.ctakes.temporal.ae.baselines.RecallBaselineEventTimeRelationAnnotator;
 import org.apache.ctakes.temporal.eval.EvaluationOfTemporalRelations_ImplBase.RemoveNonContainsRelations.RemoveGoldAttributes;
+//import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.WriteI2B2XML;
+//import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.XMLFormat;
 import org.apache.ctakes.temporal.utils.AnnotationIdCollection;
 import org.apache.ctakes.temporal.utils.TLinkTypeArray2;
 import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
 import org.apache.ctakes.typesystem.type.relation.RelationArgument;
+import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
+//import org.apache.ctakes.typesystem.type.relation.TemporalTextRelation;
 import org.apache.ctakes.typesystem.type.textsem.EventMention;
 import org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation;
 import org.apache.ctakes.typesystem.type.textsem.TimeMention;
@@ -46,23 +58,25 @@ import org.apache.uima.analysis_engine.A
 import org.apache.uima.cas.CAS;
 import org.apache.uima.cas.CASException;
 import org.apache.uima.collection.CollectionReader;
-import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
-import org.apache.uima.fit.descriptor.ConfigurationParameter;
-import org.apache.uima.fit.factory.AggregateBuilder;
-import org.apache.uima.fit.factory.AnalysisEngineFactory;
-import org.apache.uima.fit.pipeline.JCasIterator;
-import org.apache.uima.fit.pipeline.SimplePipeline;
-import org.apache.uima.fit.util.JCasUtil;
 import org.apache.uima.jcas.JCas;
 import org.apache.uima.jcas.tcas.Annotation;
 import org.apache.uima.resource.ResourceInitializationException;
 import org.apache.uima.util.FileUtils;
-import org.cleartk.eval.AnnotationStatistics;
 import org.cleartk.ml.jar.JarClassifierBuilder;
-import org.cleartk.ml.tksvmlight.TkSvmLightStringOutcomeDataWriter;
+import org.cleartk.ml.liblinear.LibLinearStringOutcomeDataWriter;
+//import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
+//import org.cleartk.ml.tksvmlight.TkSvmLightStringOutcomeDataWriter;
 import org.cleartk.ml.tksvmlight.model.CompositeKernel;
 import org.cleartk.ml.tksvmlight.model.CompositeKernel.ComboOperator;
+import org.cleartk.eval.AnnotationStatistics;
 import org.cleartk.util.ViewUriUtil;
+import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
+import org.apache.uima.fit.descriptor.ConfigurationParameter;
+import org.apache.uima.fit.factory.AggregateBuilder;
+import org.apache.uima.fit.factory.AnalysisEngineFactory;
+import org.apache.uima.fit.pipeline.JCasIterator;
+import org.apache.uima.fit.pipeline.SimplePipeline;
+import org.apache.uima.fit.util.JCasUtil;
 
 import com.google.common.base.Function;
 import com.google.common.collect.HashMultimap;
@@ -90,7 +104,7 @@ EvaluationOfTemporalRelations_ImplBase{
 
 		@Option
 		public boolean getUseGoldAttributes();
-		
+
 		@Option
 		public boolean getSkipTrain();
 	}
@@ -112,6 +126,7 @@ EvaluationOfTemporalRelations_ImplBase{
 			10.0, 1.0, "polynomial", ComboOperator.SUM, 0.1, 0.5);  // (0.3, 0.4 for tklibsvm)
 	protected static ParameterSettings ftParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS, DEFAULT_DOWNSAMPLE, "tk", 
 			1.0, 0.1, "radial basis function", ComboOperator.SUM, 0.5, 0.5);
+	private static Boolean recallModeEvaluation = true;
 
 	public static void main(String[] args) throws Exception {
 		TempRelOptions options = CliFactory.parseArguments(TempRelOptions.class, args);
@@ -135,7 +150,7 @@ EvaluationOfTemporalRelations_ImplBase{
 
 		//    for(ParameterSettings params : possibleParams){
 		try{
-			File workingDir = new File("target/eval/temporal-relations/event-time");
+			File workingDir = new File("target/eval/thyme/");
 			if(!workingDir.exists()) workingDir.mkdirs();
 			if(options.getUseTmp()){
 				File tempModelDir = File.createTempFile("temporal", null, workingDir);
@@ -159,7 +174,7 @@ EvaluationOfTemporalRelations_ImplBase{
 					options.getKernelParams(),
 					params);
 			evaluation.prepareXMIsFor(patientSets);
-	    if(options.getI2B2Output()!=null) evaluation.setI2B2Output(options.getI2B2Output() + "/temporal-relations/event-time");
+			if(options.getI2B2Output()!=null) evaluation.setI2B2Output(options.getI2B2Output() + "/temporal-relations/event-time");
 			List<Integer> training = trainItems;
 			List<Integer> testing = null;
 			if(options.getTest()){
@@ -168,10 +183,29 @@ EvaluationOfTemporalRelations_ImplBase{
 			}else{
 				testing = devItems;
 			}
+			//do closure on system, but not on gold, to calculate recall
 			evaluation.skipTrain = options.getSkipTrain();
 			params.stats = evaluation.trainAndTest(training, testing);//training);//
 			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("No closure on gold::Closure on System::Recall Mode");
+			System.err.println(params.stats);
+			
+			//do closure on gold, but not on system, to calculate precision
+			evaluation.skipTrain = true;
+			recallModeEvaluation = false;
+			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("No closure on System::Closure on Gold::Precision Mode");
 			System.err.println(params.stats);
+			
+			//do closure on train, but not on test, to calculate plain results
+			evaluation.skipTrain = true;
+			evaluation.useClosure = false;
+			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			System.err.println("Closure on train::No closure on Test::Plain Mode");
+			System.err.println(params.stats);
+			
 			if(options.getUseTmp()){
 				// won't work because it's not empty. should we be concerned with this or is it responsibility of 
 				// person invoking the tmp flag?
@@ -236,7 +270,7 @@ EvaluationOfTemporalRelations_ImplBase{
 	@Override
 	protected void train(CollectionReader collectionReader, File directory) throws Exception {
 		//	  if(this.baseline) return;
-	  if(this.skipTrain) return;
+		if(this.skipTrain) return;
 		AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();
 		aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class, BinaryTextRelation.class));
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveCrossSentenceRelations.class));
@@ -244,26 +278,35 @@ EvaluationOfTemporalRelations_ImplBase{
 			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveGoldAttributes.class));
 		}
 		if (this.useClosure) {
-			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddClosure.class));//aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddTransitiveContainsRelations.class));
-//			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddContain2Overlap.class));
-			//			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddTransitiveBeforeAndOnRelations.class));
+			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddClosure.class));//aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class));
+			//			aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddContain2Overlap.class));
+			//			aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveBeforeAndOnRelations.class));
 		}
-//		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveNonContainsRelations.class));
-		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
-		
+		//		aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(RemoveNonContainsRelations.class));
+//		aggregateBuilder.add(AnalysisEngineFactory.createPrimitiveDescription(AddFlippedOverlap.class));//add flipped overlap instances to training data
+
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveEventEventRelations.class));
-		aggregateBuilder.add(EventTimeRelationAnnotator.createDataWriterDescription(
-				//                LibSvmStringOutcomeDataWriter.class,
-				TkSvmLightStringOutcomeDataWriter.class,
-				//        TKLibSvmStringOutcomeDataWriter.class,
+		
+		//add unlabeled nearby system events as potential links: 
+		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddPotentialRelations.class));
+		
+		aggregateBuilder.add(EventTimeSelfRelationAnnotator.createDataWriterDescription(
+				LibLinearStringOutcomeDataWriter.class,
+				//				LIBSVMStringOutcomeDataWriter.class,
+				//				TKSVMlightStringOutcomeDataWriter.class,
+				//        TKLIBSVMStringOutcomeDataWriter.class,
 				//        SVMlightStringOutcomeDataWriter.class,        
-				directory,
+				new File(directory,"event-time"),
 				params.probabilityOfKeepingANegativeExample));
+		//		aggregateBuilder.add(EventEventRelationAnnotator.createDataWriterDescription(
+		//				LIBSVMStringOutcomeDataWriter.class,
+		//				new File(directory,"event-event"), 
+		//				params.probabilityOfKeepingANegativeExample));
 		SimplePipeline.runPipeline(collectionReader, aggregateBuilder.createAggregate());
 		String[] optArray;
 
 		if(this.kernelParams == null){
-			ArrayList<String> svmOptions = new ArrayList<String>();
+			ArrayList<String> svmOptions = new ArrayList<>();
 			svmOptions.add("-c"); svmOptions.add(""+params.svmCost);        // svm cost
 			svmOptions.add("-t"); svmOptions.add(""+params.svmKernelIndex); // kernel index 
 			svmOptions.add("-d"); svmOptions.add("3");                      // degree parameter for polynomial
@@ -285,7 +328,8 @@ EvaluationOfTemporalRelations_ImplBase{
 		}
 
 		//    HideOutput hider = new HideOutput();
-		JarClassifierBuilder.trainAndPackage(directory, optArray);
+		JarClassifierBuilder.trainAndPackage(new File(directory,"event-time"), "-w3","2","-w4","18","-w5","14","-w6","21","-w7","100","-w8","19","-c", optArray[1]);//"0.05");//"-h","0","-c", "1000");//optArray);
+		//		JarClassifierBuilder.trainAndPackage(new File(directory,"event-event"), "-h","0","-c", "1000");
 		//    hider.restoreOutput();
 		//    hider.close();
 	}
@@ -295,33 +339,21 @@ EvaluationOfTemporalRelations_ImplBase{
 			throws Exception {
 		AggregateBuilder aggregateBuilder = this.getPreprocessorAggregateBuilder();
 		aggregateBuilder.add(CopyFromGold.getDescription(EventMention.class, TimeMention.class));
-		
+
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(
 				RemoveCrossSentenceRelations.class,
 				RemoveCrossSentenceRelations.PARAM_SENTENCE_VIEW,
 				CAS.NAME_DEFAULT_SOFA,
 				RemoveCrossSentenceRelations.PARAM_RELATION_VIEW,
 				GOLD_VIEW_NAME));
-		if (this.useClosure) {
+
+		if (!recallModeEvaluation && this.useClosure) { //closure for gold
 			aggregateBuilder.add(
-					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createEngineDescription(AddTransitiveContainsRelations.class),
+					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
 					CAS.NAME_DEFAULT_SOFA,
 					GOLD_VIEW_NAME);
-					
-//			aggregateBuilder.add(
-//					AnalysisEngineFactory.createEngineDescription(AddContain2Overlap.class),
-//					CAS.NAME_DEFAULT_SOFA,
-//					GOLD_VIEW_NAME);
-			//			aggregateBuilder.add(
-			//					AnalysisEngineFactory.createEngineDescription(AddTransitiveBeforeAndOnRelations.class),
-			//					CAS.NAME_DEFAULT_SOFA,
-			//					GOLD_VIEW_NAME);
 		}
-		
-//		aggregateBuilder.add(
-//				AnalysisEngineFactory.createEngineDescription(RemoveNonContainsRelations.class,
-//						RemoveNonContainsRelations.PARAM_RELATION_VIEW,
-//						GOLD_VIEW_NAME));
+
 		aggregateBuilder.add(
 				AnalysisEngineFactory.createEngineDescription(RemoveEventEventRelations.class),
 				CAS.NAME_DEFAULT_SOFA,
@@ -329,17 +361,35 @@ EvaluationOfTemporalRelations_ImplBase{
 
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveRelations.class));
 		aggregateBuilder.add(this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory) :
-			EventTimeRelationAnnotator.createAnnotatorDescription(directory));
-    if(this.i2b2Output != null){
-      aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(WriteI2B2XML.class, WriteI2B2XML.PARAM_OUTPUT_DIR, this.i2b2Output), "TimexView", CAS.NAME_DEFAULT_SOFA);
-    }
-		if (this.useClosure) {//add closure for system output
+			EventTimeSelfRelationAnnotator.createEngineDescription(new File(directory,"event-time")));
+		
+		if(this.i2b2Output != null){
+			aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(WriteI2B2XML.class, WriteI2B2XML.PARAM_OUTPUT_DIR, this.i2b2Output), "TimexView", CAS.NAME_DEFAULT_SOFA);
+		}
+		
+		File outf = null;
+		if (recallModeEvaluation && this.useClosure) {//add closure for system output
+			aggregateBuilder.add(
+					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createPrimitiveDescription(AddTransitiveContainsRelations.class),
+					GOLD_VIEW_NAME,
+					CAS.NAME_DEFAULT_SOFA
+					);
 			aggregateBuilder.add(
-					AnalysisEngineFactory.createEngineDescription(AddClosure.class),//AnalysisEngineFactory.createEngineDescription(AddTransitiveContainsRelations.class),
+					AnalysisEngineFactory.createEngineDescription(RemoveEventEventRelations.class),
 					GOLD_VIEW_NAME,
 					CAS.NAME_DEFAULT_SOFA
 					);
+			outf =  new File("target/eval/thyme/SystemError_eventTime_recall_dev.txt");
+		}else if (!recallModeEvaluation && this.useClosure){
+			outf =  new File("target/eval/thyme/SystemError_eventTime_precision_dev.txt");
+		}else{
+			outf =  new File("target/eval/thyme/SystemError_eventTime_plain_dev.txt");
 		}
+
+		PrintWriter outDrop =null;
+		
+		outDrop = new PrintWriter(new BufferedWriter(new FileWriter(outf, false)));
+		
 		Function<BinaryTextRelation, ?> getSpan = new Function<BinaryTextRelation, HashableArguments>() {
 			public HashableArguments apply(BinaryTextRelation relation) {
 				return new HashableArguments(relation);
@@ -347,11 +397,11 @@ EvaluationOfTemporalRelations_ImplBase{
 		};
 		Function<BinaryTextRelation, String> getOutcome = AnnotationStatistics.annotationToFeatureValue("category");
 
-		AnnotationStatistics<String> stats = new AnnotationStatistics<String>();
-		JCasIterator casIter = new JCasIterator(collectionReader, aggregateBuilder.createAggregate());
+		AnnotationStatistics<String> stats = new AnnotationStatistics<>();
+		JCasIterator jcasIter =new JCasIterator(collectionReader, aggregateBuilder.createAggregate());
 		JCas jCas = null;
-		while(casIter.hasNext()) {
-			jCas = casIter.next();
+		while(jcasIter.hasNext()) {
+			jCas = jcasIter.next();
 			JCas goldView = jCas.getView(GOLD_VIEW_NAME);
 			JCas systemView = jCas.getView(CAS.NAME_DEFAULT_SOFA);
 			Collection<BinaryTextRelation> goldRelations = JCasUtil.select(
@@ -362,12 +412,12 @@ EvaluationOfTemporalRelations_ImplBase{
 					BinaryTextRelation.class);
 
 			//newly add
-//			systemRelations = removeNonGoldRelations(systemRelations, goldRelations);//for removing non-gold pairs
-			systemRelations = correctArgOrder(systemRelations, goldRelations);//change the argument order of "OVERLAP" relation, if the order is flipped
+			//			systemRelations = removeNonGoldRelations(systemRelations, goldRelations, getSpan);//for removing non-gold pairs
+//			systemRelations = correctArgOrder(systemRelations, goldRelations);//change the argument order of "OVERLAP" relation, if the order is flipped
 			//find duplicates in gold relations:
 			//			Collection<BinaryTextRelation> duplicateGoldRelations = getDuplicateRelations(goldRelations, getSpan);
 			//			if(!duplicateGoldRelations.isEmpty()){
-			//				System.err.println("******Duplicate gold relations in : " + ViewUriUtil.getURI(jCas).toString());
+			//				System.err.println("******Duplicate gold relations in : " + ViewURIUtil.getURI(jCas).toString());
 			//				for (BinaryTextRelation rel : duplicateGoldRelations){
 			//					System.err.println("Duplicate : "+ formatRelation(rel));
 			//				}
@@ -392,22 +442,24 @@ EvaluationOfTemporalRelations_ImplBase{
 				Set<HashableArguments> all = Sets.union(goldMap.keySet(), systemMap.keySet());
 				List<HashableArguments> sorted = Lists.newArrayList(all);
 				Collections.sort(sorted);
+				outDrop.println("Doc id: " + ViewUriUtil.getURI(jCas).toString());
 				for (HashableArguments key : sorted) {
 					BinaryTextRelation goldRelation = goldMap.get(key);
 					BinaryTextRelation systemRelation = systemMap.get(key);
 					if (goldRelation == null) {
-						System.out.println("System added: " + formatRelation(systemRelation));
+						outDrop.println("System added: " + formatRelation(systemRelation));
 					} else if (systemRelation == null) {
-						System.out.println("System dropped: " + formatRelation(goldRelation));
+						outDrop.println("System dropped: " + formatRelation(goldRelation));
 					} else if (!systemRelation.getCategory().equals(goldRelation.getCategory())) {
 						String label = systemRelation.getCategory();
-						System.out.printf("System labeled %s for %s\n", label, formatRelation(goldRelation));
+						outDrop.printf("System labeled %s for %s\n", label, formatRelation(goldRelation));
 					} else{
-						System.out.println("Nailed it! " + formatRelation(systemRelation));
+						outDrop.println("Nailed it! " + formatRelation(systemRelation));
 					}
 				}
 			}
 		}
+		outDrop.close();
 		return stats;
 	}
 
@@ -453,79 +505,79 @@ EvaluationOfTemporalRelations_ImplBase{
   }
 	 */
 
-//		private static <SPAN_TYPE> Collection<BinaryTextRelation> removeNonGoldRelations(
-//				Collection<BinaryTextRelation> systemRelations,
-//				Collection<BinaryTextRelation> goldRelations, Function<BinaryTextRelation, ?> getSpan) {
-//			//remove non-gold pairs from system relations:
-//			Set<BinaryTextRelation> goodSys = Sets.newHashSet();
-//			Set<SPAN_TYPE> goldspans = new HashSet<SPAN_TYPE>();
-//			
-//			for (BinaryTextRelation relation : goldRelations) {
-//				goldspans.add(((SPAN_TYPE) getSpan.apply(relation)));			
-//			}
-//			
-//			for (BinaryTextRelation relation : systemRelations) {
-//				if (goldspans.contains(((SPAN_TYPE) getSpan.apply(relation)))) {
-//					goodSys.add(relation);
-//				}
-//			}
-//			
-//			return goodSys;
-//		}
-	
-	private static boolean matchSpan(Annotation arg1, Annotation arg2) {
-		boolean result = false;
-		result = arg1.getBegin() == arg2.getBegin() && arg1.getEnd() == arg2.getEnd();
-		return result;
-	}
-	
-//	@SuppressWarnings("unchecked")
-//	private static <SPAN> Collection<BinaryTextRelation> getDuplicateRelations(
-//			Collection<BinaryTextRelation> goldRelations,
-//			Function<BinaryTextRelation, ?> getSpan) {
-//		Set<BinaryTextRelation> duplicateRelations = Sets.newHashSet();
-//		//build a multimap that map gold span to gold relation
-//		Multimap<SPAN, BinaryTextRelation> spanToRelation = HashMultimap.create();
-//		for (BinaryTextRelation relation : goldRelations) {
-//			spanToRelation.put((SPAN) getSpan.apply(relation), relation);			
-//		}
-//		for (SPAN span: spanToRelation.keySet()){
-//			Collection<BinaryTextRelation> relations = spanToRelation.get(span);
-//			if(relations.size()>1){//if same span maps to multiple relations
-//				duplicateRelations.addAll(relations);
-//			}
-//		}
-//		return duplicateRelations;
-//	}
-
-//	private static Collection<BinaryTextRelation> removeNonGoldRelations(
-//			Collection<BinaryTextRelation> systemRelations, Collection<BinaryTextRelation> goldRelations) {
+//	private static <SPAN_TYPE> Collection<BinaryTextRelation> removeNonGoldRelations(
+//			Collection<BinaryTextRelation> systemRelations,
+//			Collection<BinaryTextRelation> goldRelations, Function<BinaryTextRelation, ?> getSpan) {
 //		//remove non-gold pairs from system relations:
 //		Set<BinaryTextRelation> goodSys = Sets.newHashSet();
+//		Set<SPAN_TYPE> goldspans = new HashSet<>();
+//
+//		for (BinaryTextRelation relation : goldRelations) {
+//			goldspans.add(((SPAN_TYPE) getSpan.apply(relation)));			
+//		}
 //
-//		for(BinaryTextRelation sysrel : systemRelations){
-//			Annotation sysArg1 = sysrel.getArg1().getArgument();
-//			Annotation sysArg2 = sysrel.getArg2().getArgument();
-//			for(BinaryTextRelation goldrel : goldRelations){
-//				Annotation goldArg1 = goldrel.getArg1().getArgument();
-//				Annotation goldArg2 = goldrel.getArg2().getArgument();
-//				if(matchSpan(sysArg1, goldArg1) && matchSpan(sysArg2, goldArg2)){
-//					goodSys.add(sysrel);
-//					continue;
-//				}else if (matchSpan(sysArg2, goldArg1) && matchSpan(sysArg1, goldArg2)){//the order of system pair was flipped 
-//					if(sysrel.getCategory().equals("OVERLAP")){ //if the relation is overlap, and the arg order was flipped, then change back the order
-//						RelationArgument tempArg = (RelationArgument) sysrel.getArg1().clone();
-//						sysrel.setArg1((RelationArgument) sysrel.getArg2().clone());
-//						sysrel.setArg2(tempArg);
-//					}//for other types of relation, still maintain the type.
-//					goodSys.add(sysrel);
-//					continue;
-//				}
+//		for (BinaryTextRelation relation : systemRelations) {
+//			if (goldspans.contains(((SPAN_TYPE) getSpan.apply(relation)))) {
+//				goodSys.add(relation);
 //			}
 //		}
 //
 //		return goodSys;
 //	}
+//
+//	private static boolean matchSpan(Annotation arg1, Annotation arg2) {
+//		boolean result = false;
+//		result = arg1.getBegin() == arg2.getBegin() && arg1.getEnd() == arg2.getEnd();
+//		return result;
+//	}
+
+	//	@SuppressWarnings("unchecked")
+	//	private static <SPAN> Collection<BinaryTextRelation> getDuplicateRelations(
+	//			Collection<BinaryTextRelation> goldRelations,
+	//			Function<BinaryTextRelation, ?> getSpan) {
+	//		Set<BinaryTextRelation> duplicateRelations = Sets.newHashSet();
+	//		//build a multimap that map gold span to gold relation
+	//		Multimap<SPAN, BinaryTextRelation> spanToRelation = HashMultimap.create();
+	//		for (BinaryTextRelation relation : goldRelations) {
+	//			spanToRelation.put((SPAN) getSpan.apply(relation), relation);			
+	//		}
+	//		for (SPAN span: spanToRelation.keySet()){
+	//			Collection<BinaryTextRelation> relations = spanToRelation.get(span);
+	//			if(relations.size()>1){//if same span maps to multiple relations
+	//				duplicateRelations.addAll(relations);
+	//			}
+	//		}
+	//		return duplicateRelations;
+	//	}
+
+	//	private static Collection<BinaryTextRelation> removeNonGoldRelations(
+	//			Collection<BinaryTextRelation> systemRelations, Collection<BinaryTextRelation> goldRelations) {
+	//		//remove non-gold pairs from system relations:
+	//		Set<BinaryTextRelation> goodSys = Sets.newHashSet();
+	//
+	//		for(BinaryTextRelation sysrel : systemRelations){
+	//			Annotation sysArg1 = sysrel.getArg1().getArgument();
+	//			Annotation sysArg2 = sysrel.getArg2().getArgument();
+	//			for(BinaryTextRelation goldrel : goldRelations){
+	//				Annotation goldArg1 = goldrel.getArg1().getArgument();
+	//				Annotation goldArg2 = goldrel.getArg2().getArgument();
+	//				if(matchSpan(sysArg1, goldArg1) && matchSpan(sysArg2, goldArg2)){
+	//					goodSys.add(sysrel);
+	//					continue;
+	//				}else if (matchSpan(sysArg2, goldArg1) && matchSpan(sysArg1, goldArg2)){//the order of system pair was flipped 
+	//					if(sysrel.getCategory().equals("OVERLAP")){ //if the relation is overlap, and the arg order was flipped, then change back the order
+	//						RelationArgument tempArg = (RelationArgument) sysrel.getArg1().clone();
+	//						sysrel.setArg1((RelationArgument) sysrel.getArg2().clone());
+	//						sysrel.setArg2(tempArg);
+	//					}//for other types of relation, still maintain the type.
+	//					goodSys.add(sysrel);
+	//					continue;
+	//				}
+	//			}
+	//		}
+	//
+	//		return goodSys;
+	//	}
 
 
 	public static class RemoveEventEventRelations extends JCasAnnotator_ImplBase {
@@ -560,6 +612,66 @@ EvaluationOfTemporalRelations_ImplBase{
 
 		}
 	}
+	
+	public static class AddPotentialRelations extends JCasAnnotator_ImplBase {
+		public static final String PARAM_RELATION_VIEW = "RelationView";
+		@ConfigurationParameter(name = PARAM_RELATION_VIEW,mandatory=false)
+		private String relationViewName = CAS.NAME_DEFAULT_SOFA;
+
+		@Override
+		public void process(JCas jCas) throws AnalysisEngineProcessException {
+			JCas relationView;
+			try {
+				relationView = jCas.getView(this.relationViewName);
+			} catch (CASException e) {
+				throw new AnalysisEngineProcessException(e);
+			}
+
+			Map<EventMention, Collection<EventMention>> coveringMap =
+					  JCasUtil.indexCovering(relationView, EventMention.class, EventMention.class);
+			for(TemporalTextRelation relation : Lists.newArrayList(JCasUtil.select(relationView, TemporalTextRelation.class))){
+				Annotation arg1 = relation.getArg1().getArgument();
+				Annotation arg2 = relation.getArg2().getArgument();
+				EventMention event = null;
+				if(arg1 instanceof EventMention && arg2 instanceof TimeMention){
+					event = (EventMention) arg1;
+					Collection<EventMention> eventList = coveringMap.get(event);
+					for(EventMention covEvent : eventList){
+						if(!covEvent.getClass().equals(EventMention.class)){
+							createRelation(relationView, covEvent, arg2, relation.getCategory());
+						}
+					}
+				}else if(arg2 instanceof EventMention && arg1 instanceof TimeMention){
+					event = (EventMention) arg2;
+					Collection<EventMention> eventList = coveringMap.get(event);
+					for(EventMention covEvent : eventList){
+						if(!covEvent.getClass().equals(EventMention.class)){
+							createRelation(relationView, arg1, covEvent, relation.getCategory());
+						}
+					}
+				}
+			}
+
+		}
+
+		private static void createRelation(JCas jCas, Annotation arg1,
+				Annotation arg2, String category) {
+			RelationArgument relArg1 = new RelationArgument(jCas);
+			relArg1.setArgument(arg1);
+			relArg1.setRole("Arg1");
+			relArg1.addToIndexes();
+			RelationArgument relArg2 = new RelationArgument(jCas);
+			relArg2.setArgument(arg2);
+			relArg2.setRole("Arg2");
+			relArg2.addToIndexes();
+			TemporalTextRelation relation = new TemporalTextRelation(jCas);
+			relation.setArg1(relArg1);
+			relation.setArg2(relArg2);
+			relation.setCategory(category);
+			relation.addToIndexes();
+			
+		}
+	}
 
 	/*  public static class RemoveNonTLINKRelations extends JCasAnnotator_ImplBase {
     @Override
@@ -744,7 +856,7 @@ EvaluationOfTemporalRelations_ImplBase{
 			}
 
 			// look for X -> Y -> Z containment chains and add X -> Z relations
-			Deque<Annotation> todo = new ArrayDeque<Annotation>(isContainedIn.keySet());
+			Deque<Annotation> todo = new ArrayDeque<>(isContainedIn.keySet());
 			while (!todo.isEmpty()) {
 				Annotation next = todo.removeFirst();
 				for (Annotation parent : Lists.newArrayList(isContainedIn.get(next))) {
@@ -788,7 +900,7 @@ EvaluationOfTemporalRelations_ImplBase{
 		}
 
 	}
-	
+
 	public static class AddContain2Overlap extends JCasAnnotator_ImplBase {
 
 		@Override
@@ -814,7 +926,7 @@ EvaluationOfTemporalRelations_ImplBase{
 			}
 		}
 	}
-	
+
 	public static class AddFlippedOverlap extends JCasAnnotator_ImplBase {
 
 		@Override
@@ -854,19 +966,19 @@ EvaluationOfTemporalRelations_ImplBase{
 			}
 		}
 	}
-	
+
 	public static class AddClosure extends JCasAnnotator_ImplBase {
 
 		@Override
 		public void process(JCas jCas) throws AnalysisEngineProcessException {
-			
+
 			Multimap<List<Annotation>, BinaryTextRelation> annotationsToRelation = HashMultimap.create();
 			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)){
 				String relationType = relation.getCategory();
 				if(validTemporalType(relationType)){
 					Annotation arg1 = relation.getArg1().getArgument();
-			        Annotation arg2 = relation.getArg2().getArgument();
-			        annotationsToRelation.put(Arrays.asList(arg1, arg2), relation);
+					Annotation arg2 = relation.getArg2().getArgument();
+					annotationsToRelation.put(Arrays.asList(arg1, arg2), relation);
 				}
 			}
 			for (List<Annotation> span: Lists.newArrayList(annotationsToRelation.keySet())){
@@ -895,26 +1007,26 @@ EvaluationOfTemporalRelations_ImplBase{
 				}
 			}
 
-			ArrayList<BinaryTextRelation> temporalRelation = new ArrayList<BinaryTextRelation>(annotationsToRelation.values());//new ArrayList<BinaryTextRelation>();
-//			Map<List<Annotation>, BinaryTextRelation> temporalRelationLookup = new HashMap<List<Annotation>, BinaryTextRelation>();
-//
-//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)){
-//				String relationType = relation.getCategory();
-//				if(validTemporalType(relationType)){
-//					Annotation arg1 = relation.getArg1().getArgument();
-//			        Annotation arg2 = relation.getArg2().getArgument();
-//			        BinaryTextRelation tempRelation = temporalRelationLookup.get(Arrays.asList(arg1, arg2));
-//					if( tempRelation == null){
-//						temporalRelation.add(relation);					
-//				        temporalRelationLookup.put(Arrays.asList(arg1, arg2), relation);
-//					}else{//if there is duplicate
-//						relation.getArg1().removeFromIndexes();
-//						relation.getArg2().removeFromIndexes();
-//						relation.removeFromIndexes();
-//					}
-//					
-//				}
-//			}
+			ArrayList<BinaryTextRelation> temporalRelation = new ArrayList<>(annotationsToRelation.values());//new ArrayList<BinaryTextRelation>();
+			//			Map<List<Annotation>, BinaryTextRelation> temporalRelationLookup = new HashMap<List<Annotation>, BinaryTextRelation>();
+			//
+			//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)){
+			//				String relationType = relation.getCategory();
+			//				if(validTemporalType(relationType)){
+			//					Annotation arg1 = relation.getArg1().getArgument();
+			//			        Annotation arg2 = relation.getArg2().getArgument();
+			//			        BinaryTextRelation tempRelation = temporalRelationLookup.get(Arrays.asList(arg1, arg2));
+			//					if( tempRelation == null){
+			//						temporalRelation.add(relation);					
+			//				        temporalRelationLookup.put(Arrays.asList(arg1, arg2), relation);
+			//					}else{//if there is duplicate
+			//						relation.getArg1().removeFromIndexes();
+			//						relation.getArg2().removeFromIndexes();
+			//						relation.removeFromIndexes();
+			//					}
+			//					
+			//				}
+			//			}
 
 			if (!temporalRelation.isEmpty()){
 				TLinkTypeArray2 relationArray = new TLinkTypeArray2(temporalRelation, new AnnotationIdCollection(temporalRelation));
@@ -936,10 +1048,10 @@ EvaluationOfTemporalRelations_ImplBase{
 						addedCount++;
 					}		
 				}
-				
+
+				System.out.println( "**************************************************************");
+				System.out.println( "Finally added closure relations: " + addedCount );
 				System.out.println( "**************************************************************");
-			    System.out.println( "Finally added closure relations: " + addedCount );
-			    System.out.println( "**************************************************************");
 			}			
 
 		}
@@ -951,107 +1063,107 @@ EvaluationOfTemporalRelations_ImplBase{
 		}
 	}
 
-//	public static class AddTransitiveBeforeAndOnRelations extends JCasAnnotator_ImplBase {
-//
-//		@Override
-//		public void process(JCas jCas) throws AnalysisEngineProcessException {
-//
-//			// collect many-to-many mappings of containment relations 
-//			Multimap<Annotation, Annotation> contains = HashMultimap.create();
-//			Multimap<Annotation, Annotation> before = HashMultimap.create();
-//			Multimap<Annotation, Annotation> endson = HashMultimap.create();
-//			Multimap<Annotation, Annotation> beginson = HashMultimap.create();
-//			Set<BinaryTextRelation> beforeRel = Sets.newHashSet();
-//
-//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
-//				if (relation.getCategory().equals("CONTAINS")) {
-//					Annotation arg1 = relation.getArg1().getArgument();
-//					Annotation arg2 = relation.getArg2().getArgument();
-//					contains.put(arg1, arg2);
-//				}else if (relation.getCategory().equals("BEFORE")) {
-//					Annotation arg1 = relation.getArg1().getArgument();
-//					Annotation arg2 = relation.getArg2().getArgument();
-//					before.put(arg1, arg2);
-//					beforeRel.add(relation);
-//				}else if (relation.getCategory().equals("ENDS-ON")) {
-//					Annotation arg1 = relation.getArg1().getArgument();
-//					Annotation arg2 = relation.getArg2().getArgument();
-//					endson.put(arg1, arg2);
-//					if (!endson.containsEntry(arg2, arg1)) {
-//						endson.put(arg2, arg1);
-//					}
-//				}else if (relation.getCategory().equals("BEGINS-ON")) {
-//					Annotation arg1 = relation.getArg1().getArgument();
-//					Annotation arg2 = relation.getArg2().getArgument();
-//					beginson.put(arg1, arg2);
-//					if (!beginson.containsEntry(arg2, arg1)) {
-//						beginson.put(arg2, arg1);
-//					}
-//				}
-//			}
-//
-//			// for A BEFORE B, check if A and B Contain anything
-//			for (BinaryTextRelation brelation : beforeRel) {
-//				Annotation argA = brelation.getArg1().getArgument();
-//				Annotation argB = brelation.getArg2().getArgument();
-//				//add contained before
-//				for (Annotation childA : contains.get(argA)) {
-//					for (Annotation childB : contains.get(argB)) {
-//						if (!before.containsEntry(childA, childB)) {
-//							//create a new before relation:
-//							RelationArgument arg1 = new RelationArgument(jCas);
-//							arg1.setArgument(childA);
-//							RelationArgument arg2 = new RelationArgument(jCas);
-//							arg2.setArgument(childB);
-//							BinaryTextRelation relation = new BinaryTextRelation(jCas);
-//							relation.setArg1(arg1);
-//							relation.setArg2(arg2);
-//							relation.setCategory("BEFORE");
-//							arg1.addToIndexes();
-//							arg2.addToIndexes();
-//							relation.addToIndexes();
-//							before.put(childA, childB);
-//						}
-//					}
-//				}
-//				//add ends-on A
-//				for (Annotation endsOnA : endson.get(argA)) {
-//					if (!before.containsEntry(endsOnA, argB)) {
-//						//create a new before relation:
-//						RelationArgument arg1 = new RelationArgument(jCas);
-//						arg1.setArgument(endsOnA);
-//						RelationArgument arg2 = new RelationArgument(jCas);
-//						arg2.setArgument(argB);
-//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
-//						relation.setArg1(arg1);
-//						relation.setArg2(arg2);
-//						relation.setCategory("BEFORE");
-//						arg1.addToIndexes();
-//						arg2.addToIndexes();
-//						relation.addToIndexes();
-//						before.put(endsOnA, argB);
-//					}
-//				}
-//				//add begins-on B
-//				for (Annotation beginsOnB : beginson.get(argB)) {
-//					if (!before.containsEntry(argA, beginsOnB)) {
-//						//create a new before relation:
-//						RelationArgument arg1 = new RelationArgument(jCas);
-//						arg1.setArgument(argA);
-//						RelationArgument arg2 = new RelationArgument(jCas);
-//						arg2.setArgument(beginsOnB);
-//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
-//						relation.setArg1(arg1);
-//						relation.setArg2(arg2);
-//						relation.setCategory("BEFORE");
-//						arg1.addToIndexes();
-//						arg2.addToIndexes();
-//						relation.addToIndexes();
-//						before.put(argA, beginsOnB);
-//					}
-//				}
-//			}
-//		}
-//
-//	}
+	//	public static class AddTransitiveBeforeAndOnRelations extends JCasAnnotator_ImplBase {
+	//
+	//		@Override
+	//		public void process(JCas jCas) throws AnalysisEngineProcessException {
+	//
+	//			// collect many-to-many mappings of containment relations 
+	//			Multimap<Annotation, Annotation> contains = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> before = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> endson = HashMultimap.create();
+	//			Multimap<Annotation, Annotation> beginson = HashMultimap.create();
+	//			Set<BinaryTextRelation> beforeRel = Sets.newHashSet();
+	//
+	//			for (BinaryTextRelation relation : JCasUtil.select(jCas, BinaryTextRelation.class)) {
+	//				if (relation.getCategory().equals("CONTAINS")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					contains.put(arg1, arg2);
+	//				}else if (relation.getCategory().equals("BEFORE")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					before.put(arg1, arg2);
+	//					beforeRel.add(relation);
+	//				}else if (relation.getCategory().equals("ENDS-ON")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					endson.put(arg1, arg2);
+	//					if (!endson.containsEntry(arg2, arg1)) {
+	//						endson.put(arg2, arg1);
+	//					}
+	//				}else if (relation.getCategory().equals("BEGINS-ON")) {
+	//					Annotation arg1 = relation.getArg1().getArgument();
+	//					Annotation arg2 = relation.getArg2().getArgument();
+	//					beginson.put(arg1, arg2);
+	//					if (!beginson.containsEntry(arg2, arg1)) {
+	//						beginson.put(arg2, arg1);
+	//					}
+	//				}
+	//			}
+	//
+	//			// for A BEFORE B, check if A and B Contain anything
+	//			for (BinaryTextRelation brelation : beforeRel) {
+	//				Annotation argA = brelation.getArg1().getArgument();
+	//				Annotation argB = brelation.getArg2().getArgument();
+	//				//add contained before
+	//				for (Annotation childA : contains.get(argA)) {
+	//					for (Annotation childB : contains.get(argB)) {
+	//						if (!before.containsEntry(childA, childB)) {
+	//							//create a new before relation:
+	//							RelationArgument arg1 = new RelationArgument(jCas);
+	//							arg1.setArgument(childA);
+	//							RelationArgument arg2 = new RelationArgument(jCas);
+	//							arg2.setArgument(childB);
+	//							BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//							relation.setArg1(arg1);
+	//							relation.setArg2(arg2);
+	//							relation.setCategory("BEFORE");
+	//							arg1.addToIndexes();
+	//							arg2.addToIndexes();
+	//							relation.addToIndexes();
+	//							before.put(childA, childB);
+	//						}
+	//					}
+	//				}
+	//				//add ends-on A
+	//				for (Annotation endsOnA : endson.get(argA)) {
+	//					if (!before.containsEntry(endsOnA, argB)) {
+	//						//create a new before relation:
+	//						RelationArgument arg1 = new RelationArgument(jCas);
+	//						arg1.setArgument(endsOnA);
+	//						RelationArgument arg2 = new RelationArgument(jCas);
+	//						arg2.setArgument(argB);
+	//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//						relation.setArg1(arg1);
+	//						relation.setArg2(arg2);
+	//						relation.setCategory("BEFORE");
+	//						arg1.addToIndexes();
+	//						arg2.addToIndexes();
+	//						relation.addToIndexes();
+	//						before.put(endsOnA, argB);
+	//					}
+	//				}
+	//				//add begins-on B
+	//				for (Annotation beginsOnB : beginson.get(argB)) {
+	//					if (!before.containsEntry(argA, beginsOnB)) {
+	//						//create a new before relation:
+	//						RelationArgument arg1 = new RelationArgument(jCas);
+	//						arg1.setArgument(argA);
+	//						RelationArgument arg2 = new RelationArgument(jCas);
+	//						arg2.setArgument(beginsOnB);
+	//						BinaryTextRelation relation = new BinaryTextRelation(jCas);
+	//						relation.setArg1(arg1);
+	//						relation.setArg2(arg2);
+	//						relation.setCategory("BEFORE");
+	//						arg1.addToIndexes();
+	//						arg2.addToIndexes();
+	//						relation.addToIndexes();
+	//						before.put(argA, beginsOnB);
+	//					}
+	//				}
+	//			}
+	//		}
+	//
+	//	}
 }



Mime
View raw message