ctakes-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c...@apache.org
Subject svn commit: r1745621 - in /ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal: ae/ ae/feature/selection/ eval/
Date Thu, 26 May 2016 15:54:37 GMT
Author: clin
Date: Thu May 26 15:54:37 2016
New Revision: 1745621

URL: http://svn.apache.org/viewvc?rev=1745621&view=rev
Log:
add z-score code for feature normalization. Due to the low results on ET relations, z-score
was not used.
add glove 6b 50d vector for embeddings.

Added:
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/selection/ZscoreNormalizationExtractor.java
Modified:
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeSelfRelationAnnotator.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/TemporalRelationExtractorAnnotator.java
    ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java

Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeSelfRelationAnnotator.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeSelfRelationAnnotator.java?rev=1745621&r1=1745620&r2=1745621&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeSelfRelationAnnotator.java
(original)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/EventTimeSelfRelationAnnotator.java
Thu May 26 15:54:37 2016
@@ -73,6 +73,7 @@ import org.apache.uima.resource.Resource
 import org.cleartk.ml.CleartkAnnotator;
 import org.cleartk.ml.DataWriter;
 import org.cleartk.ml.feature.extractor.CleartkExtractorException;
+//import org.cleartk.ml.feature.transform.InstanceDataWriter; //used for normalization
 import org.cleartk.ml.jar.DefaultDataWriterFactory;
 import org.cleartk.ml.jar.DirectoryDataWriterFactory;
 import org.cleartk.ml.jar.GenericJarClassifierFactory;
@@ -129,7 +130,7 @@ public class EventTimeSelfRelationAnnota
 
 	@Override
 	protected List<RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation>>
getFeatureExtractors() {
-		final String vectorFile = "org/apache/ctakes/temporal/mimic_vectors.txt";
+		final String vectorFile = "org/apache/ctakes/temporal/glove.6B.50d.txt";
 		try {
 			this.embedingExtractor = new RelationEmbeddingFeatureExtractor(vectorFile);
 		} catch (CleartkExtractorException e) {
@@ -266,4 +267,19 @@ public class EventTimeSelfRelationAnnota
 
 		return category;
 	}
+
+	/**used for normalization
+	public static AnalysisEngineDescription createDataWriterDescription(Class<InstanceDataWriter>
dataWriterClass,
+			File outputDirectory, float probabilityOfKeepingANegativeExample) throws ResourceInitializationException
{
+		return AnalysisEngineFactory.createEngineDescription(
+				EventTimeSelfRelationAnnotator.class,
+				CleartkAnnotator.PARAM_IS_TRAINING,
+				true,
+				DefaultDataWriterFactory.PARAM_DATA_WRITER_CLASS_NAME,
+				dataWriterClass,
+				DirectoryDataWriterFactory.PARAM_OUTPUT_DIRECTORY,
+				outputDirectory,
+				RelationExtractorAnnotator.PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
+				probabilityOfKeepingANegativeExample);
+	}*/
 }

Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/TemporalRelationExtractorAnnotator.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/TemporalRelationExtractorAnnotator.java?rev=1745621&r1=1745620&r2=1745621&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/TemporalRelationExtractorAnnotator.java
(original)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/TemporalRelationExtractorAnnotator.java
Thu May 26 15:54:37 2016
@@ -18,6 +18,9 @@
  */
 package org.apache.ctakes.temporal.ae;
 
+//import java.io.File; //for normalization
+//import java.io.IOException;//for normalization
+//import java.net.URI;//for normalization
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -34,6 +37,7 @@ import org.apache.ctakes.relationextract
 import org.apache.ctakes.relationextractor.ae.features.PhraseChunkingExtractor;
 import org.apache.ctakes.relationextractor.ae.features.RelationFeaturesExtractor;
 import org.apache.ctakes.relationextractor.ae.features.TokenFeaturesExtractor;
+//import org.apache.ctakes.temporal.ae.feature.selection.ZscoreNormalizationExtractor;//for
normalization
 import org.apache.ctakes.temporal.utils.SoftMaxUtil;
 import org.apache.ctakes.typesystem.type.relation.BinaryTextRelation;
 import org.apache.ctakes.typesystem.type.relation.RelationArgument;
@@ -59,305 +63,351 @@ import com.google.common.collect.Lists;
 
 public abstract class TemporalRelationExtractorAnnotator extends CleartkAnnotator<String>
{
 
-  public static final String NO_RELATION_CATEGORY = "-NONE-";
+	public static final String NO_RELATION_CATEGORY = "-NONE-";
 
-  public static final String PARAM_PROB_VIEW = "ProbView";
-  @ConfigurationParameter(name=PARAM_PROB_VIEW, mandatory=false)
-  private String probViewname = null;
-
-  public static final String PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE =
-      "ProbabilityOfKeepingANegativeExample";
-  
-  public static Map<String, Integer> category_frequency = new LinkedHashMap<>();
-
-  @ConfigurationParameter(
-      name = PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
-      mandatory = false,
-      description = "probability that a negative example should be retained for training")
-  protected double probabilityOfKeepingANegativeExample = 1.0;
-
-  protected Random coin = new Random(0);
-
-  private List<RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation>>
featureExtractors = this.getFeatureExtractors();
-
-  private Class<? extends Annotation> coveringClass = getCoveringClass();
-
-  /**
-   * Defines the list of feature extractors used by the classifier. Subclasses
-   * may override this method to provide a different set of feature extractors.
-   * 
-   * @return The list of feature extractors to use.
-   */
-  protected List<RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation>>
getFeatureExtractors() {
-    return Lists.newArrayList(
-        new TokenFeaturesExtractor(),
-        new PartOfSpeechFeaturesExtractor(),
-        new PhraseChunkingExtractor(),
-        new NamedEntityFeaturesExtractor(),
-        new DependencyTreeFeaturesExtractor(),
-        new DependencyPathFeaturesExtractor());
-  }
-
-  protected Class<? extends BinaryTextRelation> getRelationClass() {
-    return BinaryTextRelation.class;
-  }
-
-  /*
-   * Defines the type of annotation that the relation exists within (sentence,
-   * document, segment)
-   */
-  protected abstract Class<? extends Annotation> getCoveringClass();
-
-  /**
-   * Selects the relevant mentions/annotations within a covering annotation for
-   * relation identification/extraction.
-   */
-  protected abstract List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
-      JCas identifiedAnnotationView,
-      Annotation coveringAnnotation);
-
-  /**
-   * Workaround for https://code.google.com/p/cleartk/issues/detail?id=346
-   * 
-   * Not intended for external use
-   */
-  static void allowClassifierModelOnClasspath(UimaContext context) {
-    String modelPathParam = GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH;
-    String modelPath = (String) context.getConfigParameterValue(modelPathParam);
-    if (modelPath != null) {
-      URL modelClasspathURL = TemporalRelationExtractorAnnotator.class.getResource(modelPath);
-      if (modelClasspathURL != null) {
-        UimaContextAdmin contextAdmin = (UimaContextAdmin) context;
-        ConfigurationManager manager = contextAdmin.getConfigurationManager();
-        String qualifiedModelPathParam = contextAdmin.getQualifiedContextName() + modelPathParam;
-        manager.setConfigParameterValue(qualifiedModelPathParam, modelClasspathURL.toString());
-      }
-    }
-  }
-
-  @Override
-  public void initialize(UimaContext context) throws ResourceInitializationException {
-    allowClassifierModelOnClasspath(context);
-    super.initialize(context);
-  }
-
-  /*
-   * Implement the standard UIMA process method.
-   */
-  @Override
-  public void process(JCas jCas) throws AnalysisEngineProcessException {
-
-    // lookup from pair of annotations to binary text relation
-    // note: assumes that there will be at most one relation per pair
-    Map<List<Annotation>, BinaryTextRelation> relationLookup;
-    relationLookup = new HashMap<>();
-    if (this.isTraining()) {
-      relationLookup = new HashMap<>();
-      for (BinaryTextRelation relation : JCasUtil.select(jCas, this.getRelationClass()))
{
-        Annotation arg1 = relation.getArg1().getArgument();
-        Annotation arg2 = relation.getArg2().getArgument();
-        // The key is a list of args so we can do bi-directional lookup
-        List<Annotation> key = Arrays.asList(arg1, arg2);
-        if(relationLookup.containsKey(key)){
-         String reln = relationLookup.get(key).getCategory();
-         System.err.println("Error in: "+ ViewUriUtil.getURI(jCas).toString());
-         System.err.println("Error! This attempted relation " + relation.getCategory() +
" already has a relation " + reln + " at this span: " + arg1.getCoveredText() + " -- " + arg2.getCoveredText());
-        }
-        relationLookup.put(key, relation);
-      }
-    }
-
-    // walk through each sentence in the text
-    for (Annotation coveringAnnotation : JCasUtil.select(jCas, coveringClass)) {
-
-      // collect all relevant relation arguments from the sentence
-      List<IdentifiedAnnotationPair> candidatePairs =
-          this.getCandidateRelationArgumentPairs(jCas, coveringAnnotation);
-
-      // walk through the pairs of annotations
-      for (IdentifiedAnnotationPair pair : candidatePairs) {
-        IdentifiedAnnotation arg1 = pair.getArg1();
-        IdentifiedAnnotation arg2 = pair.getArg2();
-        // apply all the feature extractors to extract the list of features
-        List<Feature> features = new ArrayList<>();
-        for (RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation> extractor
: this.featureExtractors) {
-        	 List<Feature> feats = extractor.extract(jCas, arg1, arg2);
-        	 if (feats != null)  features.addAll(feats);
-        }
-
-        // sanity check on feature values
-        for (Feature feature : features) {
-          if (feature.getValue() == null) {
-        	feature.setValue("NULL");
-            String message = String.format("Null value found in %s from %s", feature, features);
-            System.err.println(message);
-//            throw new IllegalArgumentException(String.format(message, feature, features));
-          }
-        }
-
-        // during training, feed the features to the data writer
-        if (this.isTraining()) {
-          String category = this.getRelationCategory(relationLookup, arg1, arg2);
-          if (category == null) {
-            continue;
-          }
-
-          //populate category_frequency count:
-          if(category_frequency.containsKey(category)){
-        	  category_frequency.put(category, category_frequency.get(category)+1);
-          }else{
-        	  category_frequency.put(category, 1);
-          }
-          // create a classification instance and write it to the training data
-          this.dataWriter.write(new Instance<>(category, features));
-        }
-
-        // during classification feed the features to the classifier and create
-        // annotations
-        else {
-//          String predictedCategory = this.classify(features);
-          Map<String,Double> scores = this.classifier.score(features);
-          
-          Map.Entry<String, Double> maxEntry = null;
-          for( Map.Entry<String, Double> entry: scores.entrySet() ){
-          	if(maxEntry == null || entry.getValue().compareTo(maxEntry.getValue()) > 0){
-          		maxEntry = entry;
-          	}
-          }
-          
-          String predictedCategory = null;
-          double confidence = 0d;
-          if(maxEntry != null){
-        	  predictedCategory = maxEntry.getKey();
-        	  confidence = maxEntry.getValue().doubleValue();
-          }
-          
-          // before creating the final relation (and possibly flipping the order of arguments)

-          // create the probabilistic copies in the other cas if that flag is set:
-          if(probViewname != null){
-            try {
-              JCas probView = jCas.getView(probViewname);
-              Map<String,Double> probs = SoftMaxUtil.getDistributionFromScores(scores);
-              
-              for(String label : probs.keySet()){
-                createRelation(probView, arg1, arg2, label, probs.get(label));
-              }
-            } catch (CASException e) {
-              e.printStackTrace();
-              throw new AnalysisEngineProcessException(e);
-            }
-          }
-
-          // add a relation annotation if a true relation was predicted
-          if (predictedCategory != null && !predictedCategory.equals(NO_RELATION_CATEGORY))
{
-
-            // if we predict an inverted relation, reverse the order of the
-            // arguments
-            if (predictedCategory.endsWith("-1")) {
-              predictedCategory = predictedCategory.substring(0, predictedCategory.length()
- 2);
-              IdentifiedAnnotation temp = arg1;
-              arg1 = arg2;
-              arg2 = temp;
-            }
-
-            createRelation(jCas, arg1, arg2, predictedCategory, confidence);
-          }
-        }
-      } // end pair in pairs
-    } // end for(Sentence)
-  }
-
-  /**
-   * Looks up the arguments in the specified lookup table and converts the
-   * relation into a label for classification
-   * 
-   * @return If this category should not be processed for training return
-   *         <i>null</i> otherwise it returns the label sent to the datawriter
-   */
-  protected String getRelationCategory(
-      Map<List<Annotation>, BinaryTextRelation> relationLookup,
-      IdentifiedAnnotation arg1,
-      IdentifiedAnnotation arg2) {
-    BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
-    String category;
-    if (relation != null) {
-      category = relation.getCategory();
-    } else if (coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
-      category = NO_RELATION_CATEGORY;
-    } else {
-      category = null;
-    }
-    return category;
-  }
-
-  /**
-   * Predict an outcome given a set of features. By default, this simply
-   * delegates to the object's <code>classifier</code>. Subclasses may override
-   * this method to implement more complex classification procedures.
-   * 
-   * @param features
-   *          The features to be classified.
-   * @return The predicted outcome (label) for the features.
-   */
-  protected String classify(List<Feature> features) throws CleartkProcessingException
{
-    return this.classifier.classify(features);
-  }
-
-  /**
-   * Create a UIMA relation type based on arguments and the relation label. This
-   * allows subclasses to create/define their own types: e.g. coreference can
-   * create CoreferenceRelation instead of BinaryTextRelation
-   * 
-   * @param jCas
-   *          - JCas object, needed to create new UIMA types
-   * @param arg1
-   *          - First argument to relation
-   * @param arg2
-   *          - Second argument to relation
-   * @param predictedCategory
-   *          - Name of relation
-   * @param confidence 
-   * 		  - Confidence score of the relation prediction
-   */
-  protected void createRelation(
-      JCas jCas,
-      IdentifiedAnnotation arg1,
-      IdentifiedAnnotation arg2,
-      String predictedCategory, 
-      double confidence) {
-    // add the relation to the CAS
-    RelationArgument relArg1 = new RelationArgument(jCas);
-    relArg1.setArgument(arg1);
-    relArg1.setRole("Argument");
-    relArg1.addToIndexes();
-    RelationArgument relArg2 = new RelationArgument(jCas);
-    relArg2.setArgument(arg2);
-    relArg2.setRole("Related_to");
-    relArg2.addToIndexes();
-    BinaryTextRelation relation = new BinaryTextRelation(jCas);
-    relation.setArg1(relArg1);
-    relation.setArg2(relArg2);
-    relation.setCategory(predictedCategory);
-    relation.setConfidence(confidence);
-    relation.addToIndexes();
-  }
-
-  public static class IdentifiedAnnotationPair {
-
-    private final IdentifiedAnnotation arg1;
-    private final IdentifiedAnnotation arg2;
-
-    public IdentifiedAnnotationPair(IdentifiedAnnotation arg1, IdentifiedAnnotation arg2)
{
-      this.arg1 = arg1;
-      this.arg2 = arg2;
-    }
-
-    public final IdentifiedAnnotation getArg1() {
-      return arg1;
-    }
-
-    public final IdentifiedAnnotation getArg2() {
-      return arg2;
-    }
-  }
+	public static final String PARAM_PROB_VIEW = "ProbView";
+	@ConfigurationParameter(name=PARAM_PROB_VIEW, mandatory=false)
+	private String probViewname = null;
+
+	public static final String PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE =
+			"ProbabilityOfKeepingANegativeExample";
+
+	public static Map<String, Integer> category_frequency = new LinkedHashMap<>();
+
+	public static final String MINMAX_EXTRACTOR_KEY = "MINMAXFeatures";
+
+	@ConfigurationParameter(
+			name = PARAM_PROBABILITY_OF_KEEPING_A_NEGATIVE_EXAMPLE,
+			mandatory = false,
+			description = "probability that a negative example should be retained for training")
+	protected double probabilityOfKeepingANegativeExample = 1.0;
+
+	protected Random coin = new Random(0);
+
+	private List<RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation>>
featureExtractors = this.getFeatureExtractors();
+
+	private Class<? extends Annotation> coveringClass = getCoveringClass();
+
+	//private ZscoreNormalizationExtractor<String, Annotation> featureTransformExtractor;//for
normalization
+
+	//protected static URI minmaxExtractorURI;//for normalization
+
+	//private static final String FEATURE_TRANSFORM_NAME = "TransformFeatures";//for normalization
+
+	/**for normalization
+	public static ZscoreNormalizationExtractor<String, Annotation> createMinMaxNormalizationExtractor()
{
+		return new ZscoreNormalizationExtractor<>(FEATURE_TRANSFORM_NAME);
+	}
+
+	public static URI createMinMaxNormalizationExtractorURI(File outputDirectoryName) {
+		minmaxExtractorURI = new File(outputDirectoryName, FEATURE_TRANSFORM_NAME + "_Zscore_extractor.dat").toURI();
+		return minmaxExtractorURI;
+	}
+	*/
+
+	/**
+	 * Defines the list of feature extractors used by the classifier. Subclasses
+	 * may override this method to provide a different set of feature extractors.
+	 * 
+	 * @return The list of feature extractors to use.
+	 */
+	protected List<RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation>>
getFeatureExtractors() {
+		return Lists.newArrayList(
+				new TokenFeaturesExtractor(),
+				new PartOfSpeechFeaturesExtractor(),
+				new PhraseChunkingExtractor(),
+				new NamedEntityFeaturesExtractor(),
+				new DependencyTreeFeaturesExtractor(),
+				new DependencyPathFeaturesExtractor());
+	}
+
+	protected Class<? extends BinaryTextRelation> getRelationClass() {
+		return BinaryTextRelation.class;
+	}
+
+	/*
+	 * Defines the type of annotation that the relation exists within (sentence,
+	 * document, segment)
+	 */
+	protected abstract Class<? extends Annotation> getCoveringClass();
+
+	/**
+	 * Selects the relevant mentions/annotations within a covering annotation for
+	 * relation identification/extraction.
+	 */
+	protected abstract List<IdentifiedAnnotationPair> getCandidateRelationArgumentPairs(
+			JCas identifiedAnnotationView,
+			Annotation coveringAnnotation);
+
+	/**
+	 * Workaround for https://code.google.com/p/cleartk/issues/detail?id=346
+	 * 
+	 * Not intended for external use
+	 */
+	static void allowClassifierModelOnClasspath(UimaContext context) {
+		String modelPathParam = GenericJarClassifierFactory.PARAM_CLASSIFIER_JAR_PATH;
+		String modelPath = (String) context.getConfigParameterValue(modelPathParam);
+		if (modelPath != null) {
+			URL modelClasspathURL = TemporalRelationExtractorAnnotator.class.getResource(modelPath);
+			if (modelClasspathURL != null) {
+				UimaContextAdmin contextAdmin = (UimaContextAdmin) context;
+				ConfigurationManager manager = contextAdmin.getConfigurationManager();
+				String qualifiedModelPathParam = contextAdmin.getQualifiedContextName() + modelPathParam;
+				manager.setConfigParameterValue(qualifiedModelPathParam, modelClasspathURL.toString());
+			}
+		}
+	}
+
+	@Override
+	public void initialize(UimaContext context) throws ResourceInitializationException {
+		allowClassifierModelOnClasspath(context);
+		super.initialize(context);
+//		minmaxExtractor = createMinMaxNormalizationExtractor();
+		/**for normalization
+		if (this.minmaxExtractorURI != null) {
+			try {
+				this.featureTransformExtractor = new ZscoreNormalizationExtractor<>(FEATURE_TRANSFORM_NAME);
+				this.featureTransformExtractor.load(this.minmaxExtractorURI);
+			} catch (IOException e) {
+				throw new ResourceInitializationException(e);
+			}
+		}*/
+	}
+
+	/*
+	 * Implement the standard UIMA process method.
+	 */
+	@Override
+	public void process(JCas jCas) throws AnalysisEngineProcessException {
+
+		// lookup from pair of annotations to binary text relation
+		// note: assumes that there will be at most one relation per pair
+		Map<List<Annotation>, BinaryTextRelation> relationLookup;
+		relationLookup = new HashMap<>();
+		if (this.isTraining()) {
+			relationLookup = new HashMap<>();
+			for (BinaryTextRelation relation : JCasUtil.select(jCas, this.getRelationClass())) {
+				Annotation arg1 = relation.getArg1().getArgument();
+				Annotation arg2 = relation.getArg2().getArgument();
+				// The key is a list of args so we can do bi-directional lookup
+				List<Annotation> key = Arrays.asList(arg1, arg2);
+				if(relationLookup.containsKey(key)){
+					String reln = relationLookup.get(key).getCategory();
+					System.err.println("Error in: "+ ViewUriUtil.getURI(jCas).toString());
+					System.err.println("Error! This attempted relation " + relation.getCategory() + " already
has a relation " + reln + " at this span: " + arg1.getCoveredText() + " -- " + arg2.getCoveredText());
+				}
+				relationLookup.put(key, relation);
+			}
+		}
+
+		// walk through each sentence in the text
+		for (Annotation coveringAnnotation : JCasUtil.select(jCas, coveringClass)) {
+
+			// collect all relevant relation arguments from the sentence
+			List<IdentifiedAnnotationPair> candidatePairs =
+					this.getCandidateRelationArgumentPairs(jCas, coveringAnnotation);
+
+			// walk through the pairs of annotations
+			for (IdentifiedAnnotationPair pair : candidatePairs) {
+				IdentifiedAnnotation arg1 = pair.getArg1();
+				IdentifiedAnnotation arg2 = pair.getArg2();
+				// apply all the feature extractors to extract the list of features
+				List<Feature> features = new ArrayList<>();
+				for (RelationFeaturesExtractor<IdentifiedAnnotation,IdentifiedAnnotation> extractor
: this.featureExtractors) {
+					List<Feature> feats = extractor.extract(jCas, arg1, arg2);
+					if (feats != null)  features.addAll(feats);
+				}
+
+				// sanity check on feature values
+				//List<Feature> transformedFeatures = new ArrayList<>();//for normalization
+				for (Feature feature : features) {
+					if (feature.getValue() == null) {
+						feature.setValue("NULL");
+						String message = String.format("Null value found in %s from %s", feature, features);
+						System.err.println(message);
+						//            throw new IllegalArgumentException(String.format(message, feature, features));
+					}
+					/**for normalization
+					//transform feature:
+					Object featureValue = feature.getValue();
+					if (this.featureTransformExtractor != null) {
+						if (featureValue instanceof Number) {
+							transformedFeatures.add(featureTransformExtractor.transform(feature));
+						}else{
+							transformedFeatures.add(feature);
+						}
+					}*/
+				}
+				
+				/**for normalization
+				//transform features:
+				if (this.featureTransformExtractor != null) {
+					features = transformedFeatures;
+				}*/
+
+				// during training, feed the features to the data writer
+				if (this.isTraining()) {
+					String category = this.getRelationCategory(relationLookup, arg1, arg2);
+					if (category == null) {
+						continue;
+					}
+
+					//populate category_frequency count:
+					if(category_frequency.containsKey(category)){
+						category_frequency.put(category, category_frequency.get(category)+1);
+					}else{
+						category_frequency.put(category, 1);
+					}
+					// create a classification instance and write it to the training data
+					this.dataWriter.write(new Instance<>(category, features));
+				}
+
+				// during classification feed the features to the classifier and create
+				// annotations
+				else {
+					//          String predictedCategory = this.classify(features);
+					Map<String,Double> scores = this.classifier.score(features);
+
+					Map.Entry<String, Double> maxEntry = null;
+					for( Map.Entry<String, Double> entry: scores.entrySet() ){
+						if(maxEntry == null || entry.getValue().compareTo(maxEntry.getValue()) > 0){
+							maxEntry = entry;
+						}
+					}
+
+					String predictedCategory = null;
+					double confidence = 0d;
+					if(maxEntry != null){
+						predictedCategory = maxEntry.getKey();
+						confidence = maxEntry.getValue().doubleValue();
+					}
+
+					// before creating the final relation (and possibly flipping the order of arguments)

+					// create the probabilistic copies in the other cas if that flag is set:
+					if(probViewname != null){
+						try {
+							JCas probView = jCas.getView(probViewname);
+							Map<String,Double> probs = SoftMaxUtil.getDistributionFromScores(scores);
+
+							for(String label : probs.keySet()){
+								createRelation(probView, arg1, arg2, label, probs.get(label));
+							}
+						} catch (CASException e) {
+							e.printStackTrace();
+							throw new AnalysisEngineProcessException(e);
+						}
+					}
+
+					// add a relation annotation if a true relation was predicted
+					if (predictedCategory != null && !predictedCategory.equals(NO_RELATION_CATEGORY))
{
+
+						// if we predict an inverted relation, reverse the order of the
+						// arguments
+						if (predictedCategory.endsWith("-1")) {
+							predictedCategory = predictedCategory.substring(0, predictedCategory.length() - 2);
+							IdentifiedAnnotation temp = arg1;
+							arg1 = arg2;
+							arg2 = temp;
+						}
+
+						createRelation(jCas, arg1, arg2, predictedCategory, confidence);
+					}
+				}
+			} // end pair in pairs
+		} // end for(Sentence)
+	}
+
+	/**
+	 * Looks up the arguments in the specified lookup table and converts the
+	 * relation into a label for classification
+	 * 
+	 * @return If this category should not be processed for training return
+	 *         <i>null</i> otherwise it returns the label sent to the datawriter
+	 */
+	protected String getRelationCategory(
+			Map<List<Annotation>, BinaryTextRelation> relationLookup,
+			IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2) {
+		BinaryTextRelation relation = relationLookup.get(Arrays.asList(arg1, arg2));
+		String category;
+		if (relation != null) {
+			category = relation.getCategory();
+		} else if (coin.nextDouble() <= this.probabilityOfKeepingANegativeExample) {
+			category = NO_RELATION_CATEGORY;
+		} else {
+			category = null;
+		}
+		return category;
+	}
+
+	/**
+	 * Predict an outcome given a set of features. By default, this simply
+	 * delegates to the object's <code>classifier</code>. Subclasses may override
+	 * this method to implement more complex classification procedures.
+	 * 
+	 * @param features
+	 *          The features to be classified.
+	 * @return The predicted outcome (label) for the features.
+	 */
+	protected String classify(List<Feature> features) throws CleartkProcessingException
{
+		return this.classifier.classify(features);
+	}
+
+	/**
+	 * Create a UIMA relation type based on arguments and the relation label. This
+	 * allows subclasses to create/define their own types: e.g. coreference can
+	 * create CoreferenceRelation instead of BinaryTextRelation
+	 * 
+	 * @param jCas
+	 *          - JCas object, needed to create new UIMA types
+	 * @param arg1
+	 *          - First argument to relation
+	 * @param arg2
+	 *          - Second argument to relation
+	 * @param predictedCategory
+	 *          - Name of relation
+	 * @param confidence 
+	 * 		  - Confidence score of the relation prediction
+	 */
+	protected void createRelation(
+			JCas jCas,
+			IdentifiedAnnotation arg1,
+			IdentifiedAnnotation arg2,
+			String predictedCategory, 
+			double confidence) {
+		// add the relation to the CAS
+		RelationArgument relArg1 = new RelationArgument(jCas);
+		relArg1.setArgument(arg1);
+		relArg1.setRole("Argument");
+		relArg1.addToIndexes();
+		RelationArgument relArg2 = new RelationArgument(jCas);
+		relArg2.setArgument(arg2);
+		relArg2.setRole("Related_to");
+		relArg2.addToIndexes();
+		BinaryTextRelation relation = new BinaryTextRelation(jCas);
+		relation.setArg1(relArg1);
+		relation.setArg2(relArg2);
+		relation.setCategory(predictedCategory);
+		relation.setConfidence(confidence);
+		relation.addToIndexes();
+	}
+
+	public static class IdentifiedAnnotationPair {
+
+		private final IdentifiedAnnotation arg1;
+		private final IdentifiedAnnotation arg2;
+
+		public IdentifiedAnnotationPair(IdentifiedAnnotation arg1, IdentifiedAnnotation arg2) {
+			this.arg1 = arg1;
+			this.arg2 = arg2;
+		}
+
+		public final IdentifiedAnnotation getArg1() {
+			return arg1;
+		}
+
+		public final IdentifiedAnnotation getArg2() {
+			return arg2;
+		}
+	}
 }

Added: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/selection/ZscoreNormalizationExtractor.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/selection/ZscoreNormalizationExtractor.java?rev=1745621&view=auto
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/selection/ZscoreNormalizationExtractor.java
(added)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/ae/feature/selection/ZscoreNormalizationExtractor.java
Thu May 26 15:54:37 2016
@@ -0,0 +1,203 @@
+package org.apache.ctakes.temporal.ae.feature.selection;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.tcas.Annotation;
+import org.cleartk.ml.Feature;
+import org.cleartk.ml.Instance;
+import org.cleartk.ml.feature.extractor.CleartkExtractorException;
+import org.cleartk.ml.feature.transform.OneToOneTrainableExtractor_ImplBase;
+
+/**
+ * Use z-score to normalize every numerical feature
+ * @author Chen Lin
+ * 
+ */
+public class ZscoreNormalizationExtractor<OUTCOME_T, FOCUS_T extends Annotation> extends
+OneToOneTrainableExtractor_ImplBase<OUTCOME_T> {
+
+	private boolean isTrained;
+
+	// This is read in after training for use in transformation
+	private Map<String, MeanStdPair> meanStdMap;
+
+	public ZscoreNormalizationExtractor(String name) {
+		super(name);
+		isTrained = false;
+	}
+
+	@Override
+	public Feature transform(Feature feature) {
+		String featureName = feature.getName();
+		Object featureValue = feature.getValue();
+		if (featureValue instanceof Number) {
+			MeanStdPair stats = this.meanStdMap.get(featureName);
+
+			double mmn = 0.5d; // this is the default value we will return if we've never seen the
feature
+			// before
+
+			double value = ((Number) feature.getValue()).doubleValue();
+			// this is the typical case
+			if (stats != null) {
+				mmn = (value - stats.mean) / (stats.std);
+			}
+			return new Feature("Zscore_NORMED_" + featureName, mmn);
+		}
+		return feature;
+	}
+
+	@Override
+	public void train(Iterable<Instance<OUTCOME_T>> instances) {
+		Map<String, ZscoreRunningStat> featureStatsMap = new HashMap<>();
+
+		// keep a running mean and standard deviation for all applicable features
+		for (Instance<OUTCOME_T> instance : instances) {
+			// Grab the matching zmus (zero mean, unit stddev) features from the set of all features
in an
+			// instance
+			for (Feature feature : instance.getFeatures()) {
+
+				String featureName = feature.getName();
+				Object featureValue = feature.getValue();
+				if (featureValue instanceof Number) {
+					ZscoreRunningStat stats;
+					if (featureStatsMap.containsKey(featureName)) {
+						stats = featureStatsMap.get(featureName);
+					} else {
+						stats = new ZscoreRunningStat();
+						featureStatsMap.put(featureName, stats);
+					}
+					stats.add(((Number) featureValue).doubleValue());
+				} else {
+					System.err.println("Ignore non-numeric feature from normalization: "+ featureName +
" with Value: " + featureValue);
+					continue;
+				}
+
+			}
+		}
+
+		this.meanStdMap = new HashMap<>();
+		for (Map.Entry<String, ZscoreRunningStat> entry : featureStatsMap.entrySet()) {
+			ZscoreRunningStat stats = entry.getValue();
+			this.meanStdMap.put(entry.getKey(), new MeanStdPair(stats.getMean(), stats.getStdDev()));
+		}
+
+		this.isTrained = true;
+	}
+
+	@Override
+	public void save(URI zmusDataUri) throws IOException {
+		// Write out tab separated values: feature_name, mean, stddev
+		File out = new File(zmusDataUri);
+		BufferedWriter writer = null;
+		writer = new BufferedWriter(new FileWriter(out));
+
+		for (Map.Entry<String, MeanStdPair> entry : this.meanStdMap.entrySet()) {
+			MeanStdPair pair = entry.getValue();
+			writer.append(String.format(Locale.ROOT, "%s\t%f\t%f\n", entry.getKey(), pair.mean, pair.std));
+		}
+		writer.close();
+	}
+
+	@Override
+	public void load(URI zmusDataUri) throws IOException {
+		// Reads in tab separated values (feature name, min, max)
+		File in = new File(zmusDataUri);
+		BufferedReader reader = null;
+		this.meanStdMap = new HashMap<>();
+		reader = new BufferedReader(new FileReader(in));
+		String line = null;
+		while ((line = reader.readLine()) != null) {
+			String[] featureMeanStddev = line.split("\\t");
+			this.meanStdMap.put(
+					featureMeanStddev[0],
+					new MeanStdPair(
+							Double.parseDouble(featureMeanStddev[1]),
+							Double.parseDouble(featureMeanStddev[2])));
+		}
+		reader.close();
+
+		this.isTrained = true;
+	}
+
+	private static class MeanStdPair {
+
+		public MeanStdPair(double mean, double std) {
+			this.mean = mean;
+			this.std  = std;
+		}
+
+		public double mean;
+
+		public double std;
+	}
+
+	public static class ZscoreRunningStat implements Serializable {
+
+		/**
+		 * for a named feature, maintain its all values, sum, size and mean,
+		 * calculate the std if needed
+		 */
+		private static final long serialVersionUID = 1L;
+		private List<Double> data;
+		private double sum;
+		private double mean;
+		private int n;
+
+		public ZscoreRunningStat() {
+			this.clear();
+		}
+
+		public void add(double x) {
+			this.n++;
+			this.sum += x;
+			this.mean = this.sum/this.n;
+		}
+
+		public void clear() {
+			this.data = new ArrayList<>();
+			this.sum 	= 0;
+			this.n	= 0;
+			this.mean	= 0;
+		}
+
+		public int getNumSamples() {
+			return this.n;
+		}
+
+		private double getVariance()
+		{
+			double temp = 0;
+			for(double a :data)
+				temp += (mean-a)*(mean-a);
+			return temp/this.n;
+		}
+
+		public double getStdDev()
+		{
+			return Math.sqrt(getVariance());
+		}
+
+		public double getMean(){
+			return this.mean;
+		}
+
+	}
+
+	public List<Feature> extract(JCas view, FOCUS_T focusAnnotation) throws CleartkExtractorException
{
+		// TODO Auto-generated method stub
+		return null;
+	}
+}

Modified: ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
URL: http://svn.apache.org/viewvc/ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java?rev=1745621&r1=1745620&r2=1745621&view=diff
==============================================================================
--- ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
(original)
+++ ctakes/trunk/ctakes-temporal/src/main/java/org/apache/ctakes/temporal/eval/EvaluationOfEventTimeRelations.java
Thu May 26 15:54:37 2016
@@ -42,6 +42,7 @@ import org.apache.ctakes.temporal.ae.Tem
 //import org.apache.ctakes.temporal.ae.EventTimeRelationAnnotator;
 //import org.apache.ctakes.temporal.ae.EventEventRelationAnnotator;
 import org.apache.ctakes.temporal.ae.baselines.RecallBaselineEventTimeRelationAnnotator;
+//import org.apache.ctakes.temporal.ae.feature.selection.ZscoreNormalizationExtractor; //for
normalization
 import org.apache.ctakes.temporal.eval.EvaluationOfTemporalRelations_ImplBase.RemoveGoldAttributes;
 import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.WriteAnaforaXML;
 //import org.apache.ctakes.temporal.eval.Evaluation_ImplBase.WriteI2B2XML;
@@ -73,6 +74,9 @@ import org.apache.uima.jcas.tcas.Annotat
 import org.apache.uima.resource.ResourceInitializationException;
 import org.apache.uima.util.FileUtils;
 import org.cleartk.eval.AnnotationStatistics;
+//import org.cleartk.ml.Instance; //for normalization
+//import org.cleartk.ml.feature.transform.InstanceDataWriter;//for normalization
+//import org.cleartk.ml.feature.transform.InstanceStream;//for normalization
 import org.cleartk.ml.jar.JarClassifierBuilder;
 import org.cleartk.ml.liblinear.LibLinearStringOutcomeDataWriter;
 import org.cleartk.ml.libsvm.LibSvmStringOutcomeDataWriter;
@@ -110,7 +114,7 @@ EvaluationOfTemporalRelations_ImplBase{
 
 		@Option
 		public boolean getSkipTrain();
-		
+
 		@Option
 		public boolean getTestOnTrain();
 	}
@@ -133,7 +137,7 @@ EvaluationOfTemporalRelations_ImplBase{
 	protected static ParameterSettings ftParams = new ParameterSettings(DEFAULT_BOTH_DIRECTIONS,
DEFAULT_DOWNSAMPLE, "tk", 
 			1.0, 0.1, "radial basis function", ComboOperator.SUM, 0.5, 0.5);
 	private static Boolean recallModeEvaluation = true;
-	
+
 	static int sysRelationCount;
 	static int closeRelationCount;
 	static int goldRelationCount;
@@ -144,7 +148,7 @@ EvaluationOfTemporalRelations_ImplBase{
 		closeRelationCount = 0;
 		goldRelationCount = 0;
 		closeGoldRelationCount = 0;
-		
+
 		TempRelOptions options = CliFactory.parseArguments(TempRelOptions.class, args);
 		List<Integer> trainItems = null;
 		List<Integer> devItems = null;
@@ -209,11 +213,11 @@ EvaluationOfTemporalRelations_ImplBase{
 			}
 
 			evaluation.printErrors = true;
-			
+
 			//sort list:
 			Collections.sort(training);
 			Collections.sort(testing);
-			
+
 			//test or train or test
 			evaluation.testOnTrain = options.getTestOnTrain();
 			if(evaluation.testOnTrain){
@@ -222,7 +226,7 @@ EvaluationOfTemporalRelations_ImplBase{
 				params.stats = evaluation.trainAndTest(training, testing);//training
 			}
 			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
-//			System.err.println("No closure on gold::Closure on System::Recall Mode");
+			//			System.err.println("No closure on gold::Closure on System::Recall Mode");
 			System.err.println(params.stats);
 
 			System.err.println("System predict relations #: "+ sysRelationCount);
@@ -230,20 +234,20 @@ EvaluationOfTemporalRelations_ImplBase{
 			System.err.println("Gold relations #: "+ goldRelationCount);
 			System.err.println("# of gold relations whose arguments are close: "+ closeGoldRelationCount);
 			//do closure on gold, but not on system, to calculate precision
-//			evaluation.skipTrain = true;
-//			recallModeEvaluation = false;
-//			params.stats = evaluation.trainAndTest(training, testing);//training);//
-//			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
-//			System.err.println("No closure on System::Closure on Gold::Precision Mode");
-//			System.err.println(params.stats);
-//
-//			//do closure on train, but not on test, to calculate plain results
-//			evaluation.skipTrain = true;
-//			evaluation.useClosure = false;
-//			params.stats = evaluation.trainAndTest(training, testing);//training);//
-//			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
-//			System.err.println("Closure on train::No closure on Test::Plain Mode");
-//			System.err.println(params.stats);
+			//			evaluation.skipTrain = true;
+			//			recallModeEvaluation = false;
+			//			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			//			System.err.println("No closure on System::Closure on Gold::Precision Mode");
+			//			System.err.println(params.stats);
+			//
+			//			//do closure on train, but not on test, to calculate plain results
+			//			evaluation.skipTrain = true;
+			//			evaluation.useClosure = false;
+			//			params.stats = evaluation.trainAndTest(training, testing);//training);//
+			//			//      System.err.println(options.getKernelParams() == null ? params : options.getKernelParams());
+			//			System.err.println("Closure on train::No closure on Test::Plain Mode");
+			//			System.err.println(params.stats);
 
 			if(options.getUseTmp()){
 				// won't work because it's not empty. should we be concerned with this or is it responsibility
of 
@@ -328,16 +332,19 @@ EvaluationOfTemporalRelations_ImplBase{
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(Overlap2Contains.class));
 
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveEventEventRelations.class));
-		
+
 		//count how many sentences have timex, and how many sentences have only one timex
 		//aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(CountSentenceContainsTimes.class));
 
 		//add unlabeled nearby system events as potential links: 
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(AddPotentialRelations.class));
 
+		//directory = new File(directory,"event-time");//for normalization
+
 		aggregateBuilder.add(EventTimeSelfRelationAnnotator.createDataWriterDescription(
 				LibLinearStringOutcomeDataWriter.class,
-//								LibSvmStringOutcomeDataWriter.class,
+				//InstanceDataWriter.class,//for normalization
+				//								LibSvmStringOutcomeDataWriter.class,
 				//				TKSVMlightStringOutcomeDataWriter.class,
 				//        TKLIBSVMStringOutcomeDataWriter.class,
 				//        SVMlightStringOutcomeDataWriter.class,        
@@ -348,6 +355,20 @@ EvaluationOfTemporalRelations_ImplBase{
 		//				new File(directory,"event-event"), 
 		//				params.probabilityOfKeepingANegativeExample));
 		SimplePipeline.runPipeline(collectionReader, aggregateBuilder.createAggregate());
+        /**
+		//normalize features:
+		Iterable<Instance<String>> instances = InstanceStream.loadFromDirectory(directory);
+		// Collect MinMax stats for feature normalization
+		ZscoreNormalizationExtractor<String, Annotation> featureTransformExtractor = TemporalRelationExtractorAnnotator.createMinMaxNormalizationExtractor();
+		featureTransformExtractor.train(instances);
+		featureTransformExtractor.save(TemporalRelationExtractorAnnotator.createMinMaxNormalizationExtractorURI(directory));
+		// now write in the libsvm format
+		LibLinearStringOutcomeDataWriter dataWriter = new LibLinearStringOutcomeDataWriter(directory);
+		for (Instance<String> instance : instances) {
+			dataWriter.write(featureTransformExtractor.transform(instance));
+		}
+		dataWriter.finish();
+		*/
 		String[] optArray;
 
 		if(this.kernelParams == null){
@@ -371,7 +392,7 @@ EvaluationOfTemporalRelations_ImplBase{
 				optArray[i] = "-" + optArray[i];
 			}
 		}
-		
+
 		//calculate class-wise weights:
 		String[] weightArray=new String[TemporalRelationExtractorAnnotator.category_frequency.size()*2+4];
 		int weight_idx = 0;
@@ -422,11 +443,11 @@ EvaluationOfTemporalRelations_ImplBase{
 				CAS.NAME_DEFAULT_SOFA,
 				GOLD_VIEW_NAME);
 		//		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveNonUMLSEtEvents.class));
-		
+
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(RemoveRelations.class));
 		aggregateBuilder.add(this.baseline ? RecallBaselineEventTimeRelationAnnotator.createAnnotatorDescription(directory)
:
 			EventTimeSelfRelationAnnotator.createEngineDescription(new File(directory,"event-time")));
-		
+
 		//count how many system predicted relations, their arguments are close to each other, without
any other event in between
 		aggregateBuilder.add(AnalysisEngineFactory.createEngineDescription(CountCloseRelation.class));
 
@@ -885,11 +906,11 @@ EvaluationOfTemporalRelations_ImplBase{
 			}
 		}
 	}*/
-	
+
 	public static class CountCloseRelation extends JCasAnnotator_ImplBase {
 
 		private String systemViewName = CAS.NAME_DEFAULT_SOFA;
-		
+
 		@Override
 		public void process(JCas jCas) throws AnalysisEngineProcessException {
 			JCas systemView, goldView;
@@ -916,7 +937,7 @@ EvaluationOfTemporalRelations_ImplBase{
 					closeRelationCount++;
 				}
 			}
-			
+
 			Map<List<Annotation>, TemporalTextRelation> relationLookup = new HashMap<>();
 			for (TemporalTextRelation relation : Lists.newArrayList(JCasUtil.select(goldView, TemporalTextRelation.class)))
{
 				Annotation arg1 = relation.getArg1().getArgument();
@@ -1180,7 +1201,7 @@ EvaluationOfTemporalRelations_ImplBase{
 	 *
 	 */
 	public static class Overlap2Contains extends JCasAnnotator_ImplBase {
-		
+
 		public static final String PARAM_RELATION_VIEW = "RelationView";
 
 		@ConfigurationParameter(name = PARAM_RELATION_VIEW,mandatory=false)
@@ -1238,7 +1259,7 @@ EvaluationOfTemporalRelations_ImplBase{
 				}else{//if the relation is new, then added it to lookup
 					relationLookup.put(key, relation);
 				}
-				
+
 			}
 
 		}




Mime
View raw message